release process: split release.sh into two

This allows the add_cosigner workflow to be done before the website links to new binaries.
(so Emzy or other builders can try to reproduce builds and add signatures before new binaries are fully live)

contrib/add_cosigner

@@ -3,12 +3,15 @@
 # This script is part of the workflow for BUILDERs to reproduce and sign the
 # release binaries. (for builders who do not have sftp access to "electrum-downloads-airlock")
 #
+# env vars:
+# - SSHUSER
+#
+#
 # - BUILDER builds all binaries and checks they match the official releases
 #   (using release.sh, and perhaps some manual steps)
 # - BUILDER creates a PR against https://github.com/spesmilo/electrum-signatures/
 #   to add their sigs for a given release, which then gets merged
-# - SFTPUSER runs `$ electrum/contrib/add_cosigner $BUILDER`
+# - SFTPUSER runs `$ SSHUSER=$SFTPUSER electrum/contrib/add_cosigner $BUILDER`
-# - SFTPUSER runs `$ SSHUSER=$SFTPUSER electrum/contrib/upload.sh`
 # - SFTPUSER runs `$ electrum/contrib/make_download $WWW_DIR`
 # -     $ (cd $WWW_DIR; git commit -a -m "add_cosigner"; git push)
 # - SFTPUSER runs `$ electrum-web/publish.sh $SFTPUSER`
@@ -18,6 +21,7 @@ import re
 import os
 import sys
 import importlib
+import subprocess
 
 
 # cd to project root
@@ -32,7 +36,7 @@ ELECTRUM_VERSION = version_module.ELECTRUM_VERSION
 APK_VERSION = version_module.APK_VERSION
 print("version", ELECTRUM_VERSION)
 
-# GPG names of cosigner
+# GPG name of cosigner
 cosigner = sys.argv[1]
 
 version = version_win = version_mac = ELECTRUM_VERSION
@@ -63,3 +67,6 @@ for shortname, filename in files.items():
     os.system(f"wget -nc {sig_url} -O {sig_path}")
     if os.system(f"gpg --verify {sig_path} {path}") != 0:
         raise Exception(sig_name)
+
+print("Calling upload.sh now... This might take some time.")
+subprocess.check_output(["./contrib/upload.sh", ])

contrib/release.sh

@@ -1,23 +1,19 @@
 #!/bin/bash
 #
-# This script, for the RELEASEMANAGER:
+# This script is used for stage 1 of the release process. It operates exclusively on the airlock.
-# - builds and uploads all binaries,
+# This script, for the RELEASEMANAGER (RM):
+# - builds and uploads all binaries to airlock,
 # - assumes all keys are available, and signs everything
 # This script, for other builders:
 # - builds all reproducible binaries,
-# - downloads binaries built by the release manager, compares and signs them,
+# - downloads binaries built by the release manager (from airlock), compares and signs them,
 # - and then uploads sigs
 # Note: the .dmg should be built separately beforehand and copied into dist/
 #       (as it is built on a separate machine)
 #
+#
 # env vars:
 # - ELECBUILD_NOCACHE: if set, forces rebuild of docker images
-# - WWW_DIR: path to "electrum-web" git clone
-#
-# additional env vars for the RELEASEMANAGER:
-# - for signing the version announcement file:
-#   - ELECTRUM_SIGNING_ADDRESS (required)
-#   - ELECTRUM_SIGNING_WALLET (required)
 #
 # "uploadserver" is set in /etc/hosts
 #
@@ -29,6 +25,20 @@
 # - update RELEASE-NOTES and version.py
 # - $ git tag -s $VERSION -m $VERSION
 #
+# -----
+# Then, typical release flow:
+# - RM runs release.sh
+# - Another SFTPUSER BUILDER runs `$ ./release.sh`
+# - now airlock contains new binaries and two sigs for each
+# - deploy.sh will verify sigs and move binaries across airlock
+# - new binaries are now publicly available on uploadserver, but not linked from website yet
+# - other BUILDERS can now also try to reproduce binaries and open PRs with sigs against spesmilo/electrum-signatures
+#   - these PRs can get merged as they come
+#   - run add_cosigner
+# - after some time, RM can run release_www.sh to create and commit website-update
+#   - then run WWW_DIR/publish.sh to update website
+# - at least two people need to run WWW_DIR/publish.sh
+#
 
 set -e
 
@@ -42,10 +52,6 @@ cd "$PROJECT_ROOT"
 # rm -rf dist/*
 # rm -f .buildozer
 
-if [ -z "$WWW_DIR" ] ; then
-    WWW_DIR=/opt/electrum-web
-fi
-
 GPGUSER=$1
 if [ -z "$GPGUSER" ]; then
     fail "usage: $0 gpg_username"
@@ -247,13 +253,6 @@ else
 
     cd "$PROJECT_ROOT"
 
-    info "updating www repo"
-    ./contrib/make_download $WWW_DIR
-    info "signing the version announcement file"
-    sig=$(./run_electrum -o signmessage $ELECTRUM_SIGNING_ADDRESS $VERSION -w $ELECTRUM_SIGNING_WALLET)
-    echo "{ \"version\":\"$VERSION\", \"signatures\":{ \"$ELECTRUM_SIGNING_ADDRESS\":\"$sig\"}}" > $WWW_DIR/version
-
-
     if [ $REV != $VERSION ]; then
         fail "versions differ, not uploading"
     fi
@@ -266,14 +265,10 @@ else
         touch dist/uploaded
     fi
 
-    # push changes to website repo
-    pushd $WWW_DIR
-    git diff
-    git commit -a -m "version $VERSION"
-    git push
-    popd
 fi
 
 
 info "release.sh finished successfully."
-info "now you should run WWW_DIR/publish.sh to sign the website commit and upload signature"
+info "After two people ran release.sh, the binaries will be publicly available on uploadserver."
+info "Then, we wait for additional signers, and run add_cosigner for them."
+info "Finally, release_www.sh needs to be run, for the website to be updated."

contrib/release_www.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+#
+# env vars:
+# - WWW_DIR: path to "electrum-web" git clone
+# - for signing the version announcement file:
+#   - ELECTRUM_SIGNING_ADDRESS (required)
+#   - ELECTRUM_SIGNING_WALLET (required)
+#
+
+set -e
+
+PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/.."
+CONTRIB="$PROJECT_ROOT/contrib"
+
+cd "$PROJECT_ROOT"
+
+. "$CONTRIB"/build_tools_util.sh
+
+
+echo -n "Remember to run add_cosigner to add any additional sigs.  Continue (y/n)? "
+read answer
+if [ "$answer" != "y" ]; then
+    echo "exit"
+    exit 1
+fi
+
+
+if [ -z "$WWW_DIR" ] ; then
+    WWW_DIR=/opt/electrum-web
+fi
+
+if [ -z "$ELECTRUM_SIGNING_WALLET" ] || [ -z "$ELECTRUM_SIGNING_ADDRESS" ]; then
+    echo "You need to set env vars ELECTRUM_SIGNING_WALLET and ELECTRUM_SIGNING_ADDRESS!"
+    exit 1
+fi
+
+VERSION=$(python3 -c "import electrum; print(electrum.version.ELECTRUM_VERSION)")
+info "VERSION: $VERSION"
+
+set -x
+
+info "updating www repo"
+./contrib/make_download "$WWW_DIR"
+info "signing the version announcement file"
+sig=$(./run_electrum -o signmessage $ELECTRUM_SIGNING_ADDRESS $VERSION -w $ELECTRUM_SIGNING_WALLET)
+echo "{ \"version\":\"$VERSION\", \"signatures\":{ \"$ELECTRUM_SIGNING_ADDRESS\":\"$sig\"}}" > "$WWW_DIR"/version
+
+# push changes to website repo
+pushd "$WWW_DIR"
+git diff
+git commit -a -m "version $VERSION"
+git push
+popd
+
+
+info "release_www.sh finished successfully."
+info "now you should run WWW_DIR/publish.sh to sign the website commit and upload signature"

contrib/trigger_deploy.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Triggers deploy.sh to maybe update the website or move binaries.
+# uploadserver needs to be defined in /etc/hosts
+
+SSHUSER=$1
+TRIGGERVERSION=$2
+if [ -z $SSHUSER ] || [ -z TRIGGERVERSION ]; then
+    echo "usage: $0 SSHUSER TRIGGERVERSION"
+    echo "e.g. $0 thomasv 3.0.0"
+    echo "e.g. $0 thomasv website"
+    exit 1
+fi
+set -ex
+cd "$(dirname "$0")"
+
+if [ "$TRIGGERVERSION" == "website" ]; then
+    rm -f trigger_website
+    touch trigger_website
+    echo "uploading file: trigger_website..."
+    sftp -oBatchMode=no -b - "$SSHUSER@uploadserver" << !
+       cd electrum-downloads-airlock
+       mput trigger_website
+       bye
+!
+else
+    rm -f trigger_binaries
+    printf "$TRIGGERVERSION" > trigger_binaries
+    echo "uploading file: trigger_binaries..."
+    sftp -oBatchMode=no -b - "$SSHUSER@uploadserver" << !
+       cd electrum-downloads-airlock
+       mput trigger_binaries
+       bye
+!
+fi
+

contrib/upload.sh

@@ -5,9 +5,10 @@
 # - ELECBUILD_UPLOADFROM
 # - SSHUSER
 
-set -e
+set -ex
 
 PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/.."
+CONTRIB="$PROJECT_ROOT/contrib"
 
 if [ -z "$SSHUSER" ]; then
     SSHUSER=thomasv
@@ -15,8 +16,8 @@ fi
 
 cd "$PROJECT_ROOT"
 
-version=$(git describe --tags --abbrev=0)
+VERSION=$(python3 -c "import electrum; print(electrum.version.ELECTRUM_VERSION)")
-echo $version
+echo "$VERSION"
 
 if [ -z "$ELECBUILD_UPLOADFROM" ]; then
     cd "$PROJECT_ROOT/dist"
@@ -30,9 +31,12 @@ fi
 
 sftp -oBatchMode=no -b - "$SSHUSER@uploadserver" << !
    cd electrum-downloads-airlock
-   -mkdir "$version"
+   -mkdir "$VERSION"
-   -chmod 777 "$version"
+   -chmod 777 "$VERSION"
-   cd "$version"
+   cd "$VERSION"
-   mput *
+   -mput *
+   -chmod 444 *  # this prevents future re-uploads of same file
    bye
 !
+
+"$CONTRIB/trigger_deploy.sh" "$SSHUSER" "$VERSION"