Merge pull request #6228 from SomberNight/202006_reproducible_targz

make targz sdist reproducible, rm zip

.travis.yml

@@ -99,6 +99,26 @@ jobs:
       script:
         - sudo docker run --name electrum-appimage-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/appimage electrum-appimage-builder-img ./build.sh
       after_success: true
+    - if: branch = master
+      name: "tarball build"
+      language: c
+      python: false
+      services:
+        - docker
+      before_install:
+        # hack: travis already cloned the repo, but we re-clone now, as we need to have umask set BEFORE cloning
+        - umask 0022
+        - mkdir fresh_clone && cd fresh_clone
+        - git clone https://github.com/$TRAVIS_REPO_SLUG.git && cd electrum
+        - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then git fetch origin pull/$TRAVIS_PULL_REQUEST/merge; fi
+        - git checkout $TRAVIS_COMMIT
+        - echo "Second git clone ready at $PWD"
+      install:
+        - sudo docker build --no-cache -t electrum-sdist-builder-img ./contrib/build-linux/sdist/
+      script:
+        - echo "Building sdist at $PWD"
+        - sudo docker run --name electrum-sdist-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/sdist electrum-sdist-builder-img ./build.sh
+      after_success: true
     - stage: release check
       install:
           - git fetch --all --tags

README.rst

@@ -121,7 +121,7 @@ Creating Binaries
 Linux (tarball)
 ---------------
 
-See :code:`contrib/build-linux/README.md`.
+See :code:`contrib/build-linux/sdist/README.md`.
 
 
 Linux (AppImage)

contrib/build-linux/README.md

@@ -1,16 +0,0 @@
-Source tarballs
-===============
-
-✗ _This script does not produce reproducible output (yet!)._
-
-1. Prepare python dependencies used by Electrum.
-
-    ```
-    contrib/make_packages
-    ```
-
-2. Create source tarball.
-
-    ```
-    contrib/make_tgz
-    ```

contrib/build-linux/appimage/README.md

@@ -61,6 +61,11 @@ diff sha256sum1 sha256sum2 > d
 cat d
 ```
 
+For file metadata, e.g. timestamps:
+```
+rsync -n -a -i --delete squashfs-root1/ squashfs-root2/
+```
+
 Useful binary comparison tools:
 - vbindiff
 - diffoscope

contrib/build-linux/sdist/Dockerfile

@@ -0,0 +1,17 @@
+FROM ubuntu:20.04@sha256:5747316366b8cc9e3021cd7286f42b2d6d81e3d743e2ab571f55bcd5df788cc8
+
+ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update -q && \
+    apt-get install -qy \
+        git \
+        gettext \
+        python3 \
+        python3-pip \
+        python3-setuptools \
+        faketime \
+        && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get autoremove -y && \
+    apt-get clean

contrib/build-linux/sdist/README.md

@@ -0,0 +1,52 @@
+Source tarballs
+===============
+
+✓ _This file should be reproducible, meaning you should be able to generate
+   distributables that match the official releases._
+
+This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another
+similar system. The docker commands should be executed in the project's root
+folder.
+
+1. Install Docker
+
+    ```
+    $ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+    $ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+    $ sudo apt-get update
+    $ sudo apt-get install -y docker-ce
+    ```
+
+2. Build image
+
+    ```
+    $ sudo docker build -t electrum-sdist-builder-img contrib/build-linux/sdist
+    ```
+
+3. Build source tarballs
+
+    It's recommended to build from a fresh clone
+    (but you can skip this if reproducibility is not necessary).
+
+    ```
+    $ FRESH_CLONE=contrib/build-linux/sdist/fresh_clone && \
+        sudo rm -rf $FRESH_CLONE && \
+        umask 0022 && \
+        mkdir -p $FRESH_CLONE && \
+        cd $FRESH_CLONE  && \
+        git clone https://github.com/spesmilo/electrum.git && \
+        cd electrum
+    ```
+
+    And then build from this directory:
+    ```
+    $ git checkout $REV
+    $ sudo docker run -it \
+        --name electrum-sdist-builder-cont \
+        -v $PWD:/opt/electrum \
+        --rm \
+        --workdir /opt/electrum/contrib/build-linux/sdist \
+        electrum-sdist-builder-img \
+        ./build.sh
+    ```
+4. The generated distributables are in `./dist`.

contrib/build-linux/sdist/build.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -e
+
+PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../../.."
+CONTRIB="$PROJECT_ROOT/contrib"
+CONTRIB_SDIST="$CONTRIB/build-linux/sdist"
+DISTDIR="$PROJECT_ROOT/dist"
+
+. "$CONTRIB"/build_tools_util.sh
+
+# note that at least py3.7 is needed, to have https://bugs.python.org/issue30693
+python3 --version || fail "python interpreter not found"
+
+# upgrade to modern pip so that it knows the flags we need.
+# we will then install a pinned version of pip as part of requirements-sdist-build
+python3 -m pip install --upgrade pip
+
+info "Installing pinned requirements."
+python3 -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-sdist-build.txt
+
+
+"$CONTRIB"/make_packages || fail "make_packages failed"
+
+"$CONTRIB_SDIST"/make_tgz || fail "make_tgz failed"
+
+
+info "done."
+ls -la "$DISTDIR"
+sha256sum "$DISTDIR"/*

contrib/build-linux/sdist/make_tgz

@@ -2,7 +2,8 @@
 
 set -e
 
-CONTRIB="$(dirname "$(readlink -e "$0")")"
+CONTRIB_SDIST="$(dirname "$(readlink -e "$0")")"
+CONTRIB="$CONTRIB_SDIST"/../..
 ROOT_FOLDER="$CONTRIB"/..
 PACKAGES="$ROOT_FOLDER"/packages/
 LOCALE="$ROOT_FOLDER"/electrum/locale/
@@ -39,5 +40,8 @@ git submodule update --init
     # we could build the kivy atlas potentially?
     #(cd electrum/gui/kivy/; make theming) || echo "building kivy atlas failed! skipping."
 
-    python3 setup.py --quiet sdist --format=zip,gztar
+    find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} +
+
+    # note: .zip sdists would not be reproducible due to https://bugs.python.org/issue40963
+    TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=gztar
 )

contrib/deterministic-build/requirements-sdist-build.txt

@@ -0,0 +1,9 @@
+pip==20.1.1 \
+    --hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \
+    --hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4
+setuptools==46.4.0 \
+    --hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \
+    --hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8
+wheel==0.34.2 \
+    --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \
+    --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e

contrib/freeze_packages.sh

@@ -20,7 +20,7 @@ which virtualenv > /dev/null 2>&1 || { echo "Please install virtualenv" && exit
 
 ${SYSTEM_PYTHON} -m hashin -h > /dev/null 2>&1 || { ${SYSTEM_PYTHON} -m pip install hashin; }
 
-for i in '' '-hw' '-binaries' '-wine-build' '-mac-build'; do
+for i in '' '-hw' '-binaries' '-wine-build' '-mac-build' '-sdist-build'; do
     rm -rf "$venv_dir"
     virtualenv -p ${SYSTEM_PYTHON} $venv_dir
 

contrib/make_packages

@@ -6,5 +6,6 @@ test -n "$CONTRIB" -a -d "$CONTRIB" || exit
 rm "$CONTRIB"/../packages/ -r
 
 #Install pure python modules in electrum directory
-python3 -m pip install -r "$CONTRIB"/deterministic-build/requirements.txt -t "$CONTRIB"/../packages
+python3 -m pip install --no-dependencies --no-binary :all: \
+    -r "$CONTRIB"/deterministic-build/requirements.txt -t "$CONTRIB"/../packages
 

contrib/requirements/requirements-sdist-build.txt

@@ -0,0 +1,3 @@
+# need modern versions of pip (and maybe other build tools), the one in apt had issues
+pip
+setuptools