From 45b8f11c68a170af74106d33f3604b7cbb9734f3 Mon Sep 17 00:00:00 2001 From: yanmaani Date: Fri, 10 Dec 2021 12:00:00 +0000 Subject: [PATCH 1/7] build: refactor out locale generation into build_locale.sh --- contrib/build-linux/sdist/make_sdist.sh | 15 +++------------ contrib/build_locale.sh | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 12 deletions(-) create mode 100755 contrib/build_locale.sh diff --git a/contrib/build-linux/sdist/make_sdist.sh b/contrib/build-linux/sdist/make_sdist.sh index 5429ec74e..d05a8a414 100755 --- a/contrib/build-linux/sdist/make_sdist.sh +++ b/contrib/build-linux/sdist/make_sdist.sh @@ -6,7 +6,7 @@ PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../../.." CONTRIB="$PROJECT_ROOT/contrib" CONTRIB_SDIST="$CONTRIB/build-linux/sdist" DISTDIR="$PROJECT_ROOT/dist" -LOCALE="$PROJECT_ROOT/electrum/locale/" +LOCALE="$PROJECT_ROOT/electrum/locale" . "$CONTRIB"/build_tools_util.sh @@ -24,21 +24,12 @@ python3 -m pip install --upgrade pip git submodule update --init ( - cd "$CONTRIB/deterministic-build/electrum-locale/" - if ! which msgfmt > /dev/null 2>&1; then - echo "Please install gettext" - exit 1 - fi # We include both source (.po) and compiled (.mo) locale files in the source dist. # Maybe we should exclude the compiled locale files? see https://askubuntu.com/a/144139 # (also see MANIFEST.in) rm -rf "$LOCALE" - for i in ./locale/*; do - dir="$PROJECT_ROOT/electrum/$i/LC_MESSAGES" - mkdir -p "$dir" - msgfmt --output-file="$dir/electrum.mo" "$i/electrum.po" || true - cp $i/electrum.po "$PROJECT_ROOT/electrum/$i/electrum.po" - done + cp -r "$CONTRIB/deterministic-build/electrum-locale/locale/" "$LOCALE/" + "$CONTRIB/build_locale.sh" "$LOCALE" ) ( diff --git a/contrib/build_locale.sh b/contrib/build_locale.sh new file mode 100755 index 000000000..d3157cafc --- /dev/null +++ b/contrib/build_locale.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +if [ ! -d "$1" ]; then + echo "usage: $0 path/to/locale" + exit 1 +fi + +if ! which msgfmt > /dev/null 2>&1; then + echo "Please install gettext" + exit 1 +fi + +for i in "$1/"*; do + mkdir -p "$i/LC_MESSAGES" + (msgfmt --output-file="$i/LC_MESSAGES/electrum.mo" "$i/electrum.po" || true) +done From a7e17ae4eef53a48113285ef2b93411334f3c011 Mon Sep 17 00:00:00 2001 From: yanmaani Date: Fri, 10 Dec 2021 12:00:00 +0000 Subject: [PATCH 2/7] build: add OMIT_UNCLEAN_FILES option to make_sdist.sh --- contrib/build-linux/sdist/make_sdist.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/contrib/build-linux/sdist/make_sdist.sh b/contrib/build-linux/sdist/make_sdist.sh index d05a8a414..b0d17b4b2 100755 --- a/contrib/build-linux/sdist/make_sdist.sh +++ b/contrib/build-linux/sdist/make_sdist.sh @@ -19,19 +19,27 @@ break_legacy_easy_install # (make_packages will later install a pinned version of pip in a venv) python3 -m pip install --upgrade pip -"$CONTRIB"/make_packages || fail "make_packages failed" +if ([ "$OMIT_UNCLEAN_FILES" != 1 ]); then + "$CONTRIB"/make_packages || fail "make_packages failed" +fi git submodule update --init ( - # We include both source (.po) and compiled (.mo) locale files in the source dist. - # Maybe we should exclude the compiled locale files? see https://askubuntu.com/a/144139 - # (also see MANIFEST.in) + # By default, include both source (.po) and compiled (.mo) locale files in the source dist. + # Set option OMIT_UNCLEAN_FILES=1 to exclude the compiled locale files + # see https://askubuntu.com/a/144139 (also see MANIFEST.in) rm -rf "$LOCALE" cp -r "$CONTRIB/deterministic-build/electrum-locale/locale/" "$LOCALE/" - "$CONTRIB/build_locale.sh" "$LOCALE" + if ([ "$OMIT_UNCLEAN_FILES" != 1 ]); then + "$CONTRIB/build_locale.sh" "$LOCALE" + fi ) +if ([ "$OMIT_UNCLEAN_FILES" = 1 ]); then + rm "$PROJECT_ROOT/electrum/paymentrequest_pb2.py" +fi + ( cd "$PROJECT_ROOT" From 2da1859110d86b18db2f3b30e5d4ffaa34c496a9 Mon Sep 17 00:00:00 2001 From: yanmaani Date: Fri, 10 Dec 2021 12:00:00 +0000 Subject: [PATCH 3/7] build: support OMIT_UNCLEAN_FILES in sdist/build.sh --- contrib/build-linux/sdist/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/build-linux/sdist/build.sh b/contrib/build-linux/sdist/build.sh index ed969cbdf..361dadcd6 100755 --- a/contrib/build-linux/sdist/build.sh +++ b/contrib/build-linux/sdist/build.sh @@ -47,6 +47,7 @@ docker run -it \ -v "$PROJECT_ROOT_OR_FRESHCLONE_ROOT":/opt/electrum \ --rm \ --workdir /opt/electrum/contrib/build-linux/sdist \ + --env OMIT_UNCLEAN_FILES \ electrum-sdist-builder-img \ ./make_sdist.sh From 04b81d6b824cd82a7a951635eeeb89cdc39ce6ac Mon Sep 17 00:00:00 2001 From: yanmaani Date: Fri, 10 Dec 2021 12:00:00 +0000 Subject: [PATCH 4/7] contrib: add generate_payreqpb2.sh script --- contrib/generate_payreqpb2.sh | 13 +++++++++++++ electrum/paymentrequest.py | 3 +-- 2 files changed, 14 insertions(+), 2 deletions(-) create mode 100755 contrib/generate_payreqpb2.sh diff --git a/contrib/generate_payreqpb2.sh b/contrib/generate_payreqpb2.sh new file mode 100755 index 000000000..87addbb36 --- /dev/null +++ b/contrib/generate_payreqpb2.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Generates the file paymentrequest_pb2.py + +CONTRIB="$(dirname "$(readlink -e "$0")")" +EL="$CONTRIB"/../electrum + +if ! which protoc > /dev/null 2>&1; then + echo "Please install 'protoc'" + echo "If you're on Debian, try 'sudo apt install protobuf-compiler'?" + exit 1 +fi + +protoc --proto_path="$EL" --python_out="$EL" "$EL"/paymentrequest.proto diff --git a/electrum/paymentrequest.py b/electrum/paymentrequest.py index acf9c8a32..043f7352f 100644 --- a/electrum/paymentrequest.py +++ b/electrum/paymentrequest.py @@ -36,8 +36,7 @@ import aiohttp try: from . import paymentrequest_pb2 as pb2 except ImportError: - # sudo apt-get install protobuf-compiler - sys.exit("Error: could not find paymentrequest_pb2.py. Create it with 'protoc --proto_path=electrum/ --python_out=electrum/ electrum/paymentrequest.proto'") + sys.exit("Error: could not find paymentrequest_pb2.py. Create it with 'contrib/generate_payreqpb2.sh'") from . import bitcoin, constants, ecc, util, transaction, x509, rsakey from .util import bh2u, bfh, make_aiohttp_session From 501c06559c2a060ed7c455b3f28c237ce0c414ae Mon Sep 17 00:00:00 2001 From: yanmaani Date: Tue, 4 Jan 2022 12:00:00 +0000 Subject: [PATCH 5/7] ci: make source-only tarballs in Cirrus --- .cirrus.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 9b6a859b7..f45e66804 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -208,7 +208,6 @@ task: path: "dist/*" task: - name: tarball build container: dockerfile: contrib/build-linux/sdist/Dockerfile cpu: 1 @@ -217,6 +216,11 @@ task: - ./contrib/build-linux/sdist/make_sdist.sh binaries_artifacts: path: "dist/*" + matrix: + - name: tarball build + - name: source-only tarball build + env: + OMIT_UNCLEAN_FILES: 1 task: name: Submodules From 9a37184a57ab80cde0faf75f05100dfd273832e4 Mon Sep 17 00:00:00 2001 From: yanmaani Date: Tue, 4 Jan 2022 12:00:00 +0000 Subject: [PATCH 6/7] build: create source-only tarball in release.sh --- contrib/build-linux/sdist/make_sdist.sh | 20 ++++++++++++++- contrib/release.sh | 33 ++++++++++++++++--------- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/contrib/build-linux/sdist/make_sdist.sh b/contrib/build-linux/sdist/make_sdist.sh index b0d17b4b2..f6381e07e 100755 --- a/contrib/build-linux/sdist/make_sdist.sh +++ b/contrib/build-linux/sdist/make_sdist.sh @@ -46,7 +46,25 @@ fi find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} + # note: .zip sdists would not be reproducible due to https://bugs.python.org/issue40963 - TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=gztar + if ([ "$OMIT_UNCLEAN_FILES" = 1 ]) + then PY_DISTDIR="dist/_sourceonly" # The DISTDIR variable of this script is only used to find where the output is *finally* placed. + else PY_DISTDIR="dist" + fi + TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=gztar --dist-dir="$PY_DISTDIR" + if ([ "$OMIT_UNCLEAN_FILES" = 1 ]); then + for fn in "$DISTDIR/_sourceonly/"*; do + # Since ELECTRUM_VERSION is not available to us in this script, we have to use a regex. + # Expression 1: Electrum-X.Y.Z.tar.gz -> Electrum-sourceonly-X.Y.Z.tar.gz + # Capture group \1 = Electrum + # Capture group \2 = X.Y.Z.tar.gz + # Expression 2: dist/_sourceonly/X.tar.gz -> dist/X.tar.gz + mv "$fn" $(sed \ + -e 's/\(.*\)-\([^-]*\)/\1-sourceonly-\2/' \ + -e 's/\/_sourceonly//' \ + <<< "$fn") + done + rmdir "$PY_DISTDIR" + fi ) diff --git a/contrib/release.sh b/contrib/release.sh index 76762e29a..351ca1dfe 100755 --- a/contrib/release.sh +++ b/contrib/release.sh @@ -88,6 +88,14 @@ else ./contrib/build-linux/sdist/build.sh fi +# create source-only tarball +srctarball="Electrum-sourceonly-$VERSION.tar.gz" +if test -f "dist/$srctarball"; then + info "file exists: $srctarball" +else + OMIT_UNCLEAN_FILES=1 ./contrib/build-linux/sdist/build.sh +fi + # appimage appimage="electrum-$REV-x86_64.AppImage" if test -f "dist/$appimage"; then @@ -186,15 +194,17 @@ if [ -z "$RELEASEMANAGER" ] ; then bye ! # check we have each binary - test -f "$tarball" || fail "tarball not found among sftp downloads" - test -f "$appimage" || fail "appimage not found among sftp downloads" - test -f "$win1" || fail "win1 not found among sftp downloads" - test -f "$win2" || fail "win2 not found among sftp downloads" - test -f "$win3" || fail "win3 not found among sftp downloads" - test -f "$apk1" || fail "apk1 not found among sftp downloads" - test -f "$apk2" || fail "apk2 not found among sftp downloads" - test -f "$dmg" || fail "dmg not found among sftp downloads" + test -f "$tarball" || fail "tarball not found among sftp downloads" + test -f "$srctarball" || fail "srctarball not found among sftp downloads" + test -f "$appimage" || fail "appimage not found among sftp downloads" + test -f "$win1" || fail "win1 not found among sftp downloads" + test -f "$win2" || fail "win2 not found among sftp downloads" + test -f "$win3" || fail "win3 not found among sftp downloads" + test -f "$apk1" || fail "apk1 not found among sftp downloads" + test -f "$apk2" || fail "apk2 not found among sftp downloads" + test -f "$dmg" || fail "dmg not found among sftp downloads" test -f "$PROJECT_ROOT/dist/$tarball" || fail "tarball not found among built files" + test -f "$PROJECT_ROOT/dist/$srctarball" || fail "tarball not found among built files" test -f "$PROJECT_ROOT/dist/$appimage" || fail "appimage not found among built files" test -f "$CONTRIB/build-wine/dist/$win1" || fail "win1 not found among built files" test -f "$CONTRIB/build-wine/dist/$win2" || fail "win2 not found among built files" @@ -203,8 +213,9 @@ if [ -z "$RELEASEMANAGER" ] ; then test -f "$PROJECT_ROOT/dist/$apk2" || fail "apk2 not found among built files" test -f "$PROJECT_ROOT/dist/$dmg" || fail "dmg not found among built files" # compare downloaded binaries against ones we built - cmp --silent "$tarball" "$PROJECT_ROOT/dist/$tarball" || fail "files are different. tarball." - cmp --silent "$appimage" "$PROJECT_ROOT/dist/$appimage" || fail "files are different. appimage." + cmp --silent "$tarball" "$PROJECT_ROOT/dist/$tarball" || fail "files are different. tarball." + cmp --silent "$srctarball" "$PROJECT_ROOT/dist/$srctarball" || fail "files are different. srctarball." + cmp --silent "$appimage" "$PROJECT_ROOT/dist/$appimage" || fail "files are different. appimage." rm -rf "$CONTRIB/build-wine/signed/" && mkdir --parents "$CONTRIB/build-wine/signed/" cp -f "$win1" "$win2" "$win3" "$CONTRIB/build-wine/signed/" "$CONTRIB/build-wine/unsign.sh" || fail "files are different. windows." @@ -214,7 +225,7 @@ if [ -z "$RELEASEMANAGER" ] ; then # all files matched. sign them. rm -rf "$PROJECT_ROOT/dist/sigs/" mkdir --parents "$PROJECT_ROOT/dist/sigs/" - for fname in "$tarball" "$appimage" "$win1" "$win2" "$win3" "$apk1" "$apk2" "$dmg" ; do + for fname in "$tarball" "$srctarball" "$appimage" "$win1" "$win2" "$win3" "$apk1" "$apk2" "$dmg" ; do signame="$fname.$GPGUSER.asc" gpg --sign --armor --detach $PUBKEY --output "$PROJECT_ROOT/dist/sigs/$signame" "$fname" done From 7d36f2ba3ab603044556b9548bea9893520747e1 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 16 Aug 2022 15:56:53 +0000 Subject: [PATCH 7/7] build: follow-up source-only tarballs --- MANIFEST.in | 3 +-- contrib/build-linux/sdist/README.md | 18 ++++++++++++++--- contrib/build-linux/sdist/make_sdist.sh | 27 ++++++++++++++----------- contrib/release.sh | 2 +- 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 5b208a575..2a74487c0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,8 +8,7 @@ recursive-include packages cacert.pem include contrib/requirements/requirements*.txt include contrib/deterministic-build/requirements*.txt -include contrib/make_libsecp256k1.sh -include contrib/build_tools_util.sh +include contrib/*.sh graft electrum prune electrum/tests diff --git a/contrib/build-linux/sdist/README.md b/contrib/build-linux/sdist/README.md index 96f8bb82b..abad65adf 100644 --- a/contrib/build-linux/sdist/README.md +++ b/contrib/build-linux/sdist/README.md @@ -1,5 +1,4 @@ -Source tarballs -=============== +# Source tarballs ✓ _This file should be reproducible, meaning you should be able to generate distributables that match the official releases._ @@ -7,18 +6,31 @@ Source tarballs This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another similar system. +We distribute two tarballs, a "normal" one (the default, recommended for users), +and a strictly source-only one (for Linux distro packagers). +The normal tarball, in addition to including everything from +the source-only one, also includes: +- compiled (`.mo`) locale files (in addition to source `.po` locale files) +- compiled (`_pb2.py`) protobuf files (in addition to source `.proto` files) +- the `packages/` folder containing source-only pure-python runtime dependencies + + +## Build steps + 1. Install Docker See `contrib/docker_notes.md`. -2. Build source tarball +2. Build tarball + (set envvar `OMIT_UNCLEAN_FILES=1` to build the "source-only" tarball) ``` $ ./build.sh ``` If you want reproducibility, try instead e.g.: ``` $ ELECBUILD_COMMIT=HEAD ELECBUILD_NOCACHE=1 ./build.sh + $ ELECBUILD_COMMIT=HEAD ELECBUILD_NOCACHE=1 OMIT_UNCLEAN_FILES=1 ./build.sh ``` 3. The generated distributables are in `./dist`. diff --git a/contrib/build-linux/sdist/make_sdist.sh b/contrib/build-linux/sdist/make_sdist.sh index f6381e07e..03b872910 100755 --- a/contrib/build-linux/sdist/make_sdist.sh +++ b/contrib/build-linux/sdist/make_sdist.sh @@ -19,6 +19,7 @@ break_legacy_easy_install # (make_packages will later install a pinned version of pip in a venv) python3 -m pip install --upgrade pip +rm -rf "$PROJECT_ROOT/packages/" if ([ "$OMIT_UNCLEAN_FILES" != 1 ]); then "$CONTRIB"/make_packages || fail "make_packages failed" fi @@ -37,7 +38,8 @@ git submodule update --init ) if ([ "$OMIT_UNCLEAN_FILES" = 1 ]); then - rm "$PROJECT_ROOT/electrum/paymentrequest_pb2.py" + # FIXME side-effecting repo... though in practice, this script probably runs in fresh_clone + rm -f "$PROJECT_ROOT/electrum/paymentrequest_pb2.py" fi ( @@ -52,17 +54,18 @@ fi fi TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=gztar --dist-dir="$PY_DISTDIR" if ([ "$OMIT_UNCLEAN_FILES" = 1 ]); then - for fn in "$DISTDIR/_sourceonly/"*; do - # Since ELECTRUM_VERSION is not available to us in this script, we have to use a regex. - # Expression 1: Electrum-X.Y.Z.tar.gz -> Electrum-sourceonly-X.Y.Z.tar.gz - # Capture group \1 = Electrum - # Capture group \2 = X.Y.Z.tar.gz - # Expression 2: dist/_sourceonly/X.tar.gz -> dist/X.tar.gz - mv "$fn" $(sed \ - -e 's/\(.*\)-\([^-]*\)/\1-sourceonly-\2/' \ - -e 's/\/_sourceonly//' \ - <<< "$fn") - done + python3 <