diff --git a/.cirrus.yml b/.cirrus.yml index 9b6a859b7..f45e66804 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -208,7 +208,6 @@ task: path: "dist/*" task: - name: tarball build container: dockerfile: contrib/build-linux/sdist/Dockerfile cpu: 1 @@ -217,6 +216,11 @@ task: - ./contrib/build-linux/sdist/make_sdist.sh binaries_artifacts: path: "dist/*" + matrix: + - name: tarball build + - name: source-only tarball build + env: + OMIT_UNCLEAN_FILES: 1 task: name: Submodules diff --git a/MANIFEST.in b/MANIFEST.in index 5b208a575..2a74487c0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,8 +8,7 @@ recursive-include packages cacert.pem include contrib/requirements/requirements*.txt include contrib/deterministic-build/requirements*.txt -include contrib/make_libsecp256k1.sh -include contrib/build_tools_util.sh +include contrib/*.sh graft electrum prune electrum/tests diff --git a/contrib/build-linux/sdist/README.md b/contrib/build-linux/sdist/README.md index 96f8bb82b..abad65adf 100644 --- a/contrib/build-linux/sdist/README.md +++ b/contrib/build-linux/sdist/README.md @@ -1,5 +1,4 @@ -Source tarballs -=============== +# Source tarballs ✓ _This file should be reproducible, meaning you should be able to generate distributables that match the official releases._ @@ -7,18 +6,31 @@ Source tarballs This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another similar system. +We distribute two tarballs, a "normal" one (the default, recommended for users), +and a strictly source-only one (for Linux distro packagers). +The normal tarball, in addition to including everything from +the source-only one, also includes: +- compiled (`.mo`) locale files (in addition to source `.po` locale files) +- compiled (`_pb2.py`) protobuf files (in addition to source `.proto` files) +- the `packages/` folder containing source-only pure-python runtime dependencies + + +## Build steps + 1. Install Docker See `contrib/docker_notes.md`. -2. Build source tarball +2. Build tarball + (set envvar `OMIT_UNCLEAN_FILES=1` to build the "source-only" tarball) ``` $ ./build.sh ``` If you want reproducibility, try instead e.g.: ``` $ ELECBUILD_COMMIT=HEAD ELECBUILD_NOCACHE=1 ./build.sh + $ ELECBUILD_COMMIT=HEAD ELECBUILD_NOCACHE=1 OMIT_UNCLEAN_FILES=1 ./build.sh ``` 3. The generated distributables are in `./dist`. diff --git a/contrib/build-linux/sdist/build.sh b/contrib/build-linux/sdist/build.sh index ed969cbdf..361dadcd6 100755 --- a/contrib/build-linux/sdist/build.sh +++ b/contrib/build-linux/sdist/build.sh @@ -47,6 +47,7 @@ docker run -it \ -v "$PROJECT_ROOT_OR_FRESHCLONE_ROOT":/opt/electrum \ --rm \ --workdir /opt/electrum/contrib/build-linux/sdist \ + --env OMIT_UNCLEAN_FILES \ electrum-sdist-builder-img \ ./make_sdist.sh diff --git a/contrib/build-linux/sdist/make_sdist.sh b/contrib/build-linux/sdist/make_sdist.sh index 5429ec74e..03b872910 100755 --- a/contrib/build-linux/sdist/make_sdist.sh +++ b/contrib/build-linux/sdist/make_sdist.sh @@ -6,7 +6,7 @@ PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../../.." CONTRIB="$PROJECT_ROOT/contrib" CONTRIB_SDIST="$CONTRIB/build-linux/sdist" DISTDIR="$PROJECT_ROOT/dist" -LOCALE="$PROJECT_ROOT/electrum/locale/" +LOCALE="$PROJECT_ROOT/electrum/locale" . "$CONTRIB"/build_tools_util.sh @@ -19,35 +19,55 @@ break_legacy_easy_install # (make_packages will later install a pinned version of pip in a venv) python3 -m pip install --upgrade pip -"$CONTRIB"/make_packages || fail "make_packages failed" +rm -rf "$PROJECT_ROOT/packages/" +if ([ "$OMIT_UNCLEAN_FILES" != 1 ]); then + "$CONTRIB"/make_packages || fail "make_packages failed" +fi git submodule update --init ( - cd "$CONTRIB/deterministic-build/electrum-locale/" - if ! which msgfmt > /dev/null 2>&1; then - echo "Please install gettext" - exit 1 - fi - # We include both source (.po) and compiled (.mo) locale files in the source dist. - # Maybe we should exclude the compiled locale files? see https://askubuntu.com/a/144139 - # (also see MANIFEST.in) + # By default, include both source (.po) and compiled (.mo) locale files in the source dist. + # Set option OMIT_UNCLEAN_FILES=1 to exclude the compiled locale files + # see https://askubuntu.com/a/144139 (also see MANIFEST.in) rm -rf "$LOCALE" - for i in ./locale/*; do - dir="$PROJECT_ROOT/electrum/$i/LC_MESSAGES" - mkdir -p "$dir" - msgfmt --output-file="$dir/electrum.mo" "$i/electrum.po" || true - cp $i/electrum.po "$PROJECT_ROOT/electrum/$i/electrum.po" - done + cp -r "$CONTRIB/deterministic-build/electrum-locale/locale/" "$LOCALE/" + if ([ "$OMIT_UNCLEAN_FILES" != 1 ]); then + "$CONTRIB/build_locale.sh" "$LOCALE" + fi ) +if ([ "$OMIT_UNCLEAN_FILES" = 1 ]); then + # FIXME side-effecting repo... though in practice, this script probably runs in fresh_clone + rm -f "$PROJECT_ROOT/electrum/paymentrequest_pb2.py" +fi + ( cd "$PROJECT_ROOT" find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} + # note: .zip sdists would not be reproducible due to https://bugs.python.org/issue40963 - TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=gztar + if ([ "$OMIT_UNCLEAN_FILES" = 1 ]) + then PY_DISTDIR="dist/_sourceonly" # The DISTDIR variable of this script is only used to find where the output is *finally* placed. + else PY_DISTDIR="dist" + fi + TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=gztar --dist-dir="$PY_DISTDIR" + if ([ "$OMIT_UNCLEAN_FILES" = 1 ]); then + python3 < /dev/null 2>&1; then + echo "Please install gettext" + exit 1 +fi + +for i in "$1/"*; do + mkdir -p "$i/LC_MESSAGES" + (msgfmt --output-file="$i/LC_MESSAGES/electrum.mo" "$i/electrum.po" || true) +done diff --git a/contrib/generate_payreqpb2.sh b/contrib/generate_payreqpb2.sh new file mode 100755 index 000000000..87addbb36 --- /dev/null +++ b/contrib/generate_payreqpb2.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Generates the file paymentrequest_pb2.py + +CONTRIB="$(dirname "$(readlink -e "$0")")" +EL="$CONTRIB"/../electrum + +if ! which protoc > /dev/null 2>&1; then + echo "Please install 'protoc'" + echo "If you're on Debian, try 'sudo apt install protobuf-compiler'?" + exit 1 +fi + +protoc --proto_path="$EL" --python_out="$EL" "$EL"/paymentrequest.proto diff --git a/contrib/release.sh b/contrib/release.sh index 613b4d2f6..04664bbdd 100755 --- a/contrib/release.sh +++ b/contrib/release.sh @@ -88,6 +88,14 @@ else ./contrib/build-linux/sdist/build.sh fi +# create source-only tarball +srctarball="Electrum-sourceonly-$VERSION.tar.gz" +if test -f "dist/$srctarball"; then + info "file exists: $srctarball" +else + OMIT_UNCLEAN_FILES=1 ./contrib/build-linux/sdist/build.sh +fi + # appimage appimage="electrum-$REV-x86_64.AppImage" if test -f "dist/$appimage"; then @@ -186,15 +194,17 @@ if [ -z "$RELEASEMANAGER" ] ; then bye ! # check we have each binary - test -f "$tarball" || fail "tarball not found among sftp downloads" - test -f "$appimage" || fail "appimage not found among sftp downloads" - test -f "$win1" || fail "win1 not found among sftp downloads" - test -f "$win2" || fail "win2 not found among sftp downloads" - test -f "$win3" || fail "win3 not found among sftp downloads" - test -f "$apk1" || fail "apk1 not found among sftp downloads" - test -f "$apk2" || fail "apk2 not found among sftp downloads" - test -f "$dmg" || fail "dmg not found among sftp downloads" + test -f "$tarball" || fail "tarball not found among sftp downloads" + test -f "$srctarball" || fail "srctarball not found among sftp downloads" + test -f "$appimage" || fail "appimage not found among sftp downloads" + test -f "$win1" || fail "win1 not found among sftp downloads" + test -f "$win2" || fail "win2 not found among sftp downloads" + test -f "$win3" || fail "win3 not found among sftp downloads" + test -f "$apk1" || fail "apk1 not found among sftp downloads" + test -f "$apk2" || fail "apk2 not found among sftp downloads" + test -f "$dmg" || fail "dmg not found among sftp downloads" test -f "$PROJECT_ROOT/dist/$tarball" || fail "tarball not found among built files" + test -f "$PROJECT_ROOT/dist/$srctarball" || fail "srctarball not found among built files" test -f "$PROJECT_ROOT/dist/$appimage" || fail "appimage not found among built files" test -f "$CONTRIB/build-wine/dist/$win1" || fail "win1 not found among built files" test -f "$CONTRIB/build-wine/dist/$win2" || fail "win2 not found among built files" @@ -203,8 +213,9 @@ if [ -z "$RELEASEMANAGER" ] ; then test -f "$PROJECT_ROOT/dist/$apk2" || fail "apk2 not found among built files" test -f "$PROJECT_ROOT/dist/$dmg" || fail "dmg not found among built files" # compare downloaded binaries against ones we built - cmp --silent "$tarball" "$PROJECT_ROOT/dist/$tarball" || fail "files are different. tarball." - cmp --silent "$appimage" "$PROJECT_ROOT/dist/$appimage" || fail "files are different. appimage." + cmp --silent "$tarball" "$PROJECT_ROOT/dist/$tarball" || fail "files are different. tarball." + cmp --silent "$srctarball" "$PROJECT_ROOT/dist/$srctarball" || fail "files are different. srctarball." + cmp --silent "$appimage" "$PROJECT_ROOT/dist/$appimage" || fail "files are different. appimage." rm -rf "$CONTRIB/build-wine/signed/" && mkdir --parents "$CONTRIB/build-wine/signed/" cp -f "$win1" "$win2" "$win3" "$CONTRIB/build-wine/signed/" "$CONTRIB/build-wine/unsign.sh" || fail "files are different. windows." @@ -214,7 +225,7 @@ if [ -z "$RELEASEMANAGER" ] ; then # all files matched. sign them. rm -rf "$PROJECT_ROOT/dist/sigs/" mkdir --parents "$PROJECT_ROOT/dist/sigs/" - for fname in "$tarball" "$appimage" "$win1" "$win2" "$win3" "$apk1" "$apk2" "$dmg" ; do + for fname in "$tarball" "$srctarball" "$appimage" "$win1" "$win2" "$win3" "$apk1" "$apk2" "$dmg" ; do signame="$fname.$GPGUSER.asc" gpg --sign --armor --detach $PUBKEY --output "$PROJECT_ROOT/dist/sigs/$signame" "$fname" done diff --git a/electrum/paymentrequest.py b/electrum/paymentrequest.py index acf9c8a32..043f7352f 100644 --- a/electrum/paymentrequest.py +++ b/electrum/paymentrequest.py @@ -36,8 +36,7 @@ import aiohttp try: from . import paymentrequest_pb2 as pb2 except ImportError: - # sudo apt-get install protobuf-compiler - sys.exit("Error: could not find paymentrequest_pb2.py. Create it with 'protoc --proto_path=electrum/ --python_out=electrum/ electrum/paymentrequest.proto'") + sys.exit("Error: could not find paymentrequest_pb2.py. Create it with 'contrib/generate_payreqpb2.sh'") from . import bitcoin, constants, ecc, util, transaction, x509, rsakey from .util import bh2u, bfh, make_aiohttp_session