build: create a standalone build script for libsecp256k1

heavily based on Electron-Cash/Electron-Cash@eda015908e9d6ea9a0adfbda9db55b929c0926ba
6 years ago · 4cec098d2d
11 changed files with 146 additions and 105 deletions
--- a/MANIFEST.in
+++ b/MANIFEST.in
@ -12,6 +12,9 @@ graft electrum
 prune electrum/tests
 graft contrib/udev

+exclude electrum/*.so
+exclude electrum/*.so.0
+
 global-exclude __pycache__
 global-exclude *.py[co~]
 global-exclude *.py.orig
--- a/contrib/build-linux/appimage/build.sh
+++ b/contrib/build-linux/appimage/build.sh
@ -10,10 +10,11 @@ BUILDDIR="$CONTRIB_APPIMAGE/build/appimage"
 APPDIR="$BUILDDIR/electrum.AppDir"
 CACHEDIR="$CONTRIB_APPIMAGE/.cache/appimage"

+export GCC_STRIP_BINARIES="1"
+
 # pinned versions
 PYTHON_VERSION=3.7.6
 PKG2APPIMAGE_COMMIT="eb8f3acdd9f11ab19b78f5cb15daa772367daf15"
-LIBSECP_VERSION="b408c6a8b287003d1ade5709e6f7bc3c7f1d5be7"
 SQUASHFSKIT_COMMIT="ae0d656efa2d0df2fcac795b6823b44462f19386"


@ -45,7 +46,6 @@ info "building python."
 tar xf "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" -C "$BUILDDIR"
 (
    cd "$BUILDDIR/Python-$PYTHON_VERSION"
-    export SOURCE_DATE_EPOCH=1530212462
    LC_ALL=C export BUILD_DATE=$(date -u -d "@$SOURCE_DATE_EPOCH" "+%b %d %Y")
    LC_ALL=C export BUILD_TIME=$(date -u -d "@$SOURCE_DATE_EPOCH" "+%H:%M:%S")
    # Patch taken from Ubuntu http://archive.ubuntu.com/ubuntu/pool/main/p/python3.7/python3.7_3.7.6-1.debian.tar.xz
@ -77,26 +77,8 @@ git clone "https://github.com/squashfskit/squashfskit.git" "$BUILDDIR/squashfski
 MKSQUASHFS="$BUILDDIR/squashfskit/squashfs-tools/mksquashfs"


-info "building libsecp256k1."
-(
-    git clone https://github.com/bitcoin-core/secp256k1 "$CACHEDIR"/secp256k1 \
-        || (cd "$CACHEDIR"/secp256k1 && git reset --hard && git pull)
-    cd "$CACHEDIR"/secp256k1
-    git reset --hard "$LIBSECP_VERSION"
-    git clean -f -x -q
-    export SOURCE_DATE_EPOCH=1530212462
-    echo "LDFLAGS = -no-undefined" >> Makefile.am
-    ./autogen.sh
-    ./configure \
-      --prefix="$APPDIR/usr" \
-      --enable-module-recovery \
-      --enable-experimental \
-      --enable-module-ecdh \
-      --disable-jni \
-      -q
-    make -j4 -s || fail "Could not build libsecp"
-    make -s install > /dev/null || fail "Could not install libsecp"
-)
+"$CONTRIB"/make_libsecp256k1.sh || fail "Could not build libsecp"
+cp -f "$PROJECT_ROOT/electrum/libsecp256k1.so.0" "$APPDIR/usr/lib/libsecp256k1.so.0" || fail "Could not copy libsecp to its destination"


 appdir_python() {
@ -224,7 +206,6 @@ rm -rf "$PYDIR"/site-packages/PyQt5/Qt.so

 # these are deleted as they were not deterministic; and are not needed anyway
 find "$APPDIR" -path '*/__pycache__*' -delete
-rm "$APPDIR"/usr/lib/libsecp256k1.a
 # note that jsonschema-*.dist-info is needed by that package as it uses 'pkg_resources.get_distribution'
 # also, see https://gitlab.com/python-devs/importlib_metadata/issues/71
 for f in "$PYDIR"/site-packages/jsonschema-*.dist-info; do mv "$f" "$(echo "$f" | sed s/\.dist-info/\.dist-info2/)"; done
--- a/contrib/build-wine/build-electrum-git.sh
+++ b/contrib/build-wine/build-electrum-git.sh
@ -6,7 +6,6 @@ NAME_ROOT=electrum
 export WINEPREFIX=/opt/wine64
 export WINEDEBUG=-all
 export PYTHONDONTWRITEBYTECODE=1
-export PYTHONHASHSEED=22

 PYHOME=c:/python3
 PYTHON="wine $PYHOME/python.exe -OO -B"
--- a/contrib/build-wine/build-secp256k1.sh
+++ b/contrib/build-wine/build-secp256k1.sh
@ -1,56 +0,0 @@
-#!/bin/bash
-# heavily based on https://github.com/ofek/coincurve/blob/417e726f553460f88d7edfa5dc67bfda397c4e4a/.travis/build_windows_wheels.sh
-
-set -e
-
-here="$(dirname "$(readlink -e "$0")")"
-LIBSECP_VERSION="b408c6a8b287003d1ade5709e6f7bc3c7f1d5be7"
-
-. "$CONTRIB"/build_tools_util.sh
-
-info "building libsecp256k1..."
-
-
-build_dll() {
-    #sudo apt-get install -y mingw-w64
-    export SOURCE_DATE_EPOCH=1530212462
-    echo "LDFLAGS = -no-undefined" >> Makefile.am
-    ./autogen.sh
-    # Note: set both --build and --host when running configure
-    # Otherwise weird voodoo magic happens with Docker and Wine.
-    # https://www.gnu.org/software/autoconf/manual/autoconf-2.69/html_node/Hosts-and-Cross_002dCompilation.html
-    LDFLAGS="-Wl,--no-insert-timestamp" ./configure \
-        --host=$1 \
-        --build=x86_64-pc-linux-gnu \
-        --enable-module-recovery \
-        --enable-experimental \
-        --enable-module-ecdh \
-        --disable-jni
-    make -j4
-    ${1}-strip .libs/libsecp256k1-0.dll
-}
-
-
-cd "$CACHEDIR"
-
-if [ -f "secp256k1/libsecp256k1.dll" ]; then
-    info "libsecp256k1.dll already built, skipping"
-    exit 0
-fi
-
-
-if [ ! -d secp256k1 ]; then
-    git clone https://github.com/bitcoin-core/secp256k1.git
-fi
-
-cd secp256k1
-git reset --hard
-git clean -f -x -q
-git checkout $LIBSECP_VERSION
-
-build_dll i686-w64-mingw32  # 64-bit would be: x86_64-w64-mingw32
-mv .libs/libsecp256k1-0.dll libsecp256k1.dll
-
-find -exec touch -d '2000-11-11T11:11:11+00:00' {} +
-
-info "building libsecp256k1 finished"
--- a/contrib/build-wine/build.sh
+++ b/contrib/build-wine/build.sh
@ -2,16 +2,19 @@

 set -e

-# Lucky number
-export PYTHONHASHSEED=22
-
 here="$(dirname "$(readlink -e "$0")")"
 test -n "$here" -a -d "$here" || exit

 export CONTRIB="$here/.."
+export PROJECT_ROOT="$CONTRIB/.."
 export CACHEDIR="$here/.cache"
 export PIP_CACHE_DIR="$CACHEDIR/pip_cache"

+export BUILD_TYPE="wine"
+export GCC_TRIPLET_HOST="i686-w64-mingw32"
+export GCC_TRIPLET_BUILD="x86_64-pc-linux-gnu"
+export GCC_STRIP_BINARIES="1"
+
 . "$CONTRIB"/build_tools_util.sh

 info "Clearing $here/build and $here/dist..."
@ -20,7 +23,11 @@ rm "$here"/dist/* -rf

 mkdir -p "$CACHEDIR" "$PIP_CACHE_DIR"

-$here/build-secp256k1.sh || fail "build-secp256k1 failed"
+if [ -f "$PROJECT_ROOT/electrum/libsecp256k1-0.dll" ]; then
+    info "libsecp256k1 already built, skipping"
+else
+    "$CONTRIB"/make_libsecp256k1.sh || fail "Could not build libsecp"
+fi

 $here/prepare-wine.sh || fail "prepare-wine failed"

--- a/contrib/build-wine/deterministic.spec
+++ b/contrib/build-wine/deterministic.spec
@ -34,7 +34,7 @@ binaries = []
 # Workaround for "Retro Look":
 binaries += [b for b in collect_dynamic_libs('PyQt5') if 'qwindowsvista' in b[0]]

-binaries += [('C:/tmp/libsecp256k1.dll', '.')]
+binaries += [('C:/tmp/libsecp256k1-0.dll', '.')]
 binaries += [('C:/tmp/libusb-1.0.dll', '.')]

 datas = [
--- a/contrib/build-wine/prepare-wine.sh
+++ b/contrib/build-wine/prepare-wine.sh
@ -89,7 +89,6 @@ info "Compiling libusb..."
    git remote add origin $LIBUSB_REPO
    git fetch --depth 1 origin $LIBUSB_COMMIT
    git checkout -b pinned FETCH_HEAD
-    export SOURCE_DATE_EPOCH=1530212462
    echo "libusb_1_0_la_LDFLAGS += -Wc,-static" >> libusb/Makefile.am
    ./bootstrap.sh || fail "Could not bootstrap libusb"
    host="i686-w64-mingw32"
@ -102,8 +101,8 @@ info "Compiling libusb..."
 cp "$CACHEDIR/libusb/libusb/.libs/libusb-1.0.dll" $WINEPREFIX/drive_c/tmp/  || fail "Could not copy libusb to its destination"


-# copy libsecp dll (already built by build-secp256k1.sh)
-cp "$CACHEDIR/secp256k1/libsecp256k1.dll" $WINEPREFIX/drive_c/tmp/ || fail "Could not copy libsecp to its destination"
+# copy libsecp dll (already built)
+cp "$PROJECT_ROOT/electrum/libsecp256k1-0.dll" $WINEPREFIX/drive_c/tmp/ || fail "Could not copy libsecp to its destination"


 info "Building PyInstaller."
--- a/contrib/build_tools_util.sh
+++ b/contrib/build_tools_util.sh
@ -70,3 +70,58 @@ function retry() {

  return $result
 }
+
+function gcc_with_triplet()
+{
+    TRIPLET="$1"
+    CMD="$2"
+    shift 2
+    if [ -n "$TRIPLET" ] ; then
+        "$TRIPLET-$CMD" "$@"
+    else
+        "$CMD" "$@"
+    fi
+}
+
+function gcc_host()
+{
+    gcc_with_triplet "$GCC_TRIPLET_HOST" "$@"
+}
+
+function gcc_build()
+{
+    gcc_with_triplet "$GCC_TRIPLET_BUILD" "$@"
+}
+
+function host_strip()
+{
+    if [ "$GCC_STRIP_BINARIES" -ne "0" ] ; then
+        case "$BUILD_TYPE" in
+            linux|wine)
+                gcc_host strip "$@"
+                ;;
+            darwin)
+                # TODO: Strip on macOS?
+                ;;
+        esac
+    fi
+}
+
+
+
+export SOURCE_DATE_EPOCH=1530212462
+export PYTHONHASHSEED=22
+# Set the build type, overridden by wine build
+export BUILD_TYPE="${BUILD_TYPE:-$(uname | tr '[:upper:]' '[:lower:]')}"
+# No additional autoconf flags by default
+export AUTOCONF_FLAGS=""
+# Add host / build flags if the triplets are set
+if [ -n "$GCC_TRIPLET_HOST" ] ; then
+    export AUTOCONF_FLAGS="$AUTOCONF_FLAGS --host=$GCC_TRIPLET_HOST"
+fi
+if [ -n "$GCC_TRIPLET_BUILD" ] ; then
+    export AUTOCONF_FLAGS="$AUTOCONF_FLAGS --build=$GCC_TRIPLET_BUILD"
+fi
+
+export GCC_STRIP_BINARIES="${GCC_STRIP_BINARIES:-0}"
+
--- a/contrib/make_libsecp256k1.sh
+++ b/contrib/make_libsecp256k1.sh
@ -0,0 +1,49 @@
+#!/bin/bash
+
+LIBSECP_VERSION="b408c6a8b287003d1ade5709e6f7bc3c7f1d5be7"
+
+set -e
+
+here=$(dirname $(realpath "$0" 2> /dev/null || grealpath "$0"))
+CONTRIB="$here"
+PROJECT_ROOT="$CONTRIB/.."
+
+. "$here"/build_tools_util.sh || (echo "Could not source build_tools_util.sh" && exit 1)
+
+pkgname="secp256k1"
+info "Building $pkgname..."
+
+(
+    cd $CONTRIB
+    if [ ! -d secp256k1 ]; then
+        git clone https://github.com/bitcoin-core/secp256k1.git
+    fi
+    cd secp256k1
+    git reset --hard
+    git clean -f -x -q
+    git checkout $LIBSECP_VERSION
+
+    if ! [ -x configure ] ; then
+        echo "libsecp256k1_la_LDFLAGS = -no-undefined" >> Makefile.am
+        echo "LDFLAGS = -no-undefined" >> Makefile.am
+        ./autogen.sh || fail "Could not run autogen for $pkgname. Please make sure you have automake and libtool installed, and try again."
+    fi
+    if ! [ -r config.status ] ; then
+        ./configure \
+            $AUTOCONF_FLAGS \
+            --prefix="$here/$pkgname/dist" \
+            --enable-module-recovery \
+            --enable-experimental \
+            --enable-module-ecdh \
+            --disable-jni \
+            --disable-tests \
+            --disable-static \
+            --enable-shared || fail "Could not configure $pkgname. Please make sure you have a C compiler installed and try again."
+    fi
+    make -j4 || fail "Could not build $pkgname"
+    make install || fail "Could not install $pkgname"
+    . "$here/$pkgname/dist/lib/libsecp256k1.la"
+    host_strip "$here/$pkgname/dist/lib/$dlname"
+    cp -fpv "$here/$pkgname/dist/lib/$dlname" "$PROJECT_ROOT/electrum" || fail "Could not copy the $pkgname binary to its destination"
+    info "$dlname has been placed in the inner 'electrum' folder."
+)
--- a/contrib/osx/make_osx
+++ b/contrib/osx/make_osx
@ -7,6 +7,8 @@ PACKAGE=Electrum
 GIT_REPO=https://github.com/spesmilo/electrum
 LIBSECP_VERSION="b408c6a8b287003d1ade5709e6f7bc3c7f1d5be7"

+export GCC_STRIP_BINARIES="1"
+
 . $(dirname "$0")/base.sh

 CONTRIB_OSX="$(dirname "$(realpath "$0")")"
@ -16,7 +18,6 @@ ROOT_FOLDER="$CONTRIB/.."
 src_dir=$(dirname "$0")
 cd $src_dir/../..

-export PYTHONHASHSEED=22
 VERSION=`git describe --tags --dirty --always`

 which brew > /dev/null 2>&1 || fail "Please install brew from https://brew.sh/ to continue"
@ -96,17 +97,10 @@ cp $BUILDDIR/libusb/1.0.22/lib/libusb-1.0.dylib contrib/osx
 echo "82c368dfd4da017ceb32b12ca885576f325503428a4966cc09302cbd62702493  contrib/osx/libusb-1.0.dylib" | \
    shasum -a 256 -c || fail "libusb checksum mismatched"

-info "Building libsecp256k1"
+info "Preparing for building libsecp256k1"
 brew install autoconf automake libtool
-git clone https://github.com/bitcoin-core/secp256k1 $BUILDDIR/secp256k1
-pushd $BUILDDIR/secp256k1
-git reset --hard $LIBSECP_VERSION
-git clean -f -x -q
-./autogen.sh
-./configure --enable-module-recovery --enable-experimental --enable-module-ecdh --disable-jni
-make -j4
-popd
-cp $BUILDDIR/secp256k1/.libs/libsecp256k1.0.dylib contrib/osx
+"$CONTRIB"/make_libsecp256k1.sh || fail "Could not build libsecp"
+cp "$ROOT_FOLDER"/electrum/libsecp256k1.0.dylib contrib/osx

 info "Building CalinsQRReader..."
 d=contrib/osx/CalinsQRReader
--- a/electrum/ecc_fast.py
+++ b/electrum/ecc_fast.py
@ -38,15 +38,25 @@ class LibModuleMissing(Exception): pass

 def load_library():
    if sys.platform == 'darwin':
-        library_path = 'libsecp256k1.0.dylib'
+        library_paths = (os.path.join(os.path.dirname(__file__), 'libsecp256k1.0.dylib'),
+                         'libsecp256k1.0.dylib')
    elif sys.platform in ('windows', 'win32'):
-        library_path = 'libsecp256k1.dll'
+        library_paths = (os.path.join(os.path.dirname(__file__), 'libsecp256k1-0.dll'),
+                         'libsecp256k1-0.dll')
    elif 'ANDROID_DATA' in os.environ:
-        library_path = 'libsecp256k1.so'
-    else:
-        library_path = 'libsecp256k1.so.0'
+        library_paths = ('libsecp256k1.so',)
+    else:  # desktop Linux and similar
+        library_paths = (os.path.join(os.path.dirname(__file__), 'libsecp256k1.so.0'),
+                         'libsecp256k1.so.0')

-    secp256k1 = ctypes.cdll.LoadLibrary(library_path)
+    secp256k1 = None
+    for libpath in library_paths:
+        try:
+            secp256k1 = ctypes.cdll.LoadLibrary(libpath)
+        except:
+            pass
+        else:
+            break
    if not secp256k1:
        _logger.error('libsecp256k1 library failed to load')
        return None