Merge #175: Add caching support for dependencies

6cfb841 cache joinmarket deps on docker (fivepiece) c99a209 get bitcoind from ppa only on linux host (fivepiece) 9005ee1 cache joinmarket deps on travis (fivepiece) 974b5ae add cache support to install.sh (fivepiece)
7 years ago · c86c776484
8 changed files with 89 additions and 52 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -1,17 +1,19 @@
 sudo: required
 dist: trusty
-addons:
-  apt:
-    sources:
-    - sourceline: 'ppa:bitcoin/bitcoin'
-      key_url: 'http://keyserver.ubuntu.com:11371/pks/lookup?op=get&search=0xD46F45428842CE5E'
-    packages:
-      - bitcoind
-      - python-qt4 python-sip
 matrix:
  include:
    - os: osx
+      env: PIP_DOWNLOAD_CACHE=$HOME/Library/Caches/pip
    - os: linux
+      env: PIP_DOWNLOAD_CACHE=$HOME/.cache/pip
+      addons:
+        apt:
+           sources:
+           - sourceline: 'ppa:bitcoin/bitcoin'
+             key_url: 'http://keyserver.ubuntu.com:11371/pks/lookup?op=get&search=0xD46F45428842CE5E'
+           packages:
+             - bitcoind
+             - python-qt4 python-sip
    - os: linux
      services: docker
      env: DOCKER_IMG_JM=xenial
@ -33,9 +35,15 @@ before_install:
  - on_docker(){ if [ -n "$DOCKER_IMG_JM" ]; then $@ ; fi; }
 cache:
  directories:
-    $HOME/downloads
+   - $HOME/downloads
+   - $HOME/.cache/pip
+   - $HOME/Library/Caches/pip
 install:
+  - mkdir -p "$HOME/downloads"
+  - mkdir -p "$TRAVIS_BUILD_DIR/deps/cache/"
+  - find "$HOME/downloads" -type f -exec cp -v {} "$TRAVIS_BUILD_DIR/deps/cache/" \;
  - on_host ./install.sh --develop --no-gpg-validation
+  - on_host find "$TRAVIS_BUILD_DIR/deps/cache/" -type f -exec cp -v {} "$HOME/downloads/" \;
 before_script:
  - on_host source jmvenv/bin/activate
 script:
--- a/install.sh
+++ b/install.sh
@ -17,10 +17,12 @@ gpg_verify_sig ()

 sha256_verify ()
 {
-    if [ "$(uname)" == "Darwin" ]; then
+    if [[ "$(uname)" == "Darwin" ]]; then
        shasum -a 256 -c <<<"$1  $2"
+        return "$?"
    else
        sha256sum -c <<<"$1  $2"
+        return "$?"
    fi
 }

@ -61,7 +63,7 @@ deb_deps_install ()

 check_skip_build ()
 {
-    if ! mkdir "$1"; then
+    if [[ ${reinstall} == false ]] && [[ -d "$1" ]]; then
        read -p "Directory ${1} exists.  Remove and recreate?  (y/n)  " q
        if [[ "${q}" =~ Y|y ]]; then
            rm -rf "./${1}"
@ -79,8 +81,9 @@ venv_setup ()
 {
    if check_skip_build 'jmvenv'; then
        return 0
+    else
+        reinstall='true'
    fi
-    rm -rf "${jm_source}/deps"
    virtualenv -p python2 "${jm_source}/jmvenv" || return 1
    source "${jm_source}/jmvenv/bin/activate" || return 1
    pip install --upgrade pip
@ -91,10 +94,10 @@ venv_setup ()
 openssl_get ()
 {
    if [[ -z "${no_gpg_validation}" ]]; then
-        openssl_file=( "${openssl_lib_tar}" "${openssl_lib_sha}" "${openssl_lib_sig}" )
+        openssl_files=( "${openssl_lib_tar}" "${openssl_lib_sig}" )
        curl --retry 5 -L "${openssl_signer_key_url}" -o openssl_signer.key
    else
-        openssl_file=( "${openssl_lib_tar}" )
+        openssl_files=( "${openssl_lib_tar}" )
    fi
    for file in ${openssl_files[@]}; do
        curl --retry 5 -L -O "${openssl_url}/${file}"
@ -122,19 +125,20 @@ openssl_install ()
 {
    openssl_version='openssl-1.0.2l'
    openssl_lib_tar="${openssl_version}.tar.gz"
-    openssl_lib_sha="${openssl_lib_tar}.sha256"
+    openssl_lib_sha='ce07195b659e75f4e1db43552860070061f156a98bb37b672b101ba6e3ddf30c'
    openssl_lib_sig="${openssl_lib_tar}.asc"
    openssl_url='https://www.openssl.org/source'
-    openssl_signer_key_url='https://pgp.mit.edu/pks/lookup?op=get&search=0xD9C4D26D0E604491'
    openssl_signer_key_id='D9C4D26D0E604491'
-    openssl_root="${jm_deps}/openssl"
+    openssl_signer_key_url="https://pgp.mit.edu/pks/lookup?op=get&search=0x${openssl_signer_key_id}"

-    if check_skip_build 'openssl'; then
+    if check_skip_build "${openssl_version}"; then
        return 0
    fi
-    pushd openssl
-    openssl_get
-    if ! sha256_verify "${openssl_lib_tar}" "${openssl_lib_sha}"; then
+    pushd cache
+    if ! sha256_verify "${openssl_lib_sha}" "${openssl_lib_tar}"; then
+        openssl_get
+    fi
+    if ! sha256_verify "${openssl_lib_sha}" "${openssl_lib_tar}"; then
        return 1
    fi
    if [[ -z "${no_gpg_validation}" ]]; then
@ -144,13 +148,14 @@ openssl_install ()
            return 1
        fi
        if gpg_verify_sig "${openssl_lib_sig}"; then
-            tar -xzf "${openssl_lib_tar}"
+            tar -xzf "${openssl_lib_tar}" -C ../
        else
            return 1
        fi
    else
-        tar -xzf "${openssl_lib_tar}"
+        tar -xzf "${openssl_lib_tar}" -C ../
    fi
+    popd
    pushd "${openssl_version}"
    if openssl_build; then
        make install_sw
@ -158,7 +163,6 @@ openssl_install ()
        return 1
    fi
    popd
-    popd
 }

 # add '--disable-docs' to libffi ./configure so makeinfo isn't needed
@ -173,6 +177,15 @@ libffi_patch_disable_docs ()
 > if BUILD_DOCS
 > #info_TEXINFOS += doc/libffi.texi
 > endif
+EOF
+
+    # autogen.sh is not happy when run from some directories, causing it
+    # to create an ltmain.sh file in our ${jm_root} directory.  weird.
+    # https://github.com/meetecho/janus-gateway/issues/290#issuecomment-125160739
+    # https://github.com/meetecho/janus-gateway/commit/ac38cfdae7185f9061569b14809af4d4052da700
+    cat <<'EOF' > autoreconf.patch
+18a19
+> AC_CONFIG_AUX_DIR([.])
 EOF

    cat <<'EOF' > configure.ac.patch
@ -186,6 +199,7 @@ EOF
 > 
 EOF
    patch Makefile.am Makefile.am.patch
+    patch configure.ac autoreconf.patch
    patch configure.ac configure.ac.patch
 }

@ -207,16 +221,19 @@ libffi_install ()
    libffi_lib_sha='96d08dee6f262beea1a18ac9a3801f64018dc4521895e9198d029d6850febe23'
    libffi_url="https://github.com/libffi/libffi/archive"

-    if check_skip_build 'libffi'; then
+    if check_skip_build "${libffi_version}"; then
        return 0
    fi
-    pushd libffi
-    curl --retry 5 -L -O "${libffi_url}/${libffi_lib_tar}"
+    pushd cache
+    if ! sha256_verify "${libffi_lib_sha}" "${libffi_lib_tar}"; then
+        curl --retry 5 -L -O "${libffi_url}/${libffi_lib_tar}"
+    fi
    if sha256_verify "${libffi_lib_sha}" "${libffi_lib_tar}"; then
-        tar -xzf "${libffi_lib_tar}"
+        tar -xzf "${libffi_lib_tar}" -C ../
    else
        return 1
    fi
+    popd
    pushd "${libffi_version}"
    if ! libffi_patch_disable_docs; then
        return 1
@ -227,7 +244,6 @@ libffi_install ()
        return 1
    fi
    popd
-    popd
 }

 libsodium_get ()
@ -259,15 +275,21 @@ libsodium_install ()
    sodium_version='libsodium-1.0.13'
    sodium_lib_tar="${sodium_version}.tar.gz"
    sodium_lib_sig="${sodium_lib_tar}.sig"
+    sodium_lib_sha='9c13accb1a9e59ab3affde0e60ef9a2149ed4d6e8f99c93c7a5b97499ee323fd'
    sodium_url='https://download.libsodium.org/libsodium/releases'
    sodium_signer_key_url='https://pgp.mit.edu/pks/lookup?op=get&search=0x210627AABA709FE1'
    sodium_signer_key_id='62F25B592B6F76DA'

-    if check_skip_build 'libsodium'; then
+    if check_skip_build "${sodium_version}"; then
        return 0
    fi
-    pushd libsodium
-    libsodium_get
+    pushd cache
+    if ! sha256_verify "${sodium_lib_sha}" "${sodium_lib_tar}"; then
+        libsodium_get
+    fi
+    if ! sha256_verify "${sodium_lib_sha}" "${sodium_lib_tar}"; then
+        return 1
+    fi
    if [[ -z "${no_gpg_validation}" ]]; then
        if gpg_verify_key libsodium_signer.key "${sodium_signer_key_id}"; then
            gpg_add_to_keyring libsodium_signer.key
@ -275,13 +297,14 @@ libsodium_install ()
            return 1
        fi
        if gpg_verify_sig "${sodium_lib_sig}"; then
-            tar -xzf "${sodium_lib_tar}"
+            tar -xzf "${sodium_lib_tar}" -C ../
        else
            return 1
        fi
    else
-        tar -xzf "${sodium_lib_tar}"
+        tar -xzf "${sodium_lib_tar}" -C ../
    fi
+    popd
    pushd "${sodium_version}"
    if libsodium_build; then
        make install
@ -289,7 +312,6 @@ libsodium_install ()
        return 1
    fi
    popd
-    popd
 }

 joinmarket_install ()
@ -383,6 +405,7 @@ main ()
    # flags
    develop_build=''
    no_gpg_validation=''
+    reinstall='false'
    parse_flags ${@}

    # os check
@ -397,9 +420,8 @@ main ()
        return 1
    fi
    source "${jm_root}/bin/activate"
-    mkdir -p deps
+    mkdir -p "deps/cache"
    pushd deps
-    rm -f ./keyring.gpg
 # openssl build disabled. using OS package manager's version.
 #    if ! openssl_install; then
 #        echo "Openssl was not built. Exiting."
--- a/test/Dockerfiles/bionic.Dockerfile
+++ b/test/Dockerfiles/bionic.Dockerfile
@ -26,8 +26,8 @@ USER chaum
 WORKDIR /home/chaum
 RUN ls -la .
 RUN ls -la ${repo_name}
-RUN ls -la ${repo_name}/deps
-RUN tar xaf ./${repo_name}/deps/${core_dist} -C /home/chaum
+RUN ls -la ${repo_name}/deps/cache
+RUN tar xaf ./${repo_name}/deps/cache/${core_dist} -C /home/chaum
 ENV PATH "/home/chaum/bitcoin-${core_version}/bin:${PATH}"
 RUN bitcoind --version | head -1

--- a/test/Dockerfiles/build_docker.sh
+++ b/test/Dockerfiles/build_docker.sh
@ -18,16 +18,23 @@ build_docker ()
    core_version='0.16.1'
    core_dist="bitcoin-${core_version}-x86_64-linux-gnu.tar.gz"
    core_url="https://bitcoin.org/bin/bitcoin-core-${core_version}/${core_dist}"
+    libffi_lib_tar='v3.2.1.tar.gz'
+    libffi_url="https://github.com/libffi/libffi/archive/${libffi_lib_tar}"
+    sodium_lib_tar='libsodium-1.0.13.tar.gz'
+    sodium_url="https://download.libsodium.org/libsodium/releases/${sodium_lib_tar}"
+    declare -A deps=( [${core_dist}]="${core_url}" [${libffi_lib_tar}]="${libffi_url}" [${sodium_lib_tar}]="${sodium_url}" )
    jm_root="${TRAVIS_BUILD_DIR}"
    owner_name="${TRAVIS_REPO_SLUG%\/*}"
    repo_name="${TRAVIS_REPO_SLUG#*\/}"

-    if [[ ! -f "${HOME}/downloads/${core_dist}" ]]; then
-        wget "${core_url}" -O "$HOME/downloads/${core_dist}"
-    fi
+    for dep in ${!deps[@]}; do
+        if [[ ! -r "${HOME}/downloads/${dep}" ]]; then
+            curl --retry 5 -L "${deps[${dep}]}" -o "$HOME/downloads/${dep}"
+        fi
+    done

-    mkdir -p "${jm_root}/deps"
-    cp "${HOME}/downloads/${core_dist}" "${jm_root}/deps/"
+    mkdir -p "${jm_root}/deps/cache"
+    find "$HOME/downloads" -type f -exec cp -v {} "${jm_root}/deps/cache/" \;
    cd "${jm_root}/../"

    docker build \
--- a/test/Dockerfiles/centos7.Dockerfile
+++ b/test/Dockerfiles/centos7.Dockerfile
@ -22,8 +22,8 @@ USER chaum
 WORKDIR /home/chaum
 RUN ls -la .
 RUN ls -la ${repo_name}
-RUN ls -la ${repo_name}/deps
-RUN tar xaf ./${repo_name}/deps/${core_dist} -C /home/chaum
+RUN ls -la ${repo_name}/deps/cache
+RUN tar xaf ./${repo_name}/deps/cache/${core_dist} -C /home/chaum
 ENV PATH "/home/chaum/bitcoin-${core_version}/bin:${PATH}"
 RUN bitcoind --version | head -1

--- a/test/Dockerfiles/fedora27.Dockerfile
+++ b/test/Dockerfiles/fedora27.Dockerfile
@ -25,8 +25,8 @@ USER chaum
 WORKDIR /home/chaum
 RUN ls -la .
 RUN ls -la ${repo_name}
-RUN ls -la ${repo_name}/deps
-RUN tar xaf ./${repo_name}/deps/${core_dist} -C /home/chaum
+RUN ls -la ${repo_name}/deps/cache
+RUN tar xaf ./${repo_name}/deps/cache/${core_dist} -C /home/chaum
 ENV PATH "/home/chaum/bitcoin-${core_version}/bin:${PATH}"
 RUN bitcoind --version | head -1

--- a/test/Dockerfiles/stretch.Dockerfile
+++ b/test/Dockerfiles/stretch.Dockerfile
@ -26,8 +26,8 @@ USER chaum
 WORKDIR /home/chaum
 RUN ls -la .
 RUN ls -la ${repo_name}
-RUN ls -la ${repo_name}/deps
-RUN tar xaf ./${repo_name}/deps/${core_dist} -C /home/chaum
+RUN ls -la ${repo_name}/deps/cache
+RUN tar xaf ./${repo_name}/deps/cache/${core_dist} -C /home/chaum
 ENV PATH "/home/chaum/bitcoin-${core_version}/bin:${PATH}"
 RUN bitcoind --version | head -1

--- a/test/Dockerfiles/xenial.Dockerfile
+++ b/test/Dockerfiles/xenial.Dockerfile
@ -26,8 +26,8 @@ USER chaum
 WORKDIR /home/chaum
 RUN ls -la .
 RUN ls -la ${repo_name}
-RUN ls -la ${repo_name}/deps
-RUN tar xaf ./${repo_name}/deps/${core_dist} -C /home/chaum
+RUN ls -la ${repo_name}/deps/cache
+RUN tar xaf ./${repo_name}/deps/cache/${core_dist} -C /home/chaum
 ENV PATH "/home/chaum/bitcoin-${core_version}/bin:${PATH}"
 RUN bitcoind --version | head -1