Merge #175: Add caching support for dependencies

6cfb841 cache joinmarket deps on docker (fivepiece)
c99a209 get bitcoind from ppa only on linux host (fivepiece)
9005ee1 cache joinmarket deps on travis (fivepiece)
974b5ae add cache support to install.sh (fivepiece)

.travis.yml

@@ -1,17 +1,19 @@
 sudo: required
 dist: trusty
-addons:
-  apt:
-    sources:
-    - sourceline: 'ppa:bitcoin/bitcoin'
-      key_url: 'http://keyserver.ubuntu.com:11371/pks/lookup?op=get&search=0xD46F45428842CE5E'
-    packages:
-      - bitcoind
-      - python-qt4 python-sip
 matrix:
   include:
     - os: osx
+      env: PIP_DOWNLOAD_CACHE=$HOME/Library/Caches/pip
     - os: linux
+      env: PIP_DOWNLOAD_CACHE=$HOME/.cache/pip
+      addons:
+        apt:
+           sources:
+           - sourceline: 'ppa:bitcoin/bitcoin'
+             key_url: 'http://keyserver.ubuntu.com:11371/pks/lookup?op=get&search=0xD46F45428842CE5E'
+           packages:
+             - bitcoind
+             - python-qt4 python-sip
     - os: linux
       services: docker
       env: DOCKER_IMG_JM=xenial
@@ -33,9 +35,15 @@ before_install:
   - on_docker(){ if [ -n "$DOCKER_IMG_JM" ]; then $@ ; fi; }
 cache:
   directories:
-    $HOME/downloads
+   - $HOME/downloads
+   - $HOME/.cache/pip
+   - $HOME/Library/Caches/pip
 install:
+  - mkdir -p "$HOME/downloads"
+  - mkdir -p "$TRAVIS_BUILD_DIR/deps/cache/"
+  - find "$HOME/downloads" -type f -exec cp -v {} "$TRAVIS_BUILD_DIR/deps/cache/" \;
   - on_host ./install.sh --develop --no-gpg-validation
+  - on_host find "$TRAVIS_BUILD_DIR/deps/cache/" -type f -exec cp -v {} "$HOME/downloads/" \;
 before_script:
   - on_host source jmvenv/bin/activate
 script:

install.sh

@@ -17,10 +17,12 @@ gpg_verify_sig ()
 
 sha256_verify ()
 {
-    if [ "$(uname)" == "Darwin" ]; then
+    if [[ "$(uname)" == "Darwin" ]]; then
         shasum -a 256 -c <<<"$1  $2"
+        return "$?"
     else
         sha256sum -c <<<"$1  $2"
+        return "$?"
     fi
 }
 
@@ -61,7 +63,7 @@ deb_deps_install ()
 
 check_skip_build ()
 {
-    if ! mkdir "$1"; then
+    if [[ ${reinstall} == false ]] && [[ -d "$1" ]]; then
         read -p "Directory ${1} exists.  Remove and recreate?  (y/n)  " q
         if [[ "${q}" =~ Y|y ]]; then
             rm -rf "./${1}"
@@ -79,8 +81,9 @@ venv_setup ()
 {
     if check_skip_build 'jmvenv'; then
         return 0
+    else
+        reinstall='true'
     fi
-    rm -rf "${jm_source}/deps"
     virtualenv -p python2 "${jm_source}/jmvenv" || return 1
     source "${jm_source}/jmvenv/bin/activate" || return 1
     pip install --upgrade pip
@@ -91,10 +94,10 @@ venv_setup ()
 openssl_get ()
 {
     if [[ -z "${no_gpg_validation}" ]]; then
-        openssl_file=( "${openssl_lib_tar}" "${openssl_lib_sha}" "${openssl_lib_sig}" )
+        openssl_files=( "${openssl_lib_tar}" "${openssl_lib_sig}" )
         curl --retry 5 -L "${openssl_signer_key_url}" -o openssl_signer.key
     else
-        openssl_file=( "${openssl_lib_tar}" )
+        openssl_files=( "${openssl_lib_tar}" )
     fi
     for file in ${openssl_files[@]}; do
         curl --retry 5 -L -O "${openssl_url}/${file}"
@@ -122,19 +125,20 @@ openssl_install ()
 {
     openssl_version='openssl-1.0.2l'
     openssl_lib_tar="${openssl_version}.tar.gz"
-    openssl_lib_sha="${openssl_lib_tar}.sha256"
+    openssl_lib_sha='ce07195b659e75f4e1db43552860070061f156a98bb37b672b101ba6e3ddf30c'
     openssl_lib_sig="${openssl_lib_tar}.asc"
     openssl_url='https://www.openssl.org/source'
-    openssl_signer_key_url='https://pgp.mit.edu/pks/lookup?op=get&search=0xD9C4D26D0E604491'
     openssl_signer_key_id='D9C4D26D0E604491'
-    openssl_root="${jm_deps}/openssl"
+    openssl_signer_key_url="https://pgp.mit.edu/pks/lookup?op=get&search=0x${openssl_signer_key_id}"
 
-    if check_skip_build 'openssl'; then
+    if check_skip_build "${openssl_version}"; then
         return 0
     fi
-    pushd openssl
+    pushd cache
-    openssl_get
+    if ! sha256_verify "${openssl_lib_sha}" "${openssl_lib_tar}"; then
-    if ! sha256_verify "${openssl_lib_tar}" "${openssl_lib_sha}"; then
+        openssl_get
+    fi
+    if ! sha256_verify "${openssl_lib_sha}" "${openssl_lib_tar}"; then
         return 1
     fi
     if [[ -z "${no_gpg_validation}" ]]; then
@@ -144,13 +148,14 @@ openssl_install ()
             return 1
         fi
         if gpg_verify_sig "${openssl_lib_sig}"; then
-            tar -xzf "${openssl_lib_tar}"
+            tar -xzf "${openssl_lib_tar}" -C ../
         else
             return 1
         fi
     else
-        tar -xzf "${openssl_lib_tar}"
+        tar -xzf "${openssl_lib_tar}" -C ../
     fi
+    popd
     pushd "${openssl_version}"
     if openssl_build; then
         make install_sw
@@ -158,7 +163,6 @@ openssl_install ()
         return 1
     fi
     popd
-    popd
 }
 
 # add '--disable-docs' to libffi ./configure so makeinfo isn't needed
@@ -173,6 +177,15 @@ libffi_patch_disable_docs ()
 > if BUILD_DOCS
 > #info_TEXINFOS += doc/libffi.texi
 > endif
+EOF
+
+    # autogen.sh is not happy when run from some directories, causing it
+    # to create an ltmain.sh file in our ${jm_root} directory.  weird.
+    # https://github.com/meetecho/janus-gateway/issues/290#issuecomment-125160739
+    # https://github.com/meetecho/janus-gateway/commit/ac38cfdae7185f9061569b14809af4d4052da700
+    cat <<'EOF' > autoreconf.patch
+18a19
+> AC_CONFIG_AUX_DIR([.])
 EOF
 
     cat <<'EOF' > configure.ac.patch
@@ -186,6 +199,7 @@ EOF
 > 
 EOF
     patch Makefile.am Makefile.am.patch
+    patch configure.ac autoreconf.patch
     patch configure.ac configure.ac.patch
 }
 
@@ -207,16 +221,19 @@ libffi_install ()
     libffi_lib_sha='96d08dee6f262beea1a18ac9a3801f64018dc4521895e9198d029d6850febe23'
     libffi_url="https://github.com/libffi/libffi/archive"
 
-    if check_skip_build 'libffi'; then
+    if check_skip_build "${libffi_version}"; then
         return 0
     fi
-    pushd libffi
+    pushd cache
-    curl --retry 5 -L -O "${libffi_url}/${libffi_lib_tar}"
+    if ! sha256_verify "${libffi_lib_sha}" "${libffi_lib_tar}"; then
+        curl --retry 5 -L -O "${libffi_url}/${libffi_lib_tar}"
+    fi
     if sha256_verify "${libffi_lib_sha}" "${libffi_lib_tar}"; then
-        tar -xzf "${libffi_lib_tar}"
+        tar -xzf "${libffi_lib_tar}" -C ../
     else
         return 1
     fi
+    popd
     pushd "${libffi_version}"
     if ! libffi_patch_disable_docs; then
         return 1
@@ -227,7 +244,6 @@ libffi_install ()
         return 1
     fi
     popd
-    popd
 }
 
 libsodium_get ()
@@ -259,15 +275,21 @@ libsodium_install ()
     sodium_version='libsodium-1.0.13'
     sodium_lib_tar="${sodium_version}.tar.gz"
     sodium_lib_sig="${sodium_lib_tar}.sig"
+    sodium_lib_sha='9c13accb1a9e59ab3affde0e60ef9a2149ed4d6e8f99c93c7a5b97499ee323fd'
     sodium_url='https://download.libsodium.org/libsodium/releases'
     sodium_signer_key_url='https://pgp.mit.edu/pks/lookup?op=get&search=0x210627AABA709FE1'
     sodium_signer_key_id='62F25B592B6F76DA'
 
-    if check_skip_build 'libsodium'; then
+    if check_skip_build "${sodium_version}"; then
         return 0
     fi
-    pushd libsodium
+    pushd cache
-    libsodium_get
+    if ! sha256_verify "${sodium_lib_sha}" "${sodium_lib_tar}"; then
+        libsodium_get
+    fi
+    if ! sha256_verify "${sodium_lib_sha}" "${sodium_lib_tar}"; then
+        return 1
+    fi
     if [[ -z "${no_gpg_validation}" ]]; then
         if gpg_verify_key libsodium_signer.key "${sodium_signer_key_id}"; then
             gpg_add_to_keyring libsodium_signer.key
@@ -275,13 +297,14 @@ libsodium_install ()
             return 1
         fi
         if gpg_verify_sig "${sodium_lib_sig}"; then
-            tar -xzf "${sodium_lib_tar}"
+            tar -xzf "${sodium_lib_tar}" -C ../
         else
             return 1
         fi
     else
-        tar -xzf "${sodium_lib_tar}"
+        tar -xzf "${sodium_lib_tar}" -C ../
     fi
+    popd
     pushd "${sodium_version}"
     if libsodium_build; then
         make install
@@ -289,7 +312,6 @@ libsodium_install ()
         return 1
     fi
     popd
-    popd
 }
 
 joinmarket_install ()
@@ -383,6 +405,7 @@ main ()
     # flags
     develop_build=''
     no_gpg_validation=''
+    reinstall='false'
     parse_flags ${@}
 
     # os check
@@ -397,9 +420,8 @@ main ()
         return 1
     fi
     source "${jm_root}/bin/activate"
-    mkdir -p deps
+    mkdir -p "deps/cache"
     pushd deps
-    rm -f ./keyring.gpg
 # openssl build disabled. using OS package manager's version.
 #    if ! openssl_install; then
 #        echo "Openssl was not built. Exiting."

test/Dockerfiles/bionic.Dockerfile

@@ -26,8 +26,8 @@ USER chaum
 WORKDIR /home/chaum
 RUN ls -la .
 RUN ls -la ${repo_name}
-RUN ls -la ${repo_name}/deps
+RUN ls -la ${repo_name}/deps/cache
-RUN tar xaf ./${repo_name}/deps/${core_dist} -C /home/chaum
+RUN tar xaf ./${repo_name}/deps/cache/${core_dist} -C /home/chaum
 ENV PATH "/home/chaum/bitcoin-${core_version}/bin:${PATH}"
 RUN bitcoind --version | head -1
 

test/Dockerfiles/build_docker.sh

@@ -18,16 +18,23 @@ build_docker ()
     core_version='0.16.1'
     core_dist="bitcoin-${core_version}-x86_64-linux-gnu.tar.gz"
     core_url="https://bitcoin.org/bin/bitcoin-core-${core_version}/${core_dist}"
+    libffi_lib_tar='v3.2.1.tar.gz'
+    libffi_url="https://github.com/libffi/libffi/archive/${libffi_lib_tar}"
+    sodium_lib_tar='libsodium-1.0.13.tar.gz'
+    sodium_url="https://download.libsodium.org/libsodium/releases/${sodium_lib_tar}"
+    declare -A deps=( [${core_dist}]="${core_url}" [${libffi_lib_tar}]="${libffi_url}" [${sodium_lib_tar}]="${sodium_url}" )
     jm_root="${TRAVIS_BUILD_DIR}"
     owner_name="${TRAVIS_REPO_SLUG%\/*}"
     repo_name="${TRAVIS_REPO_SLUG#*\/}"
 
-    if [[ ! -f "${HOME}/downloads/${core_dist}" ]]; then
+    for dep in ${!deps[@]}; do
-        wget "${core_url}" -O "$HOME/downloads/${core_dist}"
+        if [[ ! -r "${HOME}/downloads/${dep}" ]]; then
-    fi
+            curl --retry 5 -L "${deps[${dep}]}" -o "$HOME/downloads/${dep}"
+        fi
+    done
 
-    mkdir -p "${jm_root}/deps"
+    mkdir -p "${jm_root}/deps/cache"
-    cp "${HOME}/downloads/${core_dist}" "${jm_root}/deps/"
+    find "$HOME/downloads" -type f -exec cp -v {} "${jm_root}/deps/cache/" \;
     cd "${jm_root}/../"
 
     docker build \

test/Dockerfiles/centos7.Dockerfile

@@ -22,8 +22,8 @@ USER chaum
 WORKDIR /home/chaum
 RUN ls -la .
 RUN ls -la ${repo_name}
-RUN ls -la ${repo_name}/deps
+RUN ls -la ${repo_name}/deps/cache
-RUN tar xaf ./${repo_name}/deps/${core_dist} -C /home/chaum
+RUN tar xaf ./${repo_name}/deps/cache/${core_dist} -C /home/chaum
 ENV PATH "/home/chaum/bitcoin-${core_version}/bin:${PATH}"
 RUN bitcoind --version | head -1
 

test/Dockerfiles/fedora27.Dockerfile

@@ -25,8 +25,8 @@ USER chaum
 WORKDIR /home/chaum
 RUN ls -la .
 RUN ls -la ${repo_name}
-RUN ls -la ${repo_name}/deps
+RUN ls -la ${repo_name}/deps/cache
-RUN tar xaf ./${repo_name}/deps/${core_dist} -C /home/chaum
+RUN tar xaf ./${repo_name}/deps/cache/${core_dist} -C /home/chaum
 ENV PATH "/home/chaum/bitcoin-${core_version}/bin:${PATH}"
 RUN bitcoind --version | head -1
 

test/Dockerfiles/stretch.Dockerfile

@@ -26,8 +26,8 @@ USER chaum
 WORKDIR /home/chaum
 RUN ls -la .
 RUN ls -la ${repo_name}
-RUN ls -la ${repo_name}/deps
+RUN ls -la ${repo_name}/deps/cache
-RUN tar xaf ./${repo_name}/deps/${core_dist} -C /home/chaum
+RUN tar xaf ./${repo_name}/deps/cache/${core_dist} -C /home/chaum
 ENV PATH "/home/chaum/bitcoin-${core_version}/bin:${PATH}"
 RUN bitcoind --version | head -1
 

test/Dockerfiles/xenial.Dockerfile

@@ -26,8 +26,8 @@ USER chaum
 WORKDIR /home/chaum
 RUN ls -la .
 RUN ls -la ${repo_name}
-RUN ls -la ${repo_name}/deps
+RUN ls -la ${repo_name}/deps/cache
-RUN tar xaf ./${repo_name}/deps/${core_dist} -C /home/chaum
+RUN tar xaf ./${repo_name}/deps/cache/${core_dist} -C /home/chaum
 ENV PATH "/home/chaum/bitcoin-${core_version}/bin:${PATH}"
 RUN bitcoind --version | head -1