Merge JoinMarket-Org/joinmarket-clientserver#1061: Add support to build and autostart local Tor instance in jmvenv

5132342f57 update install.sh to build tor on macOS (jules23)
7a88781648 Add support to build and autostart local Tor instance in jmvenv (Kristaps Kaupe)

Pull request description:

  See https://github.com/JoinMarket-Org/joinmarket-clientserver/pull/1000#issuecomment-957379435 for context.

  This adds new option `--with-local-tor` to `install.sh` which builds Tor locally inside jmvenv. Then, when starting JM scripts that might need Tor connection, it is started automatically if nobody already listens on `127.0.0.1:9050`.

  If no problems are found with his approach in testing, we could switch to this as a default and remove all clearnet IRC configs from default config.

Top commit has no ACKs.

Tree-SHA512: caa7fcac88e57321f65a3f9fc09def94f9659855977026a19e1ac8f2f160a01237fe90220b986561bd622b05534f201ea5a29a9cdbc71013e40301992ed59e4d

install.sh

@@ -82,6 +82,30 @@ deps_install ()
     fi
 }
 
+tor_deps_install ()
+{
+    debian_deps=( \
+        'libevent-dev' \
+        'libssl-dev' \
+        'zlib1g-dev' )
+
+    darwin_deps=( \
+        'libevent' \
+        'zlib' )
+
+    if [[ ${use_os_deps_check} != '1' ]]; then
+        return 0
+    elif [[ ${install_os} == 'debian' ]]; then
+        deb_deps_install "${debian_deps[@]}"
+        return "$?"
+    elif [[ ${install_os} == 'darwin' ]]; then
+        dar_deps_install "${darwin_deps[@]}"
+        return "$?"
+    else
+        return 0
+    fi
+}
+
 deb_deps_check ()
 {
     apt-cache policy ${deb_deps[@]} | grep "Installed.*none"
@@ -115,17 +139,19 @@ dar_deps_install ()
         return 1
     fi
 
-    sudo_command=''
+    if ! which virtualenv >/dev/null; then
-    if [ "$with_sudo" == 1 ]; then
+        sudo_command=''
-        echo "
+        if [ "$with_sudo" == 1 ]; then
-            sudo password required to run :
+            echo "
+                sudo password required to run :
 
-            \`sudo pip3 install virtualenv\`
+                \`sudo pip3 install virtualenv\`
-            "
+                "
-        sudo_command="sudo"
+            sudo_command="sudo"
-    fi
+        fi
-    if $with_jmvenv && ! $sudo_command pip3 install virtualenv; then
+        if $with_jmvenv && ! $sudo_command pip3 install virtualenv; then
-        return 1
+            return 1
+        fi
     fi
 }
 
@@ -325,6 +351,49 @@ libsodium_install ()
     popd
 }
 
+tor_build ()
+{
+    $make uninstall
+    $make distclean
+    ./configure \
+        --disable-system-torrc \
+        --disable-seccomp \
+        --disable-libscrypt \
+        --disable-module-relay \
+        --disable-lzma \
+        --disable-zstd \
+        --disable-asciidoc \
+        --disable-manpage \
+        --disable-html-manual \
+        --prefix="${jm_root}"
+    $make
+    if ! $make check; then
+        return 1
+    fi
+}
+
+tor_install ()
+{
+    tor_version='tor-0.4.6.8'
+    tor_tar="${tor_version}.tar.gz"
+    tor_sha='15ce1a37b4cc175b07761e00acdcfa2c08f0d23d6c3ab9c97c464bd38cc5476a'
+    tor_url='https://dist.torproject.org'
+
+    if ! dep_get "${tor_tar}" "${tor_sha}" "${tor_url}"; then
+        return 1
+    fi
+    pushd "${tor_version}"
+    if tor_build; then
+        $make install
+        # Create blank tor config, it will default to running socks5 proxy
+        # at 127.0.0.1:9050 and should be enough for us.
+        > "${jm_root}/etc/tor/torrc"
+    else
+        return 1
+    fi
+    popd
+}
+
 joinmarket_install ()
 {
     reqs=( 'base.txt' )
@@ -379,6 +448,9 @@ parse_flags ()
                 echo 'ERROR: "--python" requires a non-empty option argument.'
                 return 1
                 ;;
+            --with-local-tor)
+                build_local_tor='1'
+                ;;
             --with-qt)
                 with_qt='1'
                 ;;
@@ -406,6 +478,7 @@ Options:
 --disable-secp-check        do not run libsecp256k1 tests (default is to run them)
 --docker-install            system wide install as root for minimal Docker installs
 --python, -p                python version (only python3 versions are supported)
+--with-local-tor            build Tor locally and autostart when needed
 --with-qt                   build the Qt GUI
 --without-qt                don't build the Qt GUI
 "
@@ -462,6 +535,7 @@ main ()
     # flags
     develop_build=''
     python='python3'
+    build_local_tor=''
     use_os_deps_check='1'
     use_secp_check='1'
     with_qt=''
@@ -498,6 +572,12 @@ main ()
         fi
         source "${jm_root}/bin/activate"
     fi
+    if [[ ${build_local_tor} == "1" ]]; then
+        if ! tor_deps_install; then
+            echo "Tor dependencies could not be installed. Exiting."
+            return 1
+        fi
+    fi
     mkdir -p "deps/cache"
     pushd deps
     if ! libsecp256k1_install; then
@@ -512,6 +592,12 @@ main ()
         echo "Libsodium was not built. Exiting."
         return 1
     fi
+    if [[ ${build_local_tor} == "1" ]]; then
+        if ! tor_install; then
+            echo "Building local Tor was requested, but not built. Exiting."
+            return 1
+        fi
+    fi
     popd
     if ! joinmarket_install; then
         echo "Joinmarket was not installed. Exiting."

jmclient/jmclient/__init__.py

@@ -26,7 +26,8 @@ from .configure import (load_test_config, process_shutdown,
     load_program_config, jm_single, get_network, update_persist_config,
     validate_address, is_burn_destination, get_irc_mchannels,
     get_blockchain_interface_instance, set_config, is_segwit_mode,
-    is_native_segwit_mode, JMPluginService, get_interest_rate, get_bondless_makers_allowance)
+    is_native_segwit_mode, JMPluginService, get_interest_rate,
+    get_bondless_makers_allowance, check_and_start_tor)
 from .blockchaininterface import (BlockchainInterface,
                                   RegtestBitcoinCoreInterface, BitcoinCoreInterface)
 from .snicker_receiver import SNICKERError, SNICKERReceiver

jmclient/jmclient/configure.py

@@ -3,7 +3,11 @@ import io
 import logging
 import os
 import re
+import socket
 import sys
+import subprocess
+import atexit
+from signal import SIGINT
 
 from configparser import ConfigParser, NoOptionError
 
@@ -743,6 +747,34 @@ def load_program_config(config_path="", bs=None, plugin_services=[]):
                 os.makedirs(plogsdir)
             p.set_log_dir(plogsdir)
 
+def gracefully_kill_subprocess(p):
+    # See https://stackoverflow.com/questions/43274476/is-there-a-way-to-check-if-a-subprocess-is-still-running
+    if p.poll() is None:
+        p.send_signal(SIGINT)
+
+def check_and_start_tor():
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    result = sock.connect_ex(("127.0.0.1", 9050))
+    sock.close()
+    if result == 0:
+        return
+    log.info("Nobody listens on 127.0.0.1:9050, trying to start Tor.")
+    tor_bin = os.path.join(sys.prefix, "bin", "tor")
+    if not os.path.exists(tor_bin):
+        log.info("Can't find our custom tor.")
+        return
+    command = [tor_bin, "-f", os.path.join(sys.prefix,
+        "etc", "tor", "torrc")]
+    # output messages from tor if loglevel is debug, they might be useful
+    if global_singleton.config.get("LOGGING", "console_log_level") == "DEBUG":
+        tor_stdout = sys.stdout
+    else:
+        tor_stdout = open(os.devnull, 'w')
+    tor_subprocess = subprocess.Popen(command, stdout=tor_stdout,
+        stderr=subprocess.STDOUT, close_fds=True)
+    atexit.register(gracefully_kill_subprocess, tor_subprocess)
+    log.debug("Started Tor subprocess with pid " + str(tor_subprocess.pid))
+
 def load_test_config(**kwargs):
     if "config_path" not in kwargs:
         load_program_config(config_path=".", **kwargs)

jmclient/jmclient/yieldgenerator.py

@@ -14,7 +14,7 @@ from jmclient import (Maker, jm_single, load_program_config,
                       JMClientProtocolFactory, start_reactor, calc_cj_fee,
                       WalletService, add_base_options, SNICKERReceiver,
                       SNICKERClientProtocolFactory, FidelityBondMixin,
-                      get_interest_rate, fmt_utxo)
+                      get_interest_rate, fmt_utxo, check_and_start_tor)
 from .wallet_utils import open_test_wallet_maybe, get_wallet_path
 from jmbase.support import EXIT_ARGERROR, EXIT_FAILURE, get_jm_version_str
 import jmbitcoin as btc
@@ -401,6 +401,8 @@ def ygmain(ygclass, nickserv_password='', gaplimit=6):
 
     load_program_config(config_path=options["datadir"])
 
+    check_and_start_tor()
+
     # As per previous note, override non-default command line settings:
     for x in ["ordertype", "txfee_contribution", "txfee_contribution_factor",
               "cjfee_a", "cjfee_r", "cjfee_factor", "minsize", "size_factor"]:

scripts/joinmarket-qt.py

@@ -73,7 +73,7 @@ from jmclient import load_program_config, get_network, update_persist_config,\
     parse_payjoin_setup, send_payjoin, JMBIP78ReceiverManager, \
     detect_script_type, general_custom_change_warning, \
     nonwallet_custom_change_warning, sweep_custom_change_warning, EngineError,\
-    TYPE_P2WPKH
+    TYPE_P2WPKH, check_and_start_tor
 from jmclient.wallet import BaseWallet
 
 from qtsupport import ScheduleWizard, TumbleRestartWizard, config_tips,\
@@ -2376,6 +2376,8 @@ if not jm_single().config.get("POLICY", "segwit") == "true":
 
 update_config_for_gui()
 
+check_and_start_tor()
+
 def onTabChange(i):
     """ Respond to change of tab.
     """

scripts/obwatch/ob-watcher.py

@@ -24,7 +24,7 @@ if sys.version_info < (3, 7):
 
 from jmbase.support import EXIT_FAILURE
 from jmbase import bintohex
-from jmclient import FidelityBondMixin, get_interest_rate
+from jmclient import FidelityBondMixin, get_interest_rate, check_and_start_tor
 from jmclient.fidelity_bond import FidelityBondProof
 
 import sybil_attack_calculations as sybil
@@ -803,6 +803,7 @@ def main():
                       default=62601)
     (options, args) = parser.parse_args()
     load_program_config(config_path=options.datadir)
+    check_and_start_tor()
     hostport = (options.host, options.port)
     mcs = [ObIRCMessageChannel(c) for c in get_irc_mchannels()]
     mcc = MessageChannelCollection(mcs)

scripts/sendpayment.py

@@ -18,7 +18,7 @@ from jmclient import Taker, load_program_config, get_schedule,\
     get_sendpayment_parser, get_max_cj_fee_values, check_regtest, \
     parse_payjoin_setup, send_payjoin, general_custom_change_warning, \
     nonwallet_custom_change_warning, sweep_custom_change_warning, \
-    EngineError
+    EngineError, check_and_start_tor
 from twisted.python.log import startLogging
 from jmbase.support import get_log, jmprint, \
     EXIT_FAILURE, EXIT_ARGERROR
@@ -63,6 +63,8 @@ def main():
                     " wallet, amount, destination address or wallet, bitcoin_uri.")
                 sys.exit(EXIT_ARGERROR)
 
+    check_and_start_tor()
+
     #without schedule file option, use the arguments to create a schedule
     #of a single transaction
     sweeping = False