Add support for BIP78 payjoins to .onion receivers

In this commit, the jmclient.payjoin module now supports
sending payments to BIP21 URIs where the pj= parameter is
to a hidden service address.
Additionally, the test/payjoinclient and test/payjoinserver
modules are edited to support optionally testing payments to
an ephemeral hidden service.

docs/PAYJOIN.md

@@ -155,6 +155,7 @@ The process here is to use the syntax of sendpayment.py:
 Notes on this:
 * Payjoins BIP78 style are done using the `sendpayment` script (there is no Qt support yet, but it will come later).
 * They are done using BIP21 URIs. These can be copy/pasted from a website (e.g. a btcpayserver invoice page), note that double quotes are required because the string contains special characters. Note also that you must see `pj=` in the URI, otherwise payjoin is not supported by that server.
+* If the url in `pj=` is `****.onion` it means you must be using Tor, remember to have Tor running on your system and change the configuration (see below) for sock5 port if necessary. If you are running the Tor browser the port is 9150 instead of 9050.
 * Don't forget to specify the mixdepth you are spending from with `-m 0`. The payment amount is of course in the URI, along with the address.
 * Pay attention to address type; this point is complicated, but: some servers will not be able to match the address type of the sender, and so won't be able to construct sensible Payjoin transactions. In that case they may fallback to the non-Payjoin payment (which is not a disaster). If you want to do a Payjoin with a server that only supports bech32, you will have to create a new Joinmarket wallet, specifying `native=true` in the `POLICY` section of `joinmarket.cfg` before you generate the wallet.
 
@@ -191,6 +192,15 @@ max_additional_fee_contribution = default
 # this is the minimum satoshis per vbyte we allow in the payjoin
 # transaction; note it is decimal, not integer.
 min_fee_rate = 1.1
+
+
+# for payjoins to hidden service endpoints, the socks5 configuration:
+onion_socks5_host = localhost
+onion_socks5_port = 9050
+# in some exceptional case the HS may be SSL configured,
+# this feature is not yet implemented in code, but here for the
+# future:
+hidden_service_ssl = false
 ```
 
 As the notes mention, you should probably find the defaults here are absolutely fine, and

jmclient/jmclient/configure.py

@@ -323,6 +323,14 @@ max_additional_fee_contribution = default
 # this is the minimum satoshis per vbyte we allow in the payjoin
 # transaction; note it is decimal, not integer.
 min_fee_rate = 1.1
+
+# for payjoins to hidden service endpoints, the socks5 configuration:
+onion_socks5_host = localhost
+onion_socks5_port = 9050
+# in some exceptional case the HS may be SSL configured,
+# this feature is not yet implemented in code, but here for the
+# future:
+hidden_service_ssl = false
 """
 
 #This allows use of the jmclient package with a

jmclient/jmclient/payjoin.py

@@ -5,7 +5,10 @@ from twisted.web.client import (Agent, readBody, ResponseFailed,
 from twisted.web.iweb import IPolicyForHTTPS
 from twisted.internet.ssl import CertificateOptions
 from twisted.internet.error import ConnectionRefusedError
+from twisted.internet.endpoints import TCP4ClientEndpoint
 from twisted.web.http_headers import Headers
+from txtorcon.web import tor_agent
+from txtorcon.socks import HostUnreachableError
 import urllib.parse as urlparse
 from urllib.parse import urlencode
 import json
@@ -451,11 +454,30 @@ def send_payjoin(manager, accept_callback=None,
 
     # Now we send the request to the server, with the encoded
     # payment PSBT
-    if not tls_whitelist:
+
-        agent = Agent(reactor)
+    # First we create a twisted web Agent object:
+
+    # TODO genericize/move out/use library function:
+    def is_hs_uri(s):
+        x = urlparse.urlparse(s)
+        if x.hostname.endswith(".onion"):
+            return (x.scheme, x.hostname, x.port)
+        return False
+
+    tor_url_data = is_hs_uri(manager.server)
+    if tor_url_data:
+        # note the return value is currently unused here
+        socks5_host = jm_single().config.get("PAYJOIN", "onion_socks5_host")
+        socks5_port = int(jm_single().config.get("PAYJOIN", "onion_socks5_port"))
+        # note: SSL not supported at the moment:
+        torEndpoint = TCP4ClientEndpoint(reactor, socks5_host, socks5_port)
+        agent = tor_agent(reactor, torEndpoint)
     else:
-        agent = Agent(reactor,
+        if not tls_whitelist:
-            contextFactory=WhitelistContextFactory(tls_whitelist))
+            agent = Agent(reactor)
+        else:
+            agent = Agent(reactor,
+                contextFactory=WhitelistContextFactory(tls_whitelist))
 
     body = BytesProducer(payment_psbt.to_base64().encode("utf-8"))
 
@@ -500,7 +522,7 @@ def send_payjoin(manager, accept_callback=None,
     # by a server rejection (which is accompanied by a non-200
     # status code returned), but by failure to communicate.
     def noResponse(failure):
-        failure.trap(ResponseFailed, ConnectionRefusedError)
+        failure.trap(ResponseFailed, ConnectionRefusedError, HostUnreachableError)
         log.error(failure.value)
         fallback_nonpayjoin_broadcast(manager, b"connection refused")
     d.addErrback(noResponse)
@@ -532,7 +554,6 @@ def fallback_nonpayjoin_broadcast(manager, err):
 
 def receive_payjoin_proposal_from_server(response, manager):
     assert isinstance(manager, JMPayjoinManager)
-
     # if the response code is not 200 OK, we must assume payjoin
     # attempt has failed, and revert to standard payment.
     if int(response.code) != 200:
@@ -554,7 +575,7 @@ def process_payjoin_proposal_from_server(response_body, manager):
             btc.PartiallySignedTransaction.from_base64(response_body)
     except Exception as e:
         log.error("Payjoin tx from server could not be parsed: " + repr(e))
-        fallback_nonpayjoin_broadcast(manager, err="Server sent invalid psbt")
+        fallback_nonpayjoin_broadcast(manager, err=b"Server sent invalid psbt")
         return
 
     log.debug("Receiver sent us this PSBT: ")
@@ -571,7 +592,7 @@ def process_payjoin_proposal_from_server(response_body, manager):
         payjoin_proposal_psbt.serialize(), with_sign_result=True)
     if err:
         log.error("Failed to sign PSBT from the receiver, error: " + err)
-        fallback_nonpayjoin_broadcast(manager, err="Failed to sign receiver PSBT")
+        fallback_nonpayjoin_broadcast(manager, err=b"Failed to sign receiver PSBT")
         return
 
     signresult, sender_signed_psbt = signresultandpsbt
@@ -579,7 +600,7 @@ def process_payjoin_proposal_from_server(response_body, manager):
     success, msg = manager.set_payjoin_psbt(payjoin_proposal_psbt, sender_signed_psbt)
     if not success:
         log.error(msg)
-        fallback_nonpayjoin_broadcast(manager, err="Receiver PSBT checks failed.")
+        fallback_nonpayjoin_broadcast(manager, err=b"Receiver PSBT checks failed.")
         return
     # All checks have passed. We can use the already signed transaction in
     # sender_signed_psbt.

test/payjoinclient.py

@@ -10,19 +10,37 @@ from jmclient.payjoin import send_payjoin, parse_payjoin_setup
 if __name__ == "__main__":
     wallet_name = sys.argv[1]
     mixdepth = int(sys.argv[2])
+
+    # for now these tests are lazy and only cover two scenarios
+    # (which may be the most likely):
+    # (1) TLS clearnet server
+    # (0) onion non-SSL server
+    # so the third argument is 0 or 1 as per that.
+    # the 4th argument, serverport, is required for (0),
+    # since it's an ephemeral HS address and must include the port
+    # Note on setting up the Hidden Service:
+    # this happens automatically when running test/payjoinserver.py
+    # under pytest, and it prints out the hidden service url after
+    # some seconds (just as it prints out the wallet hex).
+
     usessl = int(sys.argv[3])
-    bip21uri = None
+    serverport = None
     if len(sys.argv) > 4:
-        bip21uri = sys.argv[4]
+        serverport = sys.argv[4]
     load_test_config()
     jm_single().datadir = "."
     check_regtest()
-    if not bip21uri:
+    if not usessl:
-        if usessl == 0:
+        if not serverport:
-            pjurl = "http://127.0.0.1:8080"
+            print("test configuration error: usessl = 0 assumes onion "
+                  "address which must be specified as the fourth argument")
         else:
-            pjurl = "https://127.0.0.1:8080"
+            pjurl = "http://" + serverport
-        bip21uri = "bitcoin:2N7CAdEUjJW9tUHiPhDkmL9ukPtcukJMoxK?amount=0.3&pj=" + pjurl
+    else:
+        # hardcoded port for tests:
+        pjurl = "https://127.0.0.1:8080"
+    bip21uri = "bitcoin:2N7CAdEUjJW9tUHiPhDkmL9ukPtcukJMoxK?amount=0.3&pj=" + pjurl
+
     wallet_path = get_wallet_path(wallet_name, None)
     if jm_single().config.get("POLICY", "native") == "true":
         walletclass = SegwitWallet

test/payjoinserver.py

@@ -24,6 +24,29 @@ import jmbitcoin as btc
 from jmclient import load_test_config, jm_single,\
      SegwitWallet, SegwitLegacyWallet, cryptoengine
 
+import txtorcon
+
+def setup_failed(arg):
+    print("SETUP FAILED", arg)
+    reactor.stop()
+
+def create_onion_ep(t, hs_public_port):
+    return t.create_onion_endpoint(hs_public_port)
+
+def onion_listen(onion_ep, site):
+    return onion_ep.listen(site)
+
+def print_host(ep):
+    # required so tester can connect:
+    jmprint(str(ep.getHost()))
+
+def start_tor(site, hs_public_port):
+    d = txtorcon.connect(reactor)
+    d.addCallback(create_onion_ep, hs_public_port)
+    d.addErrback(setup_failed)
+    d.addCallback(onion_listen, site)
+    d.addCallback(print_host)
+
 # TODO change test for arbitrary payment requests
 payment_amt = 30000000
 
@@ -43,6 +66,8 @@ class PayjoinServer(Resource):
         super().__init__()
     isLeaf = True
     def render_GET(self, request):
+        # can be used e.g. to check if an ephemeral HS is up
+        # on Tor Browser:
         return "<html>Only for testing.</html>".encode("utf-8")
     def render_POST(self, request):
         """ The sender will use POST to send the initial
@@ -66,8 +91,9 @@ class PayjoinServer(Resource):
         receiver_utxos = {k: v for k, v in all_receiver_utxos.items(
             ) if k in receiver_utxos_keys}
     
-        # receiver will do other checks as discussed above, including payment
+        # receiver will do other checks but this is out of scope,
-        # amount; as discussed above, this is out of the scope of this PSBT test.
+        # since we only created this server (currently) to test our
+        # BIP78 client.
 
         # construct unsigned tx for payjoin-psbt:
         payjoin_tx_inputs = [(x.prevout.hash[::-1],
@@ -160,13 +186,19 @@ def test_start_payjoin_server(setup_payjoin_server):
     jmprint("\n\nTaker wallet seed : " + wallet_services[1]['seed'])
     jmprint("\n")
     server_wallet_service.sync_wallet(fast=True)
-    
     site = Site(PayjoinServer(server_wallet_service))
-    # TODO for now, just sticking with TLS test as non-encrypted
+    # TODO: this is just hardcoded manually for now:
-    # is unlikely to be used, but add that option.
+    use_tor = False
-    reactor.listenSSL(8080, site, contextFactory=get_ssl_context())
+    if use_tor:
-    #endpoint = endpoints.TCP4ServerEndpoint(reactor, 8080)
+        jmprint("Attempting to start Tor HS ...")
-    #endpoint.listen(site)
+        # port is hardcoded for test:
+        start_tor(site, 7081)
+    else:
+        # TODO for now, just sticking with TLS test as non-encrypted
+        # is unlikely to be used, but add that option.
+        reactor.listenSSL(8080, site, contextFactory=get_ssl_context())
+        #endpoint = endpoints.TCP4ServerEndpoint(reactor, 8080)
+        #endpoint.listen(site)
     reactor.run()
 
 @pytest.fixture(scope="module")

test/regtest_joinmarket.cfg

@@ -91,3 +91,10 @@ max_additional_fee_contribution = default
 # transaction; note it is decimal, not integer.
 min_fee_rate = 1.1
 
+# for payjoins to hidden service endpoints, the socks5 configuration:
+onion_socks5_host = localhost
+onion_socks5_port = 9050
+# in some exceptional case the HS may be SSL configured,
+# this feature is not yet implemented in code, but here for the
+# future:
+hidden_service_ssl = false