BIP78 sender protocol via daemon

This PR creates a client-daemon protocol for
the BIP78 sender, using the base protocol
`HTTPPassThrough` which provides tor and non-tor
agents with POST and GET request functionality.
As for Joinmarket coinjoins, the use of an in-process
daemon is the default option, but it can be isolated
by changing the `[DAEMON]` section of the config.
The receiver side of BIP78 will be addressed in a
future PR.

jmbase/jmbase/commands.py

@@ -281,3 +281,38 @@ class SNICKERRequestPowTarget(JMCommand):
 class SNICKERReceivePowTarget(JMCommand):
     arguments = [(b'server', Unicode()),
                  (b'targetbits', Integer())]
+
+""" Payjoin-related commands
+"""
+class BIP78SenderInit(JMCommand):
+    """ Initialization data for a BIP78 service.
+    See documentation of `netconfig` in
+    jmdaemon.HTTPPassThrough.on_INIT
+    """
+    arguments = [(b'netconfig', Unicode())]
+
+class BIP78SenderUp(JMCommand):
+    arguments = []
+
+class BIP78SenderOriginalPSBT(JMCommand):
+    """ Sends the payjoin url and the original
+    payment PSBT, base64 encoded,
+    from the client to the daemon,
+    to be sent as an http request to the receiver.
+    """
+    arguments = [(b'body', BigUnicode()),
+                 (b'params', Unicode())]
+
+class BIP78SenderReceiveProposal(JMCommand):
+    """ Sends the payjoin proposal PSBT, received
+    from the BIP78 Receiver, from the daemon to the client.
+    """
+    arguments = [(b'psbt', BigUnicode())]
+
+class BIP78ReceiverError(JMCommand):
+    """ Sends a message from daemon to client
+    indicating that the BIP78 receiver did not
+    accept the request, or there was a network error.
+    """
+    arguments = [(b'errormsg', Unicode()),
+                 (b'errorcode', Integer())]

jmclient/jmclient/__init__.py

@@ -29,7 +29,8 @@ from .blockchaininterface import (BlockchainInterface,
                                   RegtestBitcoinCoreInterface, BitcoinCoreInterface)
 from .snicker_receiver import SNICKERError, SNICKERReceiver
 from .client_protocol import (JMTakerClientProtocol, JMClientProtocolFactory,
-                              start_reactor, SNICKERClientProtocolFactory)
+                              start_reactor, SNICKERClientProtocolFactory,
+                              BIP78ClientProtocolFactory)
 from .podle import (set_commitment_file, get_commitment_file,
                     add_external_commitments,
                     PoDLE, generate_podle, get_podle_commitments,

jmclient/jmclient/client_protocol.py

@@ -44,6 +44,48 @@ class BaseClientProtocol(amp.AMP):
 class JMProtocolError(Exception):
     pass
 
+class BIP78ClientProtocol(BaseClientProtocol):
+
+    def __init__(self, manager, params,
+                 success_callback, failure_callback,
+                 tls_whitelist=[]):
+        self.manager = manager
+        self.success_callback = success_callback
+        self.failure_callback = failure_callback
+        self.params = params
+        if len(tls_whitelist) == 0:
+            if isinstance(jm_single().bc_interface,
+                          RegtestBitcoinCoreInterface):
+                tls_whitelist = ["127.0.0.1"]
+        self.tls_whitelist = tls_whitelist
+
+    def connectionMade(self):
+        netconfig = {"socks5_host": jm_single().config.get("PAYJOIN", "onion_socks5_host"),
+                     "socks5_port": jm_single().config.get("PAYJOIN", "onion_socks5_port"),
+                     "tls_whitelist": ",".join(self.tls_whitelist),
+                     "servers": [self.manager.server]}
+        d = self.callRemote(commands.BIP78SenderInit,
+                            netconfig=json.dumps(netconfig))
+        self.defaultCallbacks(d)
+
+    @commands.BIP78SenderUp.responder
+    def on_BIP78_SENDER_UP(self):
+        d = self.callRemote(commands.BIP78SenderOriginalPSBT,
+                            body=self.manager.initial_psbt.to_base64(),
+                            params=json.dumps(self.params))
+        self.defaultCallbacks(d)
+        return {"accepted": True}
+
+    @commands.BIP78SenderReceiveProposal.responder
+    def on_BIP78_SENDER_RECEIVE_PROPOSAL(self, psbt):
+        self.success_callback(psbt, self.manager)
+        return {"accepted": True}
+
+    @commands.BIP78ReceiverError.responder
+    def on_BIP78_RECEIVER_ERROR(self, errormsg, errorcode):
+        self.failure_callback(errormsg, errorcode, self.manager)
+        return {"accepted": True}
+
 class SNICKERClientProtocol(BaseClientProtocol):
 
     def __init__(self, client, servers, tls_whitelist=[], oneshot=False):
@@ -635,6 +677,19 @@ class SNICKERClientProtocolFactory(protocol.ClientFactory):
         self.servers = servers
         self.oneshot = oneshot
 
+class BIP78ClientProtocolFactory(protocol.ClientFactory):
+    protocol = BIP78ClientProtocol
+    def buildProtocol(self, addr):
+        return self.protocol(self.manager, self.params,
+                    self.success_callback,
+                    self.failure_callback)
+    def __init__(self, manager, params, success_callback,
+                 failure_callback):
+        self.manager = manager
+        self.params = params
+        self.success_callback = success_callback
+        self.failure_callback = failure_callback
+
 class JMClientProtocolFactory(protocol.ClientFactory):
     protocol = JMTakerClientProtocol
 
@@ -653,7 +708,8 @@ class JMClientProtocolFactory(protocol.ClientFactory):
     def buildProtocol(self, addr):
         return self.protocol(self, self.client)
 
-def start_reactor(host, port, factory=None, snickerfactory=None, ish=True,
+def start_reactor(host, port, factory=None, snickerfactory=None,
+                  bip78=False, jm_coinjoin=True, ish=True,
                   daemon=False, rs=True, gui=False): #pragma: no cover
     #(Cannot start the reactor in tests)
     #Not used in prod (twisted logging):
@@ -663,33 +719,51 @@ def start_reactor(host, port, factory=None, snickerfactory=None, ish=True,
     if daemon:
         try:
             from jmdaemon import JMDaemonServerProtocolFactory, start_daemon, \
-                 SNICKERDaemonServerProtocolFactory
+                 SNICKERDaemonServerProtocolFactory, BIP78ServerProtocolFactory
         except ImportError:
             jlog.error("Cannot start daemon without jmdaemon package; "
                        "either install it, and restart, or, if you want "
                        "to run the daemon separately, edit the DAEMON "
                        "section of the config. Quitting.")
             return
-        dfactory = JMDaemonServerProtocolFactory()
+        if jm_coinjoin:
+            dfactory = JMDaemonServerProtocolFactory()
         if snickerfactory:
             sdfactory = SNICKERDaemonServerProtocolFactory()
-        orgport = port
-        while True:
-            try:
-                start_daemon(host, port, dfactory, usessl,
-                             './ssl/key.pem', './ssl/cert.pem')
-                jlog.info("Listening on port " + str(port))
-                break
-            except Exception:
-                jlog.warn("Cannot listen on port " + str(port) + ", trying next port")
-                if port >= (orgport + 100):
-                    jlog.error("Tried 100 ports but cannot listen on any of them. Quitting.")
-                    sys.exit(EXIT_FAILURE)
-                port += 1
+        if bip78:
+            bip78factory = BIP78ServerProtocolFactory()
+        # ints are immutable in python, to pass by ref we use
+        # an array object:
+        port_a = [port]
+        def start_daemon_on_port(p, f, name, port_offset):
+            orgp = p[0]
+            while True:
+                try:
+                    start_daemon(host, p[0] - port_offset, f, usessl,
+                        './ssl/key.pem', './ssl/cert.pem')
+                    jlog.info("{} daemon listening on port {}".format(
+                        name, str(p[0] - port_offset)))
+                    break
+                except Exception:
+                    jlog.warn("Cannot listen on port " + str(
+                        p[0] - port_offset) + ", trying next port")
+                    if p[0] >= (orgp + 100):
+                        jlog.error("Tried 100 ports but cannot "
+                                   "listen on any of them. Quitting.")
+                        sys.exit(EXIT_FAILURE)
+                    p[0] += 1
+        if jm_coinjoin:
+            # TODO either re-apply this port incrementing logic
+            # to other protocols, or re-work how the ports work entirely.
+            start_daemon_on_port(port_a, dfactory, "Joinmarket", 0)
+        # (See above) For now these other two are just on ports that are 1K offsets.
         if snickerfactory:
-            start_daemon(host, port-1000, sdfactory, usessl,
-                                 './ssl/key.pem', './ssl/cert.pem')
-            jlog.info("(SNICKER) Listening on port " + str(port-1000))
+            start_daemon_on_port(port_a, sdfactory, "SNICKER", 1000)
+        if bip78:
+            start_daemon_on_port(port_a, bip78factory, "BIP78", 2000)
+
+    # Note the reactor.connect*** entries do not include BIP78 which
+    # starts in jmclient.payjoin:
     if usessl:
         if factory:
             reactor.connectSSL(host, port, factory, ClientContextFactory())

jmclient/jmclient/payjoin.py

@@ -19,13 +19,14 @@ import json
 import random
 from io import BytesIO
 from pprint import pformat
-from jmbase import BytesProducer, bintohex, jmprint
+from jmbase import bintohex, jmprint
 from .configure import get_log, jm_single
 import jmbitcoin as btc
 from .wallet import PSBTWalletMixin, SegwitLegacyWallet, SegwitWallet, estimate_tx_fee
 from .wallet_service import WalletService
 from .taker_utils import direct_send
-from jmclient import RegtestBitcoinCoreInterface, select_one_utxo, process_shutdown
+from jmclient import (RegtestBitcoinCoreInterface, select_one_utxo,
+                      process_shutdown, BIP78ClientProtocolFactory)
 
 """
 For some documentation see:
@@ -43,36 +44,6 @@ INPUT_VSIZE_LEGACY = 148
 INPUT_VSIZE_SEGWIT_LEGACY = 91
 INPUT_VSIZE_SEGWIT_NATIVE = 68
 
-# txtorcon outputs erroneous warnings about hiddenservice directory strings,
-# annoyingly, so we suppress it here:
-import warnings
-warnings.filterwarnings("ignore")
-
-""" This whitelister allows us to accept any cert for a specific
-    domain, and is to be used for testing only; the default Agent
-    behaviour of twisted.web.client.Agent for https URIs is
-    the correct one in production (i.e. uses local trust store).
-"""
-@implementer(IPolicyForHTTPS)
-class WhitelistContextFactory(object):
-    def __init__(self, good_domains=None):
-        """
-        :param good_domains: List of domains. The URLs must be in bytes
-        """
-        if not good_domains:
-            self.good_domains = []
-        else:
-            self.good_domains = good_domains
-        # by default, handle requests like a browser would
-        self.default_policy = BrowserLikePolicyForHTTPS()
-
-    def creatorForNetloc(self, hostname, port):
-        # check if the hostname is in the the whitelist,
-        # otherwise return the default policy
-        if hostname in self.good_domains:
-            return CertificateOptions(verify=False)
-        return self.default_policy.creatorForNetloc(hostname, port)
-
 class JMPayjoinManager(object):
     """ An encapsulation of state for an
     ongoing Payjoin payment. Allows reporting
@@ -435,6 +406,7 @@ class JMPayjoinManager(object):
 
         self.user_info_callback("Choosing one coin at random")
         try:
+            print('attempting to select from mixdepth: ', str(self.mixdepth))
             my_utxos = self.wallet_service.select_utxos(
                 self.mixdepth, jm_single().DUST_THRESHOLD,
                 select_fn=select_one_utxo)
@@ -519,76 +491,32 @@ def get_max_additional_fee_contribution(manager):
                   "contribution of: " + str(max_additional_fee_contribution))
     return max_additional_fee_contribution
 
-def send_payjoin(manager, accept_callback=None,
-                 info_callback=None, return_deferred=False):
-    """ Given a JMPayjoinManager object `manager`, initialised with the
-    payment request data from the server, use its wallet_service to construct
-    a payment transaction, with coins sourced from mixdepth `mixdepth`,
-    then wait for the server response, parse the PSBT, perform checks and complete sign.
-    The info and accept callbacks are to ask the user to confirm the creation of
-    the original payment transaction (None defaults to terminal/CLI processing),
-    and are as defined in `taker_utils.direct_send`.
-
-    Returns:
-    (True, None) in case of payment setup successful (response will be delivered
-     asynchronously) - the `manager` object can be inspected for more detail.
-    (False, errormsg) in case of failure.
+def make_payment_psbt(manager, accept_callback=None, info_callback=None):
+    """ Creates a valid payment transaction and PSBT for it,
+    and adds it to the JMPayjoinManager instance passed as argument.
+    Wallet should already be synced before calling here.
+    Returns True, None if successful or False, errormsg if not.
     """
-
-    # wallet should already be synced before calling here;
     # we can create a standard payment, but have it returned as a PSBT.
     assert isinstance(manager, JMPayjoinManager)
     assert manager.wallet_service.synced
-    payment_psbt = direct_send(manager.wallet_service, manager.amount, manager.mixdepth,
-                             str(manager.destination), accept_callback=accept_callback,
-                             info_callback=info_callback,
-                             with_final_psbt=True, optin_rbf=True)
+    payment_psbt = direct_send(manager.wallet_service, manager.amount,
+                               manager.mixdepth, str(manager.destination),
+                               accept_callback=accept_callback,
+                               info_callback=info_callback,
+                               with_final_psbt=True, optin_rbf=True)
     if not payment_psbt:
         return (False, "could not create non-payjoin payment")
 
-    # TLS whitelist is for regtest testing, it is treated as hostnames for
-    # which tls certificate verification is ignored.
-    tls_whitelist = None
-    if isinstance(jm_single().bc_interface, RegtestBitcoinCoreInterface):
-        tls_whitelist = [b"127.0.0.1"]
-
     manager.set_payment_tx_and_psbt(payment_psbt)
 
-    # add delayed call to broadcast this after 1 minute
-    manager.timeout_fallback_dc = reactor.callLater(60,
-                                        fallback_nonpayjoin_broadcast,
-                                        b"timeout", manager)
-
-    # Now we send the request to the server, with the encoded
-    # payment PSBT
-
-    # First we create a twisted web Agent object:
-
-    # TODO genericize/move out/use library function:
-    def is_hs_uri(s):
-        x = urlparse.urlparse(s)
-        if x.hostname.endswith(".onion"):
-            return (x.scheme, x.hostname, x.port)
-        return False
-
-    tor_url_data = is_hs_uri(manager.server)
-    if tor_url_data:
-        # note the return value is currently unused here
-        socks5_host = jm_single().config.get("PAYJOIN", "onion_socks5_host")
-        socks5_port = int(jm_single().config.get("PAYJOIN", "onion_socks5_port"))
-        # note: SSL not supported at the moment:
-        torEndpoint = TCP4ClientEndpoint(reactor, socks5_host, socks5_port)
-        agent = tor_agent(reactor, torEndpoint)
-    else:
-        if not tls_whitelist:
-            agent = Agent(reactor)
-        else:
-            agent = Agent(reactor,
-                contextFactory=WhitelistContextFactory(tls_whitelist))
-
-    body = BytesProducer(payment_psbt.to_base64().encode("utf-8"))
+    return (True, None)
 
-    #Set the query parameters for the request:
+def make_payjoin_request_params(manager):
+    """ Returns the query parameters for the request
+    to the payjoin receiver, based on the configuration
+    of the given JMPayjoinManager instance.
+    """
 
     # construct the URI from the given parameters
     pj_version = jm_single().config.getint("PAYJOIN",
@@ -614,27 +542,39 @@ def send_payjoin(manager, accept_callback=None,
     min_fee_rate = float(jm_single().config.get("PAYJOIN", "min_fee_rate"))
     params["minfeerate"] = min_fee_rate
 
-    destination_url = manager.server.encode("utf-8")
-    url_parts = list(urlparse.urlparse(destination_url))
-    url_parts[4] = urlencode(params).encode("utf-8")
-    destination_url = urlparse.urlunparse(url_parts)
-    # TODO what to use as user agent?
-    d = agent.request(b"POST", destination_url,
-        Headers({"User-Agent": ["Twisted Web Client Example"],
-                "Content-Type": ["text/plain"]}),
-        bodyProducer=body)
-    d.addCallback(receive_payjoin_proposal_from_server, manager)
-    # note that the errback (here "noResponse") is *not* triggered
-    # by a server rejection (which is accompanied by a non-200
-    # status code returned), but by failure to communicate.
-    def noResponse(failure):
-        failure.trap(ResponseFailed, ConnectionRefusedError,
-                     HostUnreachableError, ConnectionLost)
-        log.error(failure.value)
-        fallback_nonpayjoin_broadcast(b"connection failed", manager)
-    d.addErrback(noResponse)
-    if return_deferred:
-        return d
+    return params
+
+def send_payjoin(manager, accept_callback=None,
+                 info_callback=None, return_deferred=False):
+    """ Given a JMPayjoinManager object `manager`, initialised with the
+    payment request data from the server, use its wallet_service to construct
+    a payment transaction, with coins sourced from mixdepth `manager.mixdepth`,
+    then wait for the server response, parse the PSBT, perform checks and complete sign.
+    The info and accept callbacks are to ask the user to confirm the creation of
+    the original payment transaction (None defaults to terminal/CLI processing),
+    and are as defined in `taker_utils.direct_send`.
+
+    Returns:
+    (True, None) in case of payment setup successful (response will be delivered
+     asynchronously) - the `manager` object can be inspected for more detail.
+    (False, errormsg) in case of failure.
+    """
+    success, errmsg = make_payment_psbt(manager, accept_callback, info_callback)
+    if not success:
+        return (False, errmsg)
+
+    # add delayed call to broadcast this after 1 minute
+    manager.timeout_fallback_dc = reactor.callLater(60,
+                                        fallback_nonpayjoin_broadcast,
+                                        b"timeout", manager)
+
+    params = make_payjoin_request_params(manager)
+    factory = BIP78ClientProtocolFactory(manager, params,
+        process_payjoin_proposal_from_server, process_error_from_server)
+    # TODO add SSL option as for other protocol instances:
+    reactor.connectTCP(jm_single().config.get("DAEMON", "daemon_host"),
+                  jm_single().config.getint("DAEMON", "daemon_port")-2000,
+                  factory)
     return (True, None)
 
 def fallback_nonpayjoin_broadcast(err, manager):
@@ -667,19 +607,14 @@ def fallback_nonpayjoin_broadcast(err, manager):
         manager.timeout_fallback_dc.cancel()
     quit()
 
-def receive_payjoin_proposal_from_server(response, manager):
+def process_error_from_server(errormsg, errorcode, manager):
     assert isinstance(manager, JMPayjoinManager)
-    # no attempt at chunking or handling incrementally is needed
-    # here. The body should be a byte string containing the
-    # new PSBT, or a jsonified error page.
-    d = readBody(response)
-    # if the response code is not 200 OK, we must assume payjoin
-    # attempt has failed, and revert to standard payment.
-    if int(response.code) != 200:
-        log.warn("Receiver returned error code: " + str(response.code))
-        d.addCallback(fallback_nonpayjoin_broadcast, manager)
-        return
-    d.addCallback(process_payjoin_proposal_from_server, manager)
+    # payjoin attempt has failed, we revert to standard payment.
+    assert int(errorcode) != 200
+    log.warn("Receiver returned error code: {}, message: {}".format(
+        errorcode, errormsg))
+    fallback_nonpayjoin_broadcast(errormsg.encode("utf-8"), manager)
+    return
 
 def process_payjoin_proposal_from_server(response_body, manager):
     assert isinstance(manager, JMPayjoinManager)

jmclient/test/test_payjoin.py

@@ -8,15 +8,22 @@ import sys
 import pytest
 from twisted.internet import reactor
 from twisted.web.server import Site
+from twisted.web.client import readBody
+from twisted.web.http_headers import Headers
 from twisted.trial import unittest
+import urllib.parse as urlparse
+from urllib.parse import urlencode
 
-from jmbase import get_log, jmprint
+from jmbase import get_log, jmprint, BytesProducer
 from jmbitcoin import (CCoinAddress, encode_bip21_uri,
                        amount_to_btc, amount_to_sat)
 from jmclient import cryptoengine
 from jmclient import (load_test_config, jm_single,
                       SegwitLegacyWallet, SegwitWallet,
-                      PayjoinServer, parse_payjoin_setup, send_payjoin)
+                      PayjoinServer, parse_payjoin_setup,
+                      send_payjoin)
+from jmclient.payjoin import make_payjoin_request_params, make_payment_psbt
+from jmclient.payjoin import process_payjoin_proposal_from_server
 from commontest import make_wallets
 from test_coinjoin import make_wallets_to_list, create_orderbook, sync_wallets
 
@@ -75,7 +82,23 @@ class TrialTestPayjoinServer(unittest.TestCase):
         self.manager = parse_payjoin_setup(bip78_uri, self.wallet_services[1], 0)
         self.manager.mode = "testing"
         self.site = site
-        return send_payjoin(self.manager, return_deferred=True)
+        success, msg = make_payment_psbt(self.manager)
+        assert success, msg
+        params = make_payjoin_request_params(self.manager)
+        # avoiding backend daemon (testing only jmclient code here),
+        # we send the http request manually:
+        from jmbase import get_nontor_agent, wrapped_urlparse
+        serv = b"http://127.0.0.1:47083"
+        agent = get_nontor_agent()
+        body = BytesProducer(self.manager.initial_psbt.to_base64().encode("utf-8"))
+        url_parts = list(wrapped_urlparse(serv))
+        url_parts[4] = urlencode(params).encode("utf-8")
+        destination_url = urlparse.urlunparse(url_parts)
+        d = agent.request(b"POST", destination_url,
+                          Headers({"Content-Type": ["text/plain"]}),
+                          bodyProducer=body)
+        d.addCallback(bip78_receiver_response, self.manager)
+        return d
 
     def tearDown(self):
         for dc in reactor.getDelayedCalls():
@@ -85,6 +108,21 @@ class TrialTestPayjoinServer(unittest.TestCase):
                            self.ssb, self.rsb)
         assert res, "final checks failed"
 
+def bip78_receiver_response(response, manager):
+    d = readBody(response)
+    # if the response code is not 200 OK, we must assume payjoin
+    # attempt has failed, and revert to standard payment.
+    if int(response.code) != 200:
+        d.addCallback(process_receiver_errormsg, response.code)
+        return
+    d.addCallback(process_receiver_psbt, manager)
+
+def process_receiver_errormsg(r, c):
+    print("Failed: r, c: ", r, c)
+
+def process_receiver_psbt(response, manager):
+    process_payjoin_proposal_from_server(response.decode("utf-8"), manager)
+
 def getbals(wallet_service, mixdepth):
     """ Retrieves balances for a mixdepth and the 'next'
     """

jmdaemon/jmdaemon/__init__.py

@@ -9,7 +9,8 @@ from .message_channel import MessageChannel, MessageChannelCollection
 from .orderbookwatch import OrderbookWatch
 from jmbase import commands
 from .daemon_protocol import (JMDaemonServerProtocolFactory, JMDaemonServerProtocol,
-                              start_daemon, SNICKERDaemonServerProtocolFactory)
+                              start_daemon, SNICKERDaemonServerProtocolFactory,
+                              BIP78ServerProtocolFactory, BIP78ServerProtocol)
 from .protocol import (COMMAND_PREFIX, ORDER_KEYS, NICK_HASH_LENGTH,
                        NICK_MAX_ENCODED, JM_VERSION, JOINMARKET_NICK_HEADER)
 from .message_channel import MessageChannelCollection

jmdaemon/jmdaemon/daemon_protocol.py

@@ -16,7 +16,8 @@ from twisted.protocols import amp
 from twisted.internet import reactor, ssl
 from twisted.internet.protocol import ServerFactory
 from twisted.internet.error import (ConnectionLost, ConnectionAborted,
-                                    ConnectionClosed, ConnectionDone)
+                                    ConnectionClosed, ConnectionDone,
+                                    ConnectionRefusedError)
 from twisted.web.http_headers import Headers
 from twisted.web.client import ResponseFailed, readBody
 from txtorcon.socks import HostUnreachableError
@@ -131,6 +132,14 @@ class HTTPPassThrough(amp.AMP):
         destination_url = urlparse.urlunparse(url_parts)
         return (agent, destination_url)
 
+    def getDefaultHeaders(self):
+        # Deliberately sending NO headers other than
+        # Content-Type by default;
+        # this could be a tricky point for anonymity of users,
+        # as much boilerplate code will not create
+        # requests that look like this.
+        return Headers({"Content-Type": ["text/plain"]})
+
     def getRequest(self, server, success_callback, url=None, headers=None):
         """ Make GET request to server server, if response received OK,
 	passed to success_callback, which must have function signature
@@ -142,7 +151,7 @@ class HTTPPassThrough(amp.AMP):
         # Deliberately sending NO headers; this could be a tricky point
         # for anonymity of users, as much boilerplate code will not create
         # requests that look like this.
-        headers = Headers({}) if not headers else headers
+        headers = self.getDefaultHeaders() if not headers else headers
         d = agent.request(b"GET", destination_url, headers)
         d.addCallback(success_callback, server)
         # note that the errback (here "noResponse") is *not* triggered
@@ -154,13 +163,18 @@ class HTTPPassThrough(amp.AMP):
             log.msg(failure.value)
         d.addErrback(noResponse)
 
-    def postRequest(self, body, server, success_callback, headers=None):
+    def postRequest(self, body, server, success_callback,
+                    url=None, params=None, headers=None):
         """ Pass body of post request as string, will be encoded here.
 	"""
-        agent, destination_url = self.getAgentDestination(server)
+        agent, destination_url = self.getAgentDestination(server,
+                                                    params=params)
+        if url:
+            destination_url = destination_url + url
         body = BytesProducer(body.encode("utf-8"))
+        headers = self.getDefaultHeaders() if not headers else headers
         d = agent.request(b"POST", destination_url,
-            Headers({}), bodyProducer=body)
+            headers, bodyProducer=body)
         d.addCallback(success_callback, server)
         # note that the errback (here "noResponse") is *not* triggered
         # by a server rejection (which is accompanied by a non-200
@@ -169,9 +183,9 @@ class HTTPPassThrough(amp.AMP):
             failure.trap(ResponseFailed, ConnectionRefusedError,
                          HostUnreachableError, ConnectionLost)
             log.msg(failure.value)
-            self.callRemote(SNICKERProposalsServerResponse,
-                            response="failure to connect",
-                            server=server)
+            self.callRemote(BIP78ReceiverError,
+                            errormsg="failure to connect",
+                            errorcode=10000)
         d.addErrback(noResponse)
 
     def checkClientResponse(self, response):
@@ -191,6 +205,42 @@ class HTTPPassThrough(amp.AMP):
         d.addCallback(self.checkClientResponse)
         d.addErrback(self.defaultErrback)
 
+class BIP78ServerProtocol(HTTPPassThrough):
+    @BIP78SenderInit.responder
+    def on_BIP78_SENDER_INIT(self, netconfig):
+        self.on_INIT(netconfig)
+        d = self.callRemote(BIP78SenderUp)
+        self.defaultCallbacks(d)
+        return {"accepted": True}
+
+    @BIP78SenderOriginalPSBT.responder
+    def on_BIP78_SENDER_ORIGINAL_PSBT(self, body, params):
+        self.postRequest(body, self.servers[0],
+                self.bip78_receiver_response,
+                params=json.loads(params),
+                headers=Headers({"Content-Type": ["text/plain"]}))
+        return {"accepted": True}
+
+    def bip78_receiver_response(self, response, server):
+        d = readBody(response)
+        # if the response code is not 200 OK, we must assume payjoin
+        # attempt has failed, and revert to standard payment.
+        if int(response.code) != 200:
+            d.addCallback(self.process_receiver_errormsg, response.code)
+            return
+        d.addCallback(self.process_receiver_psbt)
+
+    def process_receiver_errormsg(self, response, errorcode):
+        d = self.callRemote(BIP78ReceiverError,
+                            errormsg=response.decode("utf-8"),
+                            errorcode=errorcode)
+        self.defaultCallbacks(d)
+
+    def process_receiver_psbt(self, response):
+        d = self.callRemote(BIP78SenderReceiveProposal,
+                            psbt=response.decode("utf-8"))
+        self.defaultCallbacks(d)
+
 class SNICKERDaemonServerProtocol(HTTPPassThrough):
 
     @SNICKERProposerPostProposals.responder
@@ -836,6 +886,9 @@ class JMDaemonServerProtocolFactory(ServerFactory):
 class SNICKERDaemonServerProtocolFactory(ServerFactory):
     protocol = SNICKERDaemonServerProtocol
 
+class BIP78ServerProtocolFactory(ServerFactory):
+    protocol = BIP78ServerProtocol
+
 def start_daemon(host, port, factory, usessl=False, sslkey=None, sslcert=None):
     if usessl:
         assert sslkey

scripts/joinmarket-qt.py

@@ -37,8 +37,6 @@ elif platform.system() == 'Darwin':
 else:
     MONOSPACE_FONT = 'monospace'
 
-import jmbitcoin as btc
-
 # This is required to change the decimal separator
 # to '.' regardless of the locale; TODO don't require
 # this, but will require other edits for parsing amounts.
@@ -63,6 +61,7 @@ JM_GUI_VERSION = '21dev'
 from jmbase import get_log, stop_reactor
 from jmbase.support import DUST_THRESHOLD, EXIT_FAILURE, utxo_to_utxostr,\
     bintohex, hextobin, JM_CORE_VERSION
+import jmbitcoin as btc
 from jmclient import load_program_config, get_network, update_persist_config,\
     open_test_wallet_maybe, get_wallet_path,\
     jm_single, validate_address, weighted_order_choose, Taker,\
@@ -74,7 +73,8 @@ from jmclient import load_program_config, get_network, update_persist_config,\
     NO_ROUNDING, get_max_cj_fee_values, get_default_max_absolute_fee, \
     get_default_max_relative_fee, RetryableStorageError, add_base_options, \
     BTCEngine, BTC_P2SH_P2WPKH, FidelityBondMixin, wallet_change_passphrase, \
-    parse_payjoin_setup, send_payjoin, JMBIP78ReceiverManager
+    parse_payjoin_setup, send_payjoin, JMBIP78ReceiverManager, \
+    BIP78ClientProtocolFactory
 from qtsupport import ScheduleWizard, TumbleRestartWizard, config_tips,\
     config_types, QtHandler, XStream, Buttons, OkButton, CancelButton,\
     PasswordDialog, MyTreeWidget, JMQtMessageBox, BLUE_FG,\
@@ -292,6 +292,8 @@ class SpendTab(QWidget):
         self.spendstate.reset() #trigger callback to 'ready' state
         # needed to be saved for parse_payjoin_setup()
         self.bip21_uri = None
+        # avoid re-starting BIP78 daemon unnecessarily:
+        self.bip78_daemon_started = False
 
     def switchToBIP78Payjoin(self, endpoint_url):
         self.numCPLabel.setVisible(False)
@@ -746,7 +748,18 @@ class SpendTab(QWidget):
 
         if bip78url:
             manager = parse_payjoin_setup(self.bip21_uri,
-                mainWindow.wallet_service, mixdepth, "joinmarket-qt")
+                    mainWindow.wallet_service, mixdepth, "joinmarket-qt")
+            # start BIP78 AMP protocol if not yet up:
+            if not self.bip78_daemon_started:
+                daemon = jm_single().config.getint("DAEMON", "no_daemon")
+                daemon = True if daemon == 1 else False
+                start_reactor(jm_single().config.get("DAEMON", "daemon_host"),
+                       jm_single().config.getint("DAEMON", "daemon_port"),
+                       bip78=True, jm_coinjoin=False,
+                       ish=False,
+                       daemon=daemon,
+                       gui=True)
+                self.bip78_daemon_started = True
             # disable form fields until payment is done
             self.addressInput.setEnabled(False)
             self.pjEndpointInput.setEnabled(False)

scripts/joinmarketd.py

@@ -15,11 +15,12 @@ def startup_joinmarketd(host, port, usessl, factories=None,
     startLogging(sys.stdout)
     if not factories:
         factories = [jmdaemon.JMDaemonServerProtocolFactory(),
-                     jmdaemon.SNICKERDaemonServerProtocolFactory()]
+                     jmdaemon.SNICKERDaemonServerProtocolFactory(),
+                     jmdaemon.BIP78ServerProtocolFactory()]
     for factory in factories:
         jmdaemon.start_daemon(host, port, factory, usessl,
                               './ssl/key.pem', './ssl/cert.pem')
-        port += 1
+        port -= 1000
     if finalizer:
         reactor.addSystemEventTrigger("after", "shutdown", finalizer,
                                       finalizer_args)

scripts/sendpayment.py

@@ -298,11 +298,16 @@ def main():
                 log.info("All transactions completed correctly")
             reactor.stop()
 
+    nodaemon = jm_single().config.getint("DAEMON", "no_daemon")
+    daemon = True if nodaemon == 1 else False
+    dhost = jm_single().config.get("DAEMON", "daemon_host")
+    dport = jm_single().config.getint("DAEMON", "daemon_port")
     if bip78url:
         # TODO sanity check wallet type is segwit
         manager = parse_payjoin_setup(args[1], wallet_service, options.mixdepth)
         reactor.callWhenRunning(send_payjoin, manager)
-        reactor.run()
+        # JM is default, so must be switched off explicitly in this call:
+        start_reactor(dhost, dport, bip78=True, jm_coinjoin=False, daemon=daemon)
         return
 
     else:
@@ -312,13 +317,10 @@ def main():
                       max_cj_fee=maxcjfee,
                       callbacks=(filter_orders_callback, None, taker_finished))
     clientfactory = JMClientProtocolFactory(taker)
-    nodaemon = jm_single().config.getint("DAEMON", "no_daemon")
-    daemon = True if nodaemon == 1 else False
+
     if jm_single().config.get("BLOCKCHAIN", "network") == "regtest":
         startLogging(sys.stdout)
-    start_reactor(jm_single().config.get("DAEMON", "daemon_host"),
-                  jm_single().config.getint("DAEMON", "daemon_port"),
-                  clientfactory, daemon=daemon)
+    start_reactor(dhost, dport, clientfactory, daemon=daemon)
 
 if __name__ == "__main__":
     main()