Add bip78 payjoin module and client-server test:

See:
https://github.com/bitcoin/bips/blob/master/bip-0078.mediawiki

Adds a new module jmclient.payjoin which implements
the full sender workflow from a BIP21 uri to a
payjoin broadcast, state is managed in JMPayjoinManager,
includes all checks as per documentation of
btcpayserver (and later, BIP78).
Added simple client and server implementations in
test/payjoinclient.py and test/payjoinserver.py
which allow a full end to end test on regtest.

Add TLS support to payjoin tests:
Note: the jmclient.payjoin module already
supports TLS by default (Agent object), but
here we add the ability to test without
certificate verification. Both test/payjoinclient.py
and test/payjoinserver.py now support TLS, but
the server needs a key and certificate in its
directory to run.

Adds BIP78 payjoin option to sendpayment.py

Users can use a bip21 uri with the "pj" field to
send a payment to a remote server.

Removes require_path_templates setting from KeyStore call
in PSBTWalletMixin.sign_psbt

jmbase/jmbase/__init__.py

@@ -6,6 +6,7 @@ from .support import (get_log, chunks, debug_silence, jmprint,
                       hextobin, lehextobin, utxostr_to_utxo,
                       utxo_to_utxostr, EXIT_ARGERROR, EXIT_FAILURE,
                       EXIT_SUCCESS, hexbin, dictchanger, listchanger,
-                      cv, JM_WALLET_NAME_PREFIX, JM_APP_NAME)
+                      JM_WALLET_NAME_PREFIX, JM_APP_NAME)
+from .bytesprod import BytesProducer
 from .commands import *
 

jmbase/jmbase/bytesprod.py

@@ -0,0 +1,22 @@
+""" See https://twistedmatrix.com/documents/current/web/howto/client.html
+"""
+from zope.interface import implementer
+
+from twisted.internet.defer import succeed
+from twisted.web.iweb import IBodyProducer
+
+@implementer(IBodyProducer)
+class BytesProducer(object):
+    def __init__(self, body):
+        self.body = body
+        self.length = len(body)
+
+    def startProducing(self, consumer):
+        consumer.write(self.body)
+        return succeed(None)
+
+    def pauseProducing(self):
+        pass
+
+    def stopProducing(self):
+        pass

jmbitcoin/jmbitcoin/__init__.py

@@ -11,6 +11,8 @@ from bitcointx.core import (x, b2x, b2lx, lx, COutPoint, CTxOut, CTxIn,
                             CMutableTransaction, Hash160,
                             coins_to_satoshi, satoshi_to_coins)
 from bitcointx.core.key import KeyStore
+from bitcointx.wallet import (P2SHCoinAddress, P2SHCoinAddressError,
+                              P2WPKHCoinAddress, P2WPKHCoinAddressError)
 from bitcointx.core.script import (CScript, OP_0, SignatureHash, SIGHASH_ALL,
                                    SIGVERSION_WITNESS_V0, CScriptWitness)
 from bitcointx.core.psbt import (PartiallySignedTransaction, PSBT_Input,

jmclient/jmclient/__init__.py

@@ -56,7 +56,7 @@ from .wallet_service import WalletService
 from .maker import Maker, P2EPMaker
 from .yieldgenerator import YieldGenerator, YieldGeneratorBasic, ygmain
 from .snicker_receiver import SNICKERError, SNICKERReceiver
-
+from .payjoin import parse_payjoin_setup, send_payjoin
 # Set default logging handler to avoid "No handler found" warnings.
 
 try:

jmclient/jmclient/payjoin.py

@@ -0,0 +1,503 @@
+from zope.interface import implementer
+from twisted.internet import reactor
+from twisted.web.client import (Agent, readBody, ResponseFailed,
+                                BrowserLikePolicyForHTTPS)
+from twisted.web.iweb import IPolicyForHTTPS
+from twisted.internet.ssl import CertificateOptions
+from twisted.web.http_headers import Headers
+
+import json
+from pprint import pformat
+from jmbase import BytesProducer
+from .configure import get_log, jm_single
+import jmbitcoin as btc
+from .wallet import PSBTWalletMixin, SegwitLegacyWallet, SegwitWallet
+from .wallet_service import WalletService
+from .taker_utils import direct_send
+
+"""
+For some documentation see:
+    https://github.com/btcpayserver/btcpayserver-doc/blob/master/Payjoin-spec.md
+    which is a delta to:
+    https://github.com/bitcoin/bips/blob/master/bip-0079.mediawiki
+"""
+log = get_log()
+
+""" This whitelister allows us to accept any cert for a specific
+    domain, and is to be used for testing only; the default Agent
+    behaviour of twisted.web.client.Agent for https URIs is
+    the correct one in production (i.e. uses local trust store).
+"""
+@implementer(IPolicyForHTTPS)
+class WhitelistContextFactory(object):
+    def __init__(self, good_domains=None):
+        """
+        :param good_domains: List of domains. The URLs must be in bytes
+        """
+        if not good_domains:
+            self.good_domains = []
+        else:
+            self.good_domains = good_domains
+        # by default, handle requests like a browser would
+        self.default_policy = BrowserLikePolicyForHTTPS()
+
+    def creatorForNetloc(self, hostname, port):
+        # check if the hostname is in the the whitelist,
+        # otherwise return the default policy
+        if hostname in self.good_domains:
+            return CertificateOptions(verify=False)
+        return self.default_policy.creatorForNetloc(hostname, port)
+
+class JMPayjoinManager(object):
+    """ An encapsulation of state for an
+    ongoing Payjoin payment. Allows reporting
+    details of the outcome of a Payjoin attempt.
+    """
+
+    # enum such that progress can be
+    # reported
+    JM_PJ_NONE = 0
+    JM_PJ_INIT = 1
+    JM_PJ_PAYMENT_CREATED = 2
+    JM_PJ_PAYMENT_SENT = 3
+    JM_PJ_PARTIAL_RECEIVED = 4
+    JM_PJ_PARTIAL_REJECTED = 5
+    JM_PJ_PAYJOIN_COSIGNED = 6
+    JM_PJ_PAYJOIN_BROADCAST = 7
+    JM_PJ_PAYJOIN_BROADCAST_FAILED = 8
+
+    pj_state = JM_PJ_NONE
+
+    def __init__(self, wallet_service, mixdepth, destination,
+                 amount, server):
+        assert isinstance(wallet_service, WalletService)
+        # payjoin is not supported for non-segwit wallets:
+        assert isinstance(wallet_service.wallet,
+                          (SegwitWallet, SegwitLegacyWallet))
+        # our payjoin implementation requires PSBT
+        assert isinstance(wallet_service.wallet, PSBTWalletMixin)
+        self.wallet_service = wallet_service
+        # mixdepth from which payment is sourced
+        assert isinstance(mixdepth, int)
+        self.mixdepth = mixdepth
+        assert isinstance(destination, btc.CCoinAddress)
+        self.destination = destination
+        assert isinstance(amount, int)
+        assert amount > 0
+        self.amount = amount
+        self.server = server
+        self.pj_state = self.JM_PJ_INIT
+        self.payment_tx = None
+        self.initial_psbt = None
+        self.payjoin_psbt = None
+        self.final_psbt = None
+        # change is initialized as None
+        # in case there is no change:
+        self.change_out = None
+
+    def set_payment_tx_and_psbt(self, in_psbt):
+        assert isinstance(in_psbt, btc.PartiallySignedTransaction)
+        self.initial_psbt = in_psbt
+        # any failure here is a coding error, as it is fully
+        # under our control.
+        assert self.sanity_check_initial_payment()
+        self.pj_state = self.JM_PJ_PAYMENT_CREATED
+
+    def sanity_check_initial_payment(self):
+        """ These checks are motivated by the checks specified
+        for the *receiver* in the btcpayserver implementation doc.
+        We want to make sure our payment isn't rejected.
+        We also sanity check that the payment details match
+        the initialization of this Manager object.
+        """
+        # failure to extract tx should throw an error;
+        # this PSBT must be finalized and sane.
+        self.payment_tx = self.initial_psbt.extract_transaction()
+
+        # inputs must all have witness utxo populated
+        for inp in self.initial_psbt.inputs:
+            if not inp.utxo and isinstance(inp.utxo, btc.CTxOut):
+                return False
+
+        # check that there is no xpub or derivation info
+        if self.initial_psbt.xpubs:
+            return False
+        for inp in self.initial_psbt.inputs:
+            # derivation_map is an OrderedDict, if empty
+            # it will be counted as false:
+            if inp.derivation_map:
+                return False
+        for out in self.initial_psbt.outputs:
+            if out.derivation_map:
+                return False
+
+        # TODO we can replicate the mempool check here for
+        # Core versions sufficiently high, also encapsulate
+        # it in bc_interface.
+
+        # our logic requires no more than one change output
+        # for now:
+        found_payment = 0
+        assert len(self.payment_tx.vout) in [1, 2]
+        for out in self.payment_tx.vout:
+            if out.nValue == self.amount and \
+               btc.CCoinAddress.from_scriptPubKey(
+                   out.scriptPubKey) == self.destination:
+                found_payment += 1
+            else:
+                # store this for our balance check
+                # for receiver proposal
+                self.change_out = out
+        if not found_payment == 1:
+            return False
+
+        return True
+
+    def check_receiver_proposal(self, in_pbst, signed_psbt_for_fees):
+        """ This is the most security critical part of the
+        business logic of the payjoin. We must check in detail
+        that what the server proposes does not unfairly take money
+        from us, and also conforms to acceptable structure.
+        We perform the following checks of the receiver proposal:
+         1. Check that there are more inputs (i.e. some were contributed).
+         2. Does it contain our inputs, unchanged?
+         3. Does it contain our payment output, with amount increased?
+         4. Are the other inputs finalized, and of the correct type?
+         5. Is the feerate unchanged within tolerance?
+         6. Does it contain no xpub information or derivation information?
+         7. Are the sequence numbers unchanged (and all the same) for the inputs?
+         8. Is the nLockTime and version unchanged?
+         9. Is the extra fee we pay in reduced change output less than a doubling?
+
+        If all the above checks pass we will consider this valid, and cosign.
+        Returns:
+        (False, "reason for failure")
+        (True, None)
+        """
+        assert isinstance(in_pbst, btc.PartiallySignedTransaction)
+        orig_psbt = self.initial_psbt
+        assert isinstance(orig_psbt, btc.PartiallySignedTransaction)
+        # 1
+        if len(in_pbst.inputs) <= len(orig_psbt.inputs):
+            return (False, "Receiver did not contribute inputs to payjoin.")
+        # 2
+        ourins = [(i.prevout.hash, i.prevout.n) for i in orig_psbt.unsigned_tx.vin]
+        found = [0] * len(ourins)
+        receiver_input_indices = []
+        for i, inp in enumerate(in_pbst.unsigned_tx.vin):
+            for j, inp2 in enumerate(ourins):
+                if (inp.prevout.hash, inp.prevout.n) == inp2:
+                    found[j] += 1
+                else:
+                    receiver_input_indices.append(i)
+
+        if any([found[i] != 1 for i in range(len(found))]):
+            return (False, "Receiver proposed PSBT does not contain our inputs.")
+        # 3
+        found = 0
+        for out in in_pbst.unsigned_tx.vout:
+            if btc.CCoinAddress.from_scriptPubKey(out.scriptPubKey) == \
+               self.destination and out.nValue >= self.amount:
+                found += 1
+        if found != 1:
+            return (False, "Our payment output not found exactly once or "
+                    "with wrong amount.")
+        # 4
+        for ind in receiver_input_indices:
+            # check the input is finalized
+            if not self.wallet_service.is_input_finalized(in_pbst.inputs[ind]):
+                return (False, "receiver input is not finalized.")
+            # check the utxo field of the input and see if the
+            # scriptPubKey is of the right type.
+            spk = in_pbst.inputs[ind].utxo.scriptPubKey
+            if isinstance(self.wallet_service.wallet, SegwitLegacyWallet):
+                try:
+                    btc.P2SHCoinAddress.from_scriptPubKey(spk)
+                except btc.P2SHCoinAddressError:
+                    return (False,
+                            "Receiver input type does not match ours.")
+            elif isinstance(self.wallet_service.wallet, SegwitWallet):
+                try:
+                    btc.P2WPKHCoinAddress.from_scriptPubKey(spk)
+                except btc.P2WPKHCoinAddressError:
+                    return (False,
+                            "Receiver input type does not match ours.")
+            else:
+                assert False
+        # 5
+        # To get the feerate of the psbt proposed, we use the already-signed
+        # version (so all witnesses filled in) to calculate its size,
+        # then compare that with the fee, and do the same for the
+        # pre-existing non-payjoin.
+        gffp = PSBTWalletMixin.get_fee_from_psbt
+        proposed_tx_fee = gffp(signed_psbt_for_fees)
+        nonpayjoin_tx_fee = gffp(self.initial_psbt)
+        proposed_tx_size = signed_psbt_for_fees.extract_transaction(
+            ).get_virtual_size()
+        nonpayjoin_tx_size = self.initial_psbt.extract_transaction(
+            ).get_virtual_size()
+        proposed_fee_rate = proposed_tx_fee / float(proposed_tx_size)
+        log.debug("proposed fee rate: " + str(proposed_fee_rate))
+        nonpayjoin_fee_rate = nonpayjoin_tx_fee / float(nonpayjoin_tx_size)
+        log.debug("nonpayjoin fee rate: " + str(nonpayjoin_fee_rate))
+        diff_rate = abs(proposed_fee_rate - nonpayjoin_fee_rate)/nonpayjoin_fee_rate
+        if diff_rate > 0.2:
+            log.error("Bad fee rate differential: " + str(diff_rate))
+            return (False, "fee rate of payjoin tx is more than 20% different "
+                           "from inital fee rate, rejecting.")
+        # 6
+        if in_pbst.xpubs:
+            return (False, "Receiver proposal contains xpub information.")
+        # 7
+        # we created all inputs with one sequence number, make sure everything
+        # agrees
+        # TODO - discussion with btcpayserver devs, docs will be updated,
+        # server will agree with client in future. For now disabling check
+        # (it's a very complicated issue, surprisingly!)
+        #seqno = self.initial_psbt.unsigned_tx.vin[0].nSequence
+        #for inp in in_pbst.unsigned_tx.vin:
+        #    if inp.nSequence != seqno:
+        #        return (False, "all sequence numbers are not the same.")
+        # 8
+        if in_pbst.unsigned_tx.nLockTime != \
+           self.initial_psbt.unsigned_tx.nLockTime:
+            return (False, "receiver proposal has altered nLockTime.")
+        if in_pbst.unsigned_tx.nVersion != \
+           self.initial_psbt.unsigned_tx.nVersion:
+            return (False, "receiver proposal has altered nVersion.")
+        # 9
+        if proposed_tx_fee >= 2 * nonpayjoin_tx_fee:
+            return (False, "receiver's tx fee is too large (possibly "
+                    "too many extra inputs.")
+        # as well as the overall fee, check our pay-out specifically:
+        for out in in_pbst.unsigned_tx.vout:
+            if out.scriptPubKey == self.change_out.scriptPubKey:
+                found += 1
+                if self.change_out.nValue - out.nValue > nonpayjoin_tx_fee:
+                    return (False, "Our change output was reduced too much.")
+        return (True, None)
+
+    def set_payjoin_psbt(self, in_psbt, signed_psbt_for_fees):
+        """ This is the PSBT as initially proposed
+        by the receiver, so we keep a copy of it in that
+        state. This must be a copy as the sig_psbt function
+        will update the mutable psbt it is given.
+        This must not be called until the psbt has passed
+        all sanity and validation checks.
+        """
+        assert isinstance(in_psbt, btc.PartiallySignedTransaction)
+        assert isinstance(signed_psbt_for_fees, btc.PartiallySignedTransaction)
+        success, msg = self.check_receiver_proposal(in_psbt,
+                                                    signed_psbt_for_fees)
+        if not success:
+            return (success, msg)
+        self.payjoin_psbt = in_psbt
+        self.pj_state = self.JM_PJ_PARTIAL_RECEIVED
+        return (True, None)
+
+    def set_final_payjoin_psbt(self, in_psbt):
+        """ This is the PSBT after we have co-signed
+        it. If it is in a sane state, we update our state.
+        """
+        assert isinstance(in_psbt, btc.PartiallySignedTransaction)
+        # note that this is the simplest way to check
+        # for finality and validity of PSBT:
+        assert in_psbt.extract_transaction()
+        self.final_psbt = in_psbt
+        self.pj_state = self.JM_PJ_PAYJOIN_COSIGNED
+
+    def set_broadcast(self, success):
+        if success:
+            self.pj_state = self.JM_PJ_PAYJOIN_BROADCAST
+        else:
+            self.pj_state = self.JM_PJ_PAYJOIN_BROADCAST_FAILED
+
+    def report(self, jsonified=False, verbose=False):
+        """ Returns a dict (optionally jsonified) containing
+        the following information (if they are
+        available):
+        * current status of Payjoin
+        * payment transaction (original, non payjoin)
+        * payjoin partial (PSBT) sent by receiver
+        * final payjoin transaction
+        * whether or not the payjoin transaction is
+          broadcast and/or confirmed.
+        If verbose is True, we include the full deserialization
+        of transactions and PSBTs, which is too verbose for GUI
+        display.
+        """
+        reportdict = {"name:", "PAYJOIN STATUS REPORT"}
+        reportdict["status"] = self.pj_state # TODO: string
+        if self.payment_tx:
+            txdata = btc.hrt(self.payment_tx)
+            if verbose:
+                txdata = txdata["hex"]
+            reportdict["payment-tx"] = txdata
+        if self.payjoin_psbt:
+            psbtdata = PSBTWalletMixin.hr_psbt(
+                self.payjoin_psbt) if verbose else self.payjoin_psbt.to_base64()
+            reportdict["payjoin-proposed"] = psbtdata
+        if self.final_psbt:
+            finaldata = PSBTWalletMixin.hr_psbt(
+                self.final_psbt) if verbose else self.final_psbt.to_base64()
+            reportdict["payjoin-final"] = finaldata
+        if jsonified:
+            return json.dumps(reportdict, indent=4)
+        else:
+            return reportdict
+
+def parse_payjoin_setup(bip21_uri, wallet_service, mixdepth):
+    """ Takes the payment request data from the uri and returns a
+    JMPayjoinManager object initialised for that payment.
+    """
+    assert btc.is_bip21_uri(bip21_uri), "invalid bip21 uri: " + bip21_uri
+    decoded = btc.decode_bip21_uri(bip21_uri)
+
+    assert "amount" in decoded
+    assert "address" in decoded
+    assert "pj" in decoded
+
+    amount = decoded["amount"]
+    destaddr = decoded["address"]
+    # this will throw for any invalid address:
+    destaddr = btc.CCoinAddress(destaddr)
+    server = decoded["pj"]
+
+    return JMPayjoinManager(wallet_service, mixdepth, destaddr, amount, server)
+
+def send_payjoin(manager, accept_callback=None,
+                 info_callback=None, tls_whitelist=None):
+    """ Given a JMPayjoinManager object `manager`, initialised with the
+    payment request data from the server, use its wallet_service to construct
+    a payment transaction, with coins sourced from mixdepth `mixdepth`,
+    then wait for the server response, parse the PSBT, perform checks and complete sign.
+    The info and accept callbacks are to ask the user to confirm the creation of
+    the original payment transaction (None defaults to terminal/CLI processing),
+    and are as defined in `taker_utils.direct_send`.
+
+    If `tls_whitelist` is a list of bytestrings, they are treated as hostnames
+    for which tls certificate verification is ignored. Obviously this is ONLY for
+    testing.
+
+    Returns:
+    (True, None) in case of payment setup successful (response will be delivered
+     asynchronously) - the `manager` object can be inspected for more detail.
+    (False, errormsg) in case of failure.
+    """
+
+    # wallet should already be synced before calling here;
+    # we can create a standard payment, but have it returned as a PSBT.
+    assert isinstance(manager, JMPayjoinManager)
+    assert manager.wallet_service.synced
+    payment_psbt = direct_send(manager.wallet_service, manager.amount, manager.mixdepth,
+                             str(manager.destination), accept_callback=accept_callback,
+                             info_callback=info_callback,
+                             with_final_psbt=True)
+    if not payment_psbt:
+        return (False, "could not create non-payjoin payment")
+
+    manager.set_payment_tx_and_psbt(payment_psbt)
+    # TODO add delayed call to broadcast this after 1 minute
+
+    # Now we send the request to the server, with the encoded
+    # payment PSBT
+    if not tls_whitelist:
+        agent = Agent(reactor)
+    else:
+        agent = Agent(reactor,
+            contextFactory=WhitelistContextFactory(tls_whitelist))
+
+    body = BytesProducer(payment_psbt.to_base64().encode("utf-8"))
+    # TODO what to use as user agent?
+    d = agent.request(b"POST", manager.server.encode("utf-8"),
+        Headers({"Content-Type": ['text/plain']}),
+        bodyProducer=body)
+
+    d.addCallback(receive_payjoin_proposal_from_server, manager)
+    # note that the errback (here "noresponse") is *not* triggered
+    # by a server rejection (which is accompanied by a non-200
+    # status code returned), but by failure to communicate.
+    def noResponse(failure):
+        failure.trap(ResponseFailed)
+        log.error(failure.value.reasons[0].getTraceback())
+        reactor.stop()
+    d.addErrback(noResponse)
+    return (True, None)
+
+def fallback_nonpayjoin_broadcast(manager, err):
+    assert isinstance(manager, JMPayjoinManager)
+    log.warn("Payjoin did not succeed, falling back to non-payjoin payment.")
+    log.warn("Error message was: " + str(err))
+    original_tx = manager.initial_psbt.extract_transaction()
+    if not jm_single().bc_interface.pushtx(original_tx.serialize()):
+        log.error("Unable to broadcast original payment. The payment is NOT made.")
+    log.info("We paid without coinjoin. Transaction: ")
+    log.info(btc.hrt(original_tx))
+    reactor.stop()
+
+def receive_payjoin_proposal_from_server(response, manager):
+    assert isinstance(manager, JMPayjoinManager)
+
+    # if the response code is not 200 OK, we must assume payjoin
+    # attempt has failed, and revert to standard payment.
+    if int(response.code) != 200:
+        fallback_nonpayjoin_broadcast(manager, err=response.phrase)
+        return
+    # for debugging; will be removed in future:
+    log.debug("Response headers:")
+    log.debug(pformat(list(response.headers.getAllRawHeaders())))
+    # no attempt at chunking or handling incrementally is needed
+    # here. The body should be a byte string containing the
+    # new PSBT.
+    d = readBody(response)
+    d.addCallback(process_payjoin_proposal_from_server, manager)
+
+def process_payjoin_proposal_from_server(response_body, manager):
+    assert isinstance(manager, JMPayjoinManager)
+    try:
+        payjoin_proposal_psbt = \
+            btc.PartiallySignedTransaction.from_base64(response_body)
+    except Exception as e:
+        log.error("Payjoin tx from server could not be parsed: " + repr(e))
+        fallback_nonpayjoin_broadcast(manager, err="Server sent invalid psbt")
+        return
+
+    log.debug("Receiver sent us this PSBT: ")
+    log.debug(manager.wallet_service.hr_psbt(payjoin_proposal_psbt))
+    # we need to add back in our utxo information to the received psbt,
+    # since the servers remove it (not sure why?)
+    for i, inp in enumerate(payjoin_proposal_psbt.unsigned_tx.vin):
+        for j, inp2 in enumerate(manager.initial_psbt.unsigned_tx.vin):
+                    if (inp.prevout.hash, inp.prevout.n) == (
+                        inp2.prevout.hash, inp2.prevout.n):
+                        payjoin_proposal_psbt.inputs[i].utxo = \
+                            manager.initial_psbt.inputs[j].utxo
+    signresultandpsbt, err = manager.wallet_service.sign_psbt(
+        payjoin_proposal_psbt.serialize(), with_sign_result=True)
+    if err:
+        log.error("Failed to sign PSBT from the receiver, error: " + err)
+        fallback_nonpayjoin_broadcast(manager, err="Failed to sign receiver PSBT")
+        return
+
+    signresult, sender_signed_psbt = signresultandpsbt
+    assert signresult.is_final
+    success, msg = manager.set_payjoin_psbt(payjoin_proposal_psbt, sender_signed_psbt)
+    if not success:
+        log.error(msg)
+        fallback_nonpayjoin_broadcast(manager, err="Receiver PSBT checks failed.")
+        return
+    # All checks have passed. We can use the already signed transaction in
+    # sender_signed_psbt.
+    log.info("Our final signed PSBT is:\n{}".format(
+        manager.wallet_service.hr_psbt(sender_signed_psbt)))    
+    manager.set_final_payjoin_psbt(sender_signed_psbt)
+
+    # broadcast the tx
+    extracted_tx = sender_signed_psbt.extract_transaction()
+    log.info("Here is the final payjoin transaction:")
+    log.info(btc.hrt(extracted_tx))
+    if not jm_single().bc_interface.pushtx(extracted_tx.serialize()):
+        log.info("The above transaction failed to broadcast.")
+    else:
+        log.info("Payjoin transactoin broadcast successfully.")
+    reactor.stop()

jmclient/jmclient/taker_utils.py

@@ -22,10 +22,12 @@ Currently re-used by CLI script tumbler.py and joinmarket-qt
 
 def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False,
                 accept_callback=None, info_callback=None,
-                return_transaction=False, with_final_psbt=False):
+                return_transaction=False, with_final_psbt=False,
+                optin_rbf=False):
     """Send coins directly from one mixdepth to one destination address;
     does not need IRC. Sweep as for normal sendpayment (set amount=0).
     If answeryes is True, callback/command line query is not performed.
+    If optin_rbf is True, the nSequence values are changed as appropriate.
     If accept_callback is None, command line input for acceptance is assumed,
     else this callback is called:
     accept_callback:
@@ -142,6 +144,10 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False,
         log.info("Using a change value of: " + amount_to_str(changeval) + ".")
     tx = make_shuffled_tx(list(utxos.keys()), outs, 2, tx_locktime)
 
+    if optin_rbf:
+        for inp in tx.vin:
+            inp.nSequence = 0xffffffff - 2
+
     inscripts = {}
     spent_outs = []
     for i, txinp in enumerate(tx.vin):

jmclient/jmclient/wallet.py

@@ -1007,6 +1007,30 @@ class PSBTWalletMixin(object):
     def __init__(self, storage, **kwargs):
         super(PSBTWalletMixin, self).__init__(storage, **kwargs)
 
+    @staticmethod
+    def get_fee_from_psbt(in_psbt):
+        assert isinstance(in_psbt, btc.PartiallySignedTransaction)
+        spent = sum(in_psbt.get_input_amounts())
+        paid = sum((x.nValue for x in in_psbt.unsigned_tx.vout))
+        return spent - paid
+
+    def is_input_finalized(self, psbt_input):
+        """ This should be a convenience method in python-bitcointx.
+        However note: this is not a static method and tacitly
+        assumes that the input under examination is of the wallet's
+        type.
+        """
+        assert isinstance(psbt_input, btc.PSBT_Input)
+        if not psbt_input.utxo:
+            return False
+        if isinstance(self, (LegacyWallet, SegwitLegacyWallet)):
+            if not psbt_input.final_script_sig:
+                return False
+        if isinstance(self, (SegwitLegacyWallet, SegwitWallet)):
+            if not psbt_input.final_script_witness:
+                return False
+        return True
+
     @staticmethod
     def hr_psbt(in_psbt):
         """ Returns a jsonified indented string with all relevant
@@ -1196,8 +1220,7 @@ class PSBTWalletMixin(object):
             for k2, v2 in v.items():
                 privkeys.append(self._get_priv_from_path(v2[0]))
         jmckeys = list(btc.JMCKey(x[0][:-1]) for x in privkeys)
-        new_keystore = btc.KeyStore.from_iterable(jmckeys,
+        new_keystore = btc.KeyStore.from_iterable(jmckeys)
-                                                  require_path_templates=False)
 
         # for p2sh inputs that we want to sign, the redeem_script
         # field must be populated by us, as the counterparty did not

scripts/sendpayment.py

@@ -15,7 +15,8 @@ from jmclient import Taker, P2EPTaker, load_program_config, get_schedule,\
     JMClientProtocolFactory, start_reactor, validate_address, is_burn_destination, \
     jm_single, estimate_tx_fee, direct_send, WalletService,\
     open_test_wallet_maybe, get_wallet_path, NO_ROUNDING, \
-    get_sendpayment_parser, get_max_cj_fee_values, check_regtest
+    get_sendpayment_parser, get_max_cj_fee_values, check_regtest, \
+    parse_payjoin_setup, send_payjoin
 from twisted.python.log import startLogging
 from jmbase.support import get_log, set_logging_level, jmprint, \
     EXIT_FAILURE, EXIT_ARGERROR, DUST_THRESHOLD
@@ -51,8 +52,8 @@ def main():
     (options, args) = parser.parse_args()
     load_program_config(config_path=options.datadir)
     if options.p2ep and len(args) != 3:
-        parser.error("PayJoin requires exactly three arguments: "
+        parser.error("Joinmarket peer-to-peer PayJoin requires exactly three "
-                     "wallet, amount and destination address.")
+                     "arguments: wallet, amount and destination address.")
         sys.exit(EXIT_ARGERROR)
     elif options.schedule == '':
         if ((len(args) < 2) or
@@ -65,6 +66,7 @@ def main():
     #without schedule file option, use the arguments to create a schedule
     #of a single transaction
     sweeping = False
+    bip79 = False
     if options.schedule == '':
         if btc.is_bip21_uri(args[1]):
             parsed = btc.decode_bip21_uri(args[1])
@@ -75,7 +77,15 @@ def main():
                 sys.exit(EXIT_ARGERROR)
             destaddr = parsed['address']
             if 'jmnick' in parsed:
+                if "pj" in parsed:
+                    parser.error("Cannot specify both BIP79 and Joinmarket "
+                                 "peer-to-peer payjoin at the same time!")
+                    sys.exit(EXIT_ARGERROR)
                 options.p2ep = parsed['jmnick']
+            elif "pj" in parsed:
+                # note that this is a URL; its validity
+                # checking is deferred to twisted.web.client.Agent
+                bip79 = parsed["pj"]
         else:
             amount = btc.amount_to_sat(args[1])
             if amount == 0:
@@ -107,6 +117,9 @@ def main():
         schedule = [[options.mixdepth, amount, options.makercount,
                      destaddr, 0.0, NO_ROUNDING, 0]]
     else:
+        if btc.is_bip21_uri(args[1]):
+            parser.error("Schedule files are not compatible with bip21 uris.")
+            sys.exit(EXIT_ARGERROR)
         if options.p2ep:
             parser.error("Schedule files are not compatible with PayJoin")
             sys.exit(EXIT_FAILURE)
@@ -146,7 +159,8 @@ def main():
         fee_per_cp_guess))
 
     maxcjfee = (1, float('inf'))
-    if not options.p2ep and not options.pickorders and options.makercount != 0:
+    if not (options.p2ep or bip79) and not options.pickorders and \
+       options.makercount != 0:
         maxcjfee = get_max_cj_fee_values(jm_single().config, options)
         log.info("Using maximum coinjoin fee limits per maker of {:.4%}, {} "
                  "".format(maxcjfee[0], btc.amount_to_str(maxcjfee[1])))
@@ -189,7 +203,7 @@ def main():
             log.info("Estimated miner/tx fees for this coinjoin amount: {:.1%}"
                 .format(exp_tx_fees_ratio))
 
-    if options.makercount == 0 and not options.p2ep:
+    if options.makercount == 0 and not options.p2ep and not bip79:
         tx = direct_send(wallet_service, amount, mixdepth, destaddr,
                          options.answeryes, with_final_psbt=options.with_psbt)
         if options.with_psbt:
@@ -299,6 +313,14 @@ def main():
             reactor.stop()
         taker = P2EPTaker(options.p2ep, wallet_service, schedule,
                           callbacks=(None, None, p2ep_on_finished_callback))
+
+    elif bip79:
+        # TODO sanity check wallet type is segwit
+        manager = parse_payjoin_setup(args[1], wallet_service, options.mixdepth)
+        reactor.callWhenRunning(send_payjoin, manager)
+        reactor.run()
+        return
+
     else:
         taker = Taker(wallet_service,
                       schedule,

test/payjoinclient.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+import sys
+from twisted.internet import reactor
+from jmclient.cli_options import check_regtest
+from jmclient import (get_wallet_path, WalletService, open_test_wallet_maybe,
+                      jm_single, load_test_config,
+                      SegwitLegacyWallet, SegwitWallet)
+from jmclient.payjoin import send_payjoin, parse_payjoin_setup
+
+if __name__ == "__main__":
+    wallet_name = sys.argv[1]
+    mixdepth = int(sys.argv[2])
+    usessl = int(sys.argv[3])
+    bip21uri = None
+    if len(sys.argv) > 4:
+        bip21uri = sys.argv[4]
+    load_test_config()
+    jm_single().datadir = "."
+    check_regtest()
+    if not bip21uri:
+        if usessl == 0:
+            pjurl = "http://127.0.0.1:8080"
+        else:
+            pjurl = "https://127.0.0.1:8080"
+            bip21uri = "bitcoin:2N7CAdEUjJW9tUHiPhDkmL9ukPtcukJMoxK?amount=0.3&pj=" + pjurl
+    wallet_path = get_wallet_path(wallet_name, None)
+    if jm_single().config.get("POLICY", "native") == "true":
+        walletclass = SegwitWallet
+    else:
+        walletclass = SegwitLegacyWallet
+    wallet = open_test_wallet_maybe(
+        wallet_path, wallet_name, 4,
+        wallet_password_stdin=False,
+        test_wallet_cls=walletclass,
+        gap_limit=6)
+    wallet_service = WalletService(wallet)
+    # in this script, we need the wallet synced before
+    # logic processing for some paths, so do it now:
+    while not wallet_service.synced:
+        wallet_service.sync_wallet(fast=True)
+    # the sync call here will now be a no-op:
+    wallet_service.startService()
+    manager = parse_payjoin_setup(bip21uri, wallet_service, mixdepth)
+    if usessl == 0:
+        tlshostnames = None
+    else:
+        tlshostnames = [b"127.0.0.1"]
+    reactor.callWhenRunning(send_payjoin, manager, tls_whitelist=tlshostnames)
+    reactor.run()

test/payjoinserver.py

@@ -0,0 +1,178 @@
+#! /usr/bin/env python
+""" Creates a very simple server for payjoin
+    payment requests; uses regtest and a single
+    JM wallet, provides a hex seed for the sender
+    side of the test.
+    Use the same command line setup as for ygrunner.py,
+    except you needn't specify --nirc=
+    NOTE: to run this test you will need a `key.pem`
+    and a `cert.pem` in this (test/) directory,
+    created in the standard way for ssl certificates.
+    Note that (in test) the client will not verify
+    them.
+"""
+import os
+from twisted.web.server import Site
+from twisted.web.resource import Resource
+from twisted.internet import ssl
+from twisted.internet import reactor, endpoints
+from io import BytesIO
+from common import make_wallets
+import pytest
+from jmbase import jmprint
+import jmbitcoin as btc
+from jmclient import load_test_config, jm_single,\
+     SegwitWallet, SegwitLegacyWallet, cryptoengine
+
+# TODO change test for arbitrary payment requests
+payment_amt = 30000000
+
+dir_path = os.path.dirname(os.path.realpath(__file__))
+
+def get_ssl_context():
+    """Construct an SSL context factory from the user's privatekey/cert.
+    Here just hardcoded for tests.
+    Note this is off by default since the cert needs setting up.
+    """
+    return ssl.DefaultOpenSSLContextFactory(os.path.join(dir_path, "key.pem"),
+                                            os.path.join(dir_path, "cert.pem"))
+
+class PayjoinServer(Resource):
+    def __init__(self, wallet_service):
+        self.wallet_service = wallet_service
+        super().__init__()
+    isLeaf = True
+    def render_GET(self, request):
+        return "<html>Only for testing.</html>".encode("utf-8")
+    def render_POST(self, request):
+        """ The sender will use POST to send the initial
+        payment transaction.
+        """
+        jmprint("The server got this POST request: ")
+        print(request)
+        print(request.method)
+        print(request.uri)
+        print(request.args)
+        print(request.path)
+        print(request.content)
+        proposed_tx = request.content
+        assert isinstance(proposed_tx, BytesIO)
+        payment_psbt_base64 = proposed_tx.read()
+        payment_psbt = btc.PartiallySignedTransaction.from_base64(
+            payment_psbt_base64)
+        all_receiver_utxos = self.wallet_service.get_all_utxos()
+        # TODO is there a less verbose way to get any 2 utxos from the dict?
+        receiver_utxos_keys = list(all_receiver_utxos.keys())[:2]
+        receiver_utxos = {k: v for k, v in all_receiver_utxos.items(
+            ) if k in receiver_utxos_keys}
+    
+        # receiver will do other checks as discussed above, including payment
+        # amount; as discussed above, this is out of the scope of this PSBT test.
+    
+        # construct unsigned tx for payjoin-psbt:
+        payjoin_tx_inputs = [(x.prevout.hash[::-1],
+                    x.prevout.n) for x in payment_psbt.unsigned_tx.vin]
+        payjoin_tx_inputs.extend(receiver_utxos.keys())
+        # find payment output and change output
+        pay_out = None
+        change_out = None
+        for o in payment_psbt.unsigned_tx.vout:
+            jm_out_fmt = {"value": o.nValue,
+            "address": str(btc.CCoinAddress.from_scriptPubKey(
+            o.scriptPubKey))}
+            if o.nValue == payment_amt:
+                assert pay_out is None
+                pay_out = jm_out_fmt
+            else:
+                assert change_out is None
+                change_out = jm_out_fmt
+    
+        # we now know there were two outputs and know which is payment.
+        # bump payment output with our input:
+        outs = [pay_out, change_out]
+        our_inputs_val = sum([v["value"] for _, v in receiver_utxos.items()])
+        pay_out["value"] += our_inputs_val
+        print("we bumped the payment output value by: ", our_inputs_val)
+        print("It is now: ", pay_out["value"])
+        unsigned_payjoin_tx = btc.make_shuffled_tx(payjoin_tx_inputs, outs,
+                                    version=payment_psbt.unsigned_tx.nVersion,
+                                    locktime=payment_psbt.unsigned_tx.nLockTime)
+        print("we created this unsigned tx: ")
+        print(btc.hrt(unsigned_payjoin_tx))
+        # to create the PSBT we need the spent_outs for each input,
+        # in the right order:
+        spent_outs = []
+        for i, inp in enumerate(unsigned_payjoin_tx.vin):
+            input_found = False
+            for j, inp2 in enumerate(payment_psbt.unsigned_tx.vin):
+                if inp.prevout == inp2.prevout:
+                    spent_outs.append(payment_psbt.inputs[j].utxo)
+                    input_found = True
+                    break
+            if input_found:
+                continue
+            # if we got here this input is ours, we must find
+            # it from our original utxo choice list:
+            for ru in receiver_utxos.keys():
+                if (inp.prevout.hash[::-1], inp.prevout.n) == ru:
+                    spent_outs.append(
+                        self.wallet_service.witness_utxos_to_psbt_utxos(
+                            {ru: receiver_utxos[ru]})[0])
+                    input_found = True
+                    break
+            # there should be no other inputs:
+            assert input_found
+    
+        r_payjoin_psbt = self.wallet_service.create_psbt_from_tx(unsigned_payjoin_tx,
+                                                      spent_outs=spent_outs)
+        print("Receiver created payjoin PSBT:\n{}".format(
+            self.wallet_service.hr_psbt(r_payjoin_psbt)))
+    
+        signresultandpsbt, err = self.wallet_service.sign_psbt(r_payjoin_psbt.serialize(),
+                                                    with_sign_result=True)
+        assert not err, err
+        signresult, receiver_signed_psbt = signresultandpsbt
+        assert signresult.num_inputs_final == len(receiver_utxos)
+        assert not signresult.is_final
+    
+        print("Receiver signing successful. Payjoin PSBT is now:\n{}".format(
+            self.wallet_service.hr_psbt(receiver_signed_psbt)))
+        content = receiver_signed_psbt.to_base64()
+        request.setHeader(b"content-length", ("%d" % len(content)).encode("ascii"))
+        return content.encode("ascii")
+    
+    
+
+def test_start_payjoin_server(setup_payjoin_server):
+    # set up the wallet that the server owns, and the wallet for
+    # the sender too (print the seed):
+    if jm_single().config.get("POLICY", "native") == "true":
+        walletclass = SegwitWallet
+    else:
+        walletclass = SegwitLegacyWallet
+
+    wallet_services = make_wallets(2,
+                                   wallet_structures=[[1, 3, 0, 0, 0]] * 2,
+                                   mean_amt=2,
+                                   walletclass=walletclass)
+    #the server bot uses the first wallet, the sender the second
+    server_wallet_service = wallet_services[0]['wallet']
+    jmprint("\n\nTaker wallet seed : " + wallet_services[1]['seed'])
+    jmprint("\n")
+    server_wallet_service.sync_wallet(fast=True)
+    
+    site = Site(PayjoinServer(server_wallet_service))
+    # TODO for now, just sticking with TLS test as non-encrypted
+    # is unlikely to be used, but add that option.
+    reactor.listenSSL(8080, site, contextFactory=get_ssl_context())
+    #endpoint = endpoints.TCP4ServerEndpoint(reactor, 8080)
+    #endpoint.listen(site)
+    reactor.run()
+
+@pytest.fixture(scope="module")
+def setup_payjoin_server():
+    load_test_config()
+    jm_single().bc_interface.tick_forward_chain_interval = 10
+    jm_single().bc_interface.simulate_blocks()
+    # handles the custom regtest hrp for bech32
+    cryptoengine.BTC_P2WPKH.VBYTE = 100