From ca0de5c31200983ecf23ace12918971414ba36c1 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Mon, 4 May 2020 01:13:30 +0100 Subject: [PATCH] Add bip78 payjoin module and client-server test: See: https://github.com/bitcoin/bips/blob/master/bip-0078.mediawiki Adds a new module jmclient.payjoin which implements the full sender workflow from a BIP21 uri to a payjoin broadcast, state is managed in JMPayjoinManager, includes all checks as per documentation of btcpayserver (and later, BIP78). Added simple client and server implementations in test/payjoinclient.py and test/payjoinserver.py which allow a full end to end test on regtest. Add TLS support to payjoin tests: Note: the jmclient.payjoin module already supports TLS by default (Agent object), but here we add the ability to test without certificate verification. Both test/payjoinclient.py and test/payjoinserver.py now support TLS, but the server needs a key and certificate in its directory to run. Adds BIP78 payjoin option to sendpayment.py Users can use a bip21 uri with the "pj" field to send a payment to a remote server. Removes require_path_templates setting from KeyStore call in PSBTWalletMixin.sign_psbt --- jmbase/jmbase/__init__.py | 3 +- jmbase/jmbase/bytesprod.py | 22 ++ jmbitcoin/jmbitcoin/__init__.py | 2 + jmclient/jmclient/__init__.py | 2 +- jmclient/jmclient/payjoin.py | 503 +++++++++++++++++++++++++++++++ jmclient/jmclient/taker_utils.py | 8 +- jmclient/jmclient/wallet.py | 27 +- scripts/sendpayment.py | 32 +- test/payjoinclient.py | 49 +++ test/payjoinserver.py | 178 +++++++++++ 10 files changed, 816 insertions(+), 10 deletions(-) create mode 100644 jmbase/jmbase/bytesprod.py create mode 100644 jmclient/jmclient/payjoin.py create mode 100644 test/payjoinclient.py create mode 100644 test/payjoinserver.py diff --git a/jmbase/jmbase/__init__.py b/jmbase/jmbase/__init__.py index cdbbedb..e49ea0a 100644 --- a/jmbase/jmbase/__init__.py +++ b/jmbase/jmbase/__init__.py @@ -6,6 +6,7 @@ from .support import (get_log, chunks, debug_silence, jmprint, hextobin, lehextobin, utxostr_to_utxo, utxo_to_utxostr, EXIT_ARGERROR, EXIT_FAILURE, EXIT_SUCCESS, hexbin, dictchanger, listchanger, - cv, JM_WALLET_NAME_PREFIX, JM_APP_NAME) + JM_WALLET_NAME_PREFIX, JM_APP_NAME) +from .bytesprod import BytesProducer from .commands import * diff --git a/jmbase/jmbase/bytesprod.py b/jmbase/jmbase/bytesprod.py new file mode 100644 index 0000000..04e9fb7 --- /dev/null +++ b/jmbase/jmbase/bytesprod.py @@ -0,0 +1,22 @@ +""" See https://twistedmatrix.com/documents/current/web/howto/client.html +""" +from zope.interface import implementer + +from twisted.internet.defer import succeed +from twisted.web.iweb import IBodyProducer + +@implementer(IBodyProducer) +class BytesProducer(object): + def __init__(self, body): + self.body = body + self.length = len(body) + + def startProducing(self, consumer): + consumer.write(self.body) + return succeed(None) + + def pauseProducing(self): + pass + + def stopProducing(self): + pass \ No newline at end of file diff --git a/jmbitcoin/jmbitcoin/__init__.py b/jmbitcoin/jmbitcoin/__init__.py index ae2e0e2..6ab579f 100644 --- a/jmbitcoin/jmbitcoin/__init__.py +++ b/jmbitcoin/jmbitcoin/__init__.py @@ -11,6 +11,8 @@ from bitcointx.core import (x, b2x, b2lx, lx, COutPoint, CTxOut, CTxIn, CMutableTransaction, Hash160, coins_to_satoshi, satoshi_to_coins) from bitcointx.core.key import KeyStore +from bitcointx.wallet import (P2SHCoinAddress, P2SHCoinAddressError, + P2WPKHCoinAddress, P2WPKHCoinAddressError) from bitcointx.core.script import (CScript, OP_0, SignatureHash, SIGHASH_ALL, SIGVERSION_WITNESS_V0, CScriptWitness) from bitcointx.core.psbt import (PartiallySignedTransaction, PSBT_Input, diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index f088b94..d222478 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -56,7 +56,7 @@ from .wallet_service import WalletService from .maker import Maker, P2EPMaker from .yieldgenerator import YieldGenerator, YieldGeneratorBasic, ygmain from .snicker_receiver import SNICKERError, SNICKERReceiver - +from .payjoin import parse_payjoin_setup, send_payjoin # Set default logging handler to avoid "No handler found" warnings. try: diff --git a/jmclient/jmclient/payjoin.py b/jmclient/jmclient/payjoin.py new file mode 100644 index 0000000..9c116dd --- /dev/null +++ b/jmclient/jmclient/payjoin.py @@ -0,0 +1,503 @@ +from zope.interface import implementer +from twisted.internet import reactor +from twisted.web.client import (Agent, readBody, ResponseFailed, + BrowserLikePolicyForHTTPS) +from twisted.web.iweb import IPolicyForHTTPS +from twisted.internet.ssl import CertificateOptions +from twisted.web.http_headers import Headers + +import json +from pprint import pformat +from jmbase import BytesProducer +from .configure import get_log, jm_single +import jmbitcoin as btc +from .wallet import PSBTWalletMixin, SegwitLegacyWallet, SegwitWallet +from .wallet_service import WalletService +from .taker_utils import direct_send + +""" +For some documentation see: + https://github.com/btcpayserver/btcpayserver-doc/blob/master/Payjoin-spec.md + which is a delta to: + https://github.com/bitcoin/bips/blob/master/bip-0079.mediawiki +""" +log = get_log() + +""" This whitelister allows us to accept any cert for a specific + domain, and is to be used for testing only; the default Agent + behaviour of twisted.web.client.Agent for https URIs is + the correct one in production (i.e. uses local trust store). +""" +@implementer(IPolicyForHTTPS) +class WhitelistContextFactory(object): + def __init__(self, good_domains=None): + """ + :param good_domains: List of domains. The URLs must be in bytes + """ + if not good_domains: + self.good_domains = [] + else: + self.good_domains = good_domains + # by default, handle requests like a browser would + self.default_policy = BrowserLikePolicyForHTTPS() + + def creatorForNetloc(self, hostname, port): + # check if the hostname is in the the whitelist, + # otherwise return the default policy + if hostname in self.good_domains: + return CertificateOptions(verify=False) + return self.default_policy.creatorForNetloc(hostname, port) + +class JMPayjoinManager(object): + """ An encapsulation of state for an + ongoing Payjoin payment. Allows reporting + details of the outcome of a Payjoin attempt. + """ + + # enum such that progress can be + # reported + JM_PJ_NONE = 0 + JM_PJ_INIT = 1 + JM_PJ_PAYMENT_CREATED = 2 + JM_PJ_PAYMENT_SENT = 3 + JM_PJ_PARTIAL_RECEIVED = 4 + JM_PJ_PARTIAL_REJECTED = 5 + JM_PJ_PAYJOIN_COSIGNED = 6 + JM_PJ_PAYJOIN_BROADCAST = 7 + JM_PJ_PAYJOIN_BROADCAST_FAILED = 8 + + pj_state = JM_PJ_NONE + + def __init__(self, wallet_service, mixdepth, destination, + amount, server): + assert isinstance(wallet_service, WalletService) + # payjoin is not supported for non-segwit wallets: + assert isinstance(wallet_service.wallet, + (SegwitWallet, SegwitLegacyWallet)) + # our payjoin implementation requires PSBT + assert isinstance(wallet_service.wallet, PSBTWalletMixin) + self.wallet_service = wallet_service + # mixdepth from which payment is sourced + assert isinstance(mixdepth, int) + self.mixdepth = mixdepth + assert isinstance(destination, btc.CCoinAddress) + self.destination = destination + assert isinstance(amount, int) + assert amount > 0 + self.amount = amount + self.server = server + self.pj_state = self.JM_PJ_INIT + self.payment_tx = None + self.initial_psbt = None + self.payjoin_psbt = None + self.final_psbt = None + # change is initialized as None + # in case there is no change: + self.change_out = None + + def set_payment_tx_and_psbt(self, in_psbt): + assert isinstance(in_psbt, btc.PartiallySignedTransaction) + self.initial_psbt = in_psbt + # any failure here is a coding error, as it is fully + # under our control. + assert self.sanity_check_initial_payment() + self.pj_state = self.JM_PJ_PAYMENT_CREATED + + def sanity_check_initial_payment(self): + """ These checks are motivated by the checks specified + for the *receiver* in the btcpayserver implementation doc. + We want to make sure our payment isn't rejected. + We also sanity check that the payment details match + the initialization of this Manager object. + """ + # failure to extract tx should throw an error; + # this PSBT must be finalized and sane. + self.payment_tx = self.initial_psbt.extract_transaction() + + # inputs must all have witness utxo populated + for inp in self.initial_psbt.inputs: + if not inp.utxo and isinstance(inp.utxo, btc.CTxOut): + return False + + # check that there is no xpub or derivation info + if self.initial_psbt.xpubs: + return False + for inp in self.initial_psbt.inputs: + # derivation_map is an OrderedDict, if empty + # it will be counted as false: + if inp.derivation_map: + return False + for out in self.initial_psbt.outputs: + if out.derivation_map: + return False + + # TODO we can replicate the mempool check here for + # Core versions sufficiently high, also encapsulate + # it in bc_interface. + + # our logic requires no more than one change output + # for now: + found_payment = 0 + assert len(self.payment_tx.vout) in [1, 2] + for out in self.payment_tx.vout: + if out.nValue == self.amount and \ + btc.CCoinAddress.from_scriptPubKey( + out.scriptPubKey) == self.destination: + found_payment += 1 + else: + # store this for our balance check + # for receiver proposal + self.change_out = out + if not found_payment == 1: + return False + + return True + + def check_receiver_proposal(self, in_pbst, signed_psbt_for_fees): + """ This is the most security critical part of the + business logic of the payjoin. We must check in detail + that what the server proposes does not unfairly take money + from us, and also conforms to acceptable structure. + We perform the following checks of the receiver proposal: + 1. Check that there are more inputs (i.e. some were contributed). + 2. Does it contain our inputs, unchanged? + 3. Does it contain our payment output, with amount increased? + 4. Are the other inputs finalized, and of the correct type? + 5. Is the feerate unchanged within tolerance? + 6. Does it contain no xpub information or derivation information? + 7. Are the sequence numbers unchanged (and all the same) for the inputs? + 8. Is the nLockTime and version unchanged? + 9. Is the extra fee we pay in reduced change output less than a doubling? + + If all the above checks pass we will consider this valid, and cosign. + Returns: + (False, "reason for failure") + (True, None) + """ + assert isinstance(in_pbst, btc.PartiallySignedTransaction) + orig_psbt = self.initial_psbt + assert isinstance(orig_psbt, btc.PartiallySignedTransaction) + # 1 + if len(in_pbst.inputs) <= len(orig_psbt.inputs): + return (False, "Receiver did not contribute inputs to payjoin.") + # 2 + ourins = [(i.prevout.hash, i.prevout.n) for i in orig_psbt.unsigned_tx.vin] + found = [0] * len(ourins) + receiver_input_indices = [] + for i, inp in enumerate(in_pbst.unsigned_tx.vin): + for j, inp2 in enumerate(ourins): + if (inp.prevout.hash, inp.prevout.n) == inp2: + found[j] += 1 + else: + receiver_input_indices.append(i) + + if any([found[i] != 1 for i in range(len(found))]): + return (False, "Receiver proposed PSBT does not contain our inputs.") + # 3 + found = 0 + for out in in_pbst.unsigned_tx.vout: + if btc.CCoinAddress.from_scriptPubKey(out.scriptPubKey) == \ + self.destination and out.nValue >= self.amount: + found += 1 + if found != 1: + return (False, "Our payment output not found exactly once or " + "with wrong amount.") + # 4 + for ind in receiver_input_indices: + # check the input is finalized + if not self.wallet_service.is_input_finalized(in_pbst.inputs[ind]): + return (False, "receiver input is not finalized.") + # check the utxo field of the input and see if the + # scriptPubKey is of the right type. + spk = in_pbst.inputs[ind].utxo.scriptPubKey + if isinstance(self.wallet_service.wallet, SegwitLegacyWallet): + try: + btc.P2SHCoinAddress.from_scriptPubKey(spk) + except btc.P2SHCoinAddressError: + return (False, + "Receiver input type does not match ours.") + elif isinstance(self.wallet_service.wallet, SegwitWallet): + try: + btc.P2WPKHCoinAddress.from_scriptPubKey(spk) + except btc.P2WPKHCoinAddressError: + return (False, + "Receiver input type does not match ours.") + else: + assert False + # 5 + # To get the feerate of the psbt proposed, we use the already-signed + # version (so all witnesses filled in) to calculate its size, + # then compare that with the fee, and do the same for the + # pre-existing non-payjoin. + gffp = PSBTWalletMixin.get_fee_from_psbt + proposed_tx_fee = gffp(signed_psbt_for_fees) + nonpayjoin_tx_fee = gffp(self.initial_psbt) + proposed_tx_size = signed_psbt_for_fees.extract_transaction( + ).get_virtual_size() + nonpayjoin_tx_size = self.initial_psbt.extract_transaction( + ).get_virtual_size() + proposed_fee_rate = proposed_tx_fee / float(proposed_tx_size) + log.debug("proposed fee rate: " + str(proposed_fee_rate)) + nonpayjoin_fee_rate = nonpayjoin_tx_fee / float(nonpayjoin_tx_size) + log.debug("nonpayjoin fee rate: " + str(nonpayjoin_fee_rate)) + diff_rate = abs(proposed_fee_rate - nonpayjoin_fee_rate)/nonpayjoin_fee_rate + if diff_rate > 0.2: + log.error("Bad fee rate differential: " + str(diff_rate)) + return (False, "fee rate of payjoin tx is more than 20% different " + "from inital fee rate, rejecting.") + # 6 + if in_pbst.xpubs: + return (False, "Receiver proposal contains xpub information.") + # 7 + # we created all inputs with one sequence number, make sure everything + # agrees + # TODO - discussion with btcpayserver devs, docs will be updated, + # server will agree with client in future. For now disabling check + # (it's a very complicated issue, surprisingly!) + #seqno = self.initial_psbt.unsigned_tx.vin[0].nSequence + #for inp in in_pbst.unsigned_tx.vin: + # if inp.nSequence != seqno: + # return (False, "all sequence numbers are not the same.") + # 8 + if in_pbst.unsigned_tx.nLockTime != \ + self.initial_psbt.unsigned_tx.nLockTime: + return (False, "receiver proposal has altered nLockTime.") + if in_pbst.unsigned_tx.nVersion != \ + self.initial_psbt.unsigned_tx.nVersion: + return (False, "receiver proposal has altered nVersion.") + # 9 + if proposed_tx_fee >= 2 * nonpayjoin_tx_fee: + return (False, "receiver's tx fee is too large (possibly " + "too many extra inputs.") + # as well as the overall fee, check our pay-out specifically: + for out in in_pbst.unsigned_tx.vout: + if out.scriptPubKey == self.change_out.scriptPubKey: + found += 1 + if self.change_out.nValue - out.nValue > nonpayjoin_tx_fee: + return (False, "Our change output was reduced too much.") + return (True, None) + + def set_payjoin_psbt(self, in_psbt, signed_psbt_for_fees): + """ This is the PSBT as initially proposed + by the receiver, so we keep a copy of it in that + state. This must be a copy as the sig_psbt function + will update the mutable psbt it is given. + This must not be called until the psbt has passed + all sanity and validation checks. + """ + assert isinstance(in_psbt, btc.PartiallySignedTransaction) + assert isinstance(signed_psbt_for_fees, btc.PartiallySignedTransaction) + success, msg = self.check_receiver_proposal(in_psbt, + signed_psbt_for_fees) + if not success: + return (success, msg) + self.payjoin_psbt = in_psbt + self.pj_state = self.JM_PJ_PARTIAL_RECEIVED + return (True, None) + + def set_final_payjoin_psbt(self, in_psbt): + """ This is the PSBT after we have co-signed + it. If it is in a sane state, we update our state. + """ + assert isinstance(in_psbt, btc.PartiallySignedTransaction) + # note that this is the simplest way to check + # for finality and validity of PSBT: + assert in_psbt.extract_transaction() + self.final_psbt = in_psbt + self.pj_state = self.JM_PJ_PAYJOIN_COSIGNED + + def set_broadcast(self, success): + if success: + self.pj_state = self.JM_PJ_PAYJOIN_BROADCAST + else: + self.pj_state = self.JM_PJ_PAYJOIN_BROADCAST_FAILED + + def report(self, jsonified=False, verbose=False): + """ Returns a dict (optionally jsonified) containing + the following information (if they are + available): + * current status of Payjoin + * payment transaction (original, non payjoin) + * payjoin partial (PSBT) sent by receiver + * final payjoin transaction + * whether or not the payjoin transaction is + broadcast and/or confirmed. + If verbose is True, we include the full deserialization + of transactions and PSBTs, which is too verbose for GUI + display. + """ + reportdict = {"name:", "PAYJOIN STATUS REPORT"} + reportdict["status"] = self.pj_state # TODO: string + if self.payment_tx: + txdata = btc.hrt(self.payment_tx) + if verbose: + txdata = txdata["hex"] + reportdict["payment-tx"] = txdata + if self.payjoin_psbt: + psbtdata = PSBTWalletMixin.hr_psbt( + self.payjoin_psbt) if verbose else self.payjoin_psbt.to_base64() + reportdict["payjoin-proposed"] = psbtdata + if self.final_psbt: + finaldata = PSBTWalletMixin.hr_psbt( + self.final_psbt) if verbose else self.final_psbt.to_base64() + reportdict["payjoin-final"] = finaldata + if jsonified: + return json.dumps(reportdict, indent=4) + else: + return reportdict + +def parse_payjoin_setup(bip21_uri, wallet_service, mixdepth): + """ Takes the payment request data from the uri and returns a + JMPayjoinManager object initialised for that payment. + """ + assert btc.is_bip21_uri(bip21_uri), "invalid bip21 uri: " + bip21_uri + decoded = btc.decode_bip21_uri(bip21_uri) + + assert "amount" in decoded + assert "address" in decoded + assert "pj" in decoded + + amount = decoded["amount"] + destaddr = decoded["address"] + # this will throw for any invalid address: + destaddr = btc.CCoinAddress(destaddr) + server = decoded["pj"] + + return JMPayjoinManager(wallet_service, mixdepth, destaddr, amount, server) + +def send_payjoin(manager, accept_callback=None, + info_callback=None, tls_whitelist=None): + """ Given a JMPayjoinManager object `manager`, initialised with the + payment request data from the server, use its wallet_service to construct + a payment transaction, with coins sourced from mixdepth `mixdepth`, + then wait for the server response, parse the PSBT, perform checks and complete sign. + The info and accept callbacks are to ask the user to confirm the creation of + the original payment transaction (None defaults to terminal/CLI processing), + and are as defined in `taker_utils.direct_send`. + + If `tls_whitelist` is a list of bytestrings, they are treated as hostnames + for which tls certificate verification is ignored. Obviously this is ONLY for + testing. + + Returns: + (True, None) in case of payment setup successful (response will be delivered + asynchronously) - the `manager` object can be inspected for more detail. + (False, errormsg) in case of failure. + """ + + # wallet should already be synced before calling here; + # we can create a standard payment, but have it returned as a PSBT. + assert isinstance(manager, JMPayjoinManager) + assert manager.wallet_service.synced + payment_psbt = direct_send(manager.wallet_service, manager.amount, manager.mixdepth, + str(manager.destination), accept_callback=accept_callback, + info_callback=info_callback, + with_final_psbt=True) + if not payment_psbt: + return (False, "could not create non-payjoin payment") + + manager.set_payment_tx_and_psbt(payment_psbt) + # TODO add delayed call to broadcast this after 1 minute + + # Now we send the request to the server, with the encoded + # payment PSBT + if not tls_whitelist: + agent = Agent(reactor) + else: + agent = Agent(reactor, + contextFactory=WhitelistContextFactory(tls_whitelist)) + + body = BytesProducer(payment_psbt.to_base64().encode("utf-8")) + # TODO what to use as user agent? + d = agent.request(b"POST", manager.server.encode("utf-8"), + Headers({"Content-Type": ['text/plain']}), + bodyProducer=body) + + d.addCallback(receive_payjoin_proposal_from_server, manager) + # note that the errback (here "noresponse") is *not* triggered + # by a server rejection (which is accompanied by a non-200 + # status code returned), but by failure to communicate. + def noResponse(failure): + failure.trap(ResponseFailed) + log.error(failure.value.reasons[0].getTraceback()) + reactor.stop() + d.addErrback(noResponse) + return (True, None) + +def fallback_nonpayjoin_broadcast(manager, err): + assert isinstance(manager, JMPayjoinManager) + log.warn("Payjoin did not succeed, falling back to non-payjoin payment.") + log.warn("Error message was: " + str(err)) + original_tx = manager.initial_psbt.extract_transaction() + if not jm_single().bc_interface.pushtx(original_tx.serialize()): + log.error("Unable to broadcast original payment. The payment is NOT made.") + log.info("We paid without coinjoin. Transaction: ") + log.info(btc.hrt(original_tx)) + reactor.stop() + +def receive_payjoin_proposal_from_server(response, manager): + assert isinstance(manager, JMPayjoinManager) + + # if the response code is not 200 OK, we must assume payjoin + # attempt has failed, and revert to standard payment. + if int(response.code) != 200: + fallback_nonpayjoin_broadcast(manager, err=response.phrase) + return + # for debugging; will be removed in future: + log.debug("Response headers:") + log.debug(pformat(list(response.headers.getAllRawHeaders()))) + # no attempt at chunking or handling incrementally is needed + # here. The body should be a byte string containing the + # new PSBT. + d = readBody(response) + d.addCallback(process_payjoin_proposal_from_server, manager) + +def process_payjoin_proposal_from_server(response_body, manager): + assert isinstance(manager, JMPayjoinManager) + try: + payjoin_proposal_psbt = \ + btc.PartiallySignedTransaction.from_base64(response_body) + except Exception as e: + log.error("Payjoin tx from server could not be parsed: " + repr(e)) + fallback_nonpayjoin_broadcast(manager, err="Server sent invalid psbt") + return + + log.debug("Receiver sent us this PSBT: ") + log.debug(manager.wallet_service.hr_psbt(payjoin_proposal_psbt)) + # we need to add back in our utxo information to the received psbt, + # since the servers remove it (not sure why?) + for i, inp in enumerate(payjoin_proposal_psbt.unsigned_tx.vin): + for j, inp2 in enumerate(manager.initial_psbt.unsigned_tx.vin): + if (inp.prevout.hash, inp.prevout.n) == ( + inp2.prevout.hash, inp2.prevout.n): + payjoin_proposal_psbt.inputs[i].utxo = \ + manager.initial_psbt.inputs[j].utxo + signresultandpsbt, err = manager.wallet_service.sign_psbt( + payjoin_proposal_psbt.serialize(), with_sign_result=True) + if err: + log.error("Failed to sign PSBT from the receiver, error: " + err) + fallback_nonpayjoin_broadcast(manager, err="Failed to sign receiver PSBT") + return + + signresult, sender_signed_psbt = signresultandpsbt + assert signresult.is_final + success, msg = manager.set_payjoin_psbt(payjoin_proposal_psbt, sender_signed_psbt) + if not success: + log.error(msg) + fallback_nonpayjoin_broadcast(manager, err="Receiver PSBT checks failed.") + return + # All checks have passed. We can use the already signed transaction in + # sender_signed_psbt. + log.info("Our final signed PSBT is:\n{}".format( + manager.wallet_service.hr_psbt(sender_signed_psbt))) + manager.set_final_payjoin_psbt(sender_signed_psbt) + + # broadcast the tx + extracted_tx = sender_signed_psbt.extract_transaction() + log.info("Here is the final payjoin transaction:") + log.info(btc.hrt(extracted_tx)) + if not jm_single().bc_interface.pushtx(extracted_tx.serialize()): + log.info("The above transaction failed to broadcast.") + else: + log.info("Payjoin transactoin broadcast successfully.") + reactor.stop() diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index b2bed0e..d8e8a2d 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -22,10 +22,12 @@ Currently re-used by CLI script tumbler.py and joinmarket-qt def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, accept_callback=None, info_callback=None, - return_transaction=False, with_final_psbt=False): + return_transaction=False, with_final_psbt=False, + optin_rbf=False): """Send coins directly from one mixdepth to one destination address; does not need IRC. Sweep as for normal sendpayment (set amount=0). If answeryes is True, callback/command line query is not performed. + If optin_rbf is True, the nSequence values are changed as appropriate. If accept_callback is None, command line input for acceptance is assumed, else this callback is called: accept_callback: @@ -142,6 +144,10 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, log.info("Using a change value of: " + amount_to_str(changeval) + ".") tx = make_shuffled_tx(list(utxos.keys()), outs, 2, tx_locktime) + if optin_rbf: + for inp in tx.vin: + inp.nSequence = 0xffffffff - 2 + inscripts = {} spent_outs = [] for i, txinp in enumerate(tx.vin): diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index fb46a99..5185113 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -1007,6 +1007,30 @@ class PSBTWalletMixin(object): def __init__(self, storage, **kwargs): super(PSBTWalletMixin, self).__init__(storage, **kwargs) + @staticmethod + def get_fee_from_psbt(in_psbt): + assert isinstance(in_psbt, btc.PartiallySignedTransaction) + spent = sum(in_psbt.get_input_amounts()) + paid = sum((x.nValue for x in in_psbt.unsigned_tx.vout)) + return spent - paid + + def is_input_finalized(self, psbt_input): + """ This should be a convenience method in python-bitcointx. + However note: this is not a static method and tacitly + assumes that the input under examination is of the wallet's + type. + """ + assert isinstance(psbt_input, btc.PSBT_Input) + if not psbt_input.utxo: + return False + if isinstance(self, (LegacyWallet, SegwitLegacyWallet)): + if not psbt_input.final_script_sig: + return False + if isinstance(self, (SegwitLegacyWallet, SegwitWallet)): + if not psbt_input.final_script_witness: + return False + return True + @staticmethod def hr_psbt(in_psbt): """ Returns a jsonified indented string with all relevant @@ -1196,8 +1220,7 @@ class PSBTWalletMixin(object): for k2, v2 in v.items(): privkeys.append(self._get_priv_from_path(v2[0])) jmckeys = list(btc.JMCKey(x[0][:-1]) for x in privkeys) - new_keystore = btc.KeyStore.from_iterable(jmckeys, - require_path_templates=False) + new_keystore = btc.KeyStore.from_iterable(jmckeys) # for p2sh inputs that we want to sign, the redeem_script # field must be populated by us, as the counterparty did not diff --git a/scripts/sendpayment.py b/scripts/sendpayment.py index f71bef9..af401a5 100755 --- a/scripts/sendpayment.py +++ b/scripts/sendpayment.py @@ -15,7 +15,8 @@ from jmclient import Taker, P2EPTaker, load_program_config, get_schedule,\ JMClientProtocolFactory, start_reactor, validate_address, is_burn_destination, \ jm_single, estimate_tx_fee, direct_send, WalletService,\ open_test_wallet_maybe, get_wallet_path, NO_ROUNDING, \ - get_sendpayment_parser, get_max_cj_fee_values, check_regtest + get_sendpayment_parser, get_max_cj_fee_values, check_regtest, \ + parse_payjoin_setup, send_payjoin from twisted.python.log import startLogging from jmbase.support import get_log, set_logging_level, jmprint, \ EXIT_FAILURE, EXIT_ARGERROR, DUST_THRESHOLD @@ -51,8 +52,8 @@ def main(): (options, args) = parser.parse_args() load_program_config(config_path=options.datadir) if options.p2ep and len(args) != 3: - parser.error("PayJoin requires exactly three arguments: " - "wallet, amount and destination address.") + parser.error("Joinmarket peer-to-peer PayJoin requires exactly three " + "arguments: wallet, amount and destination address.") sys.exit(EXIT_ARGERROR) elif options.schedule == '': if ((len(args) < 2) or @@ -65,6 +66,7 @@ def main(): #without schedule file option, use the arguments to create a schedule #of a single transaction sweeping = False + bip79 = False if options.schedule == '': if btc.is_bip21_uri(args[1]): parsed = btc.decode_bip21_uri(args[1]) @@ -75,7 +77,15 @@ def main(): sys.exit(EXIT_ARGERROR) destaddr = parsed['address'] if 'jmnick' in parsed: + if "pj" in parsed: + parser.error("Cannot specify both BIP79 and Joinmarket " + "peer-to-peer payjoin at the same time!") + sys.exit(EXIT_ARGERROR) options.p2ep = parsed['jmnick'] + elif "pj" in parsed: + # note that this is a URL; its validity + # checking is deferred to twisted.web.client.Agent + bip79 = parsed["pj"] else: amount = btc.amount_to_sat(args[1]) if amount == 0: @@ -107,6 +117,9 @@ def main(): schedule = [[options.mixdepth, amount, options.makercount, destaddr, 0.0, NO_ROUNDING, 0]] else: + if btc.is_bip21_uri(args[1]): + parser.error("Schedule files are not compatible with bip21 uris.") + sys.exit(EXIT_ARGERROR) if options.p2ep: parser.error("Schedule files are not compatible with PayJoin") sys.exit(EXIT_FAILURE) @@ -146,7 +159,8 @@ def main(): fee_per_cp_guess)) maxcjfee = (1, float('inf')) - if not options.p2ep and not options.pickorders and options.makercount != 0: + if not (options.p2ep or bip79) and not options.pickorders and \ + options.makercount != 0: maxcjfee = get_max_cj_fee_values(jm_single().config, options) log.info("Using maximum coinjoin fee limits per maker of {:.4%}, {} " "".format(maxcjfee[0], btc.amount_to_str(maxcjfee[1]))) @@ -189,7 +203,7 @@ def main(): log.info("Estimated miner/tx fees for this coinjoin amount: {:.1%}" .format(exp_tx_fees_ratio)) - if options.makercount == 0 and not options.p2ep: + if options.makercount == 0 and not options.p2ep and not bip79: tx = direct_send(wallet_service, amount, mixdepth, destaddr, options.answeryes, with_final_psbt=options.with_psbt) if options.with_psbt: @@ -299,6 +313,14 @@ def main(): reactor.stop() taker = P2EPTaker(options.p2ep, wallet_service, schedule, callbacks=(None, None, p2ep_on_finished_callback)) + + elif bip79: + # TODO sanity check wallet type is segwit + manager = parse_payjoin_setup(args[1], wallet_service, options.mixdepth) + reactor.callWhenRunning(send_payjoin, manager) + reactor.run() + return + else: taker = Taker(wallet_service, schedule, diff --git a/test/payjoinclient.py b/test/payjoinclient.py new file mode 100644 index 0000000..764b8b6 --- /dev/null +++ b/test/payjoinclient.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +import sys +from twisted.internet import reactor +from jmclient.cli_options import check_regtest +from jmclient import (get_wallet_path, WalletService, open_test_wallet_maybe, + jm_single, load_test_config, + SegwitLegacyWallet, SegwitWallet) +from jmclient.payjoin import send_payjoin, parse_payjoin_setup + +if __name__ == "__main__": + wallet_name = sys.argv[1] + mixdepth = int(sys.argv[2]) + usessl = int(sys.argv[3]) + bip21uri = None + if len(sys.argv) > 4: + bip21uri = sys.argv[4] + load_test_config() + jm_single().datadir = "." + check_regtest() + if not bip21uri: + if usessl == 0: + pjurl = "http://127.0.0.1:8080" + else: + pjurl = "https://127.0.0.1:8080" + bip21uri = "bitcoin:2N7CAdEUjJW9tUHiPhDkmL9ukPtcukJMoxK?amount=0.3&pj=" + pjurl + wallet_path = get_wallet_path(wallet_name, None) + if jm_single().config.get("POLICY", "native") == "true": + walletclass = SegwitWallet + else: + walletclass = SegwitLegacyWallet + wallet = open_test_wallet_maybe( + wallet_path, wallet_name, 4, + wallet_password_stdin=False, + test_wallet_cls=walletclass, + gap_limit=6) + wallet_service = WalletService(wallet) + # in this script, we need the wallet synced before + # logic processing for some paths, so do it now: + while not wallet_service.synced: + wallet_service.sync_wallet(fast=True) + # the sync call here will now be a no-op: + wallet_service.startService() + manager = parse_payjoin_setup(bip21uri, wallet_service, mixdepth) + if usessl == 0: + tlshostnames = None + else: + tlshostnames = [b"127.0.0.1"] + reactor.callWhenRunning(send_payjoin, manager, tls_whitelist=tlshostnames) + reactor.run() diff --git a/test/payjoinserver.py b/test/payjoinserver.py new file mode 100644 index 0000000..cda5e5c --- /dev/null +++ b/test/payjoinserver.py @@ -0,0 +1,178 @@ +#! /usr/bin/env python +""" Creates a very simple server for payjoin + payment requests; uses regtest and a single + JM wallet, provides a hex seed for the sender + side of the test. + Use the same command line setup as for ygrunner.py, + except you needn't specify --nirc= + NOTE: to run this test you will need a `key.pem` + and a `cert.pem` in this (test/) directory, + created in the standard way for ssl certificates. + Note that (in test) the client will not verify + them. +""" +import os +from twisted.web.server import Site +from twisted.web.resource import Resource +from twisted.internet import ssl +from twisted.internet import reactor, endpoints +from io import BytesIO +from common import make_wallets +import pytest +from jmbase import jmprint +import jmbitcoin as btc +from jmclient import load_test_config, jm_single,\ + SegwitWallet, SegwitLegacyWallet, cryptoengine + +# TODO change test for arbitrary payment requests +payment_amt = 30000000 + +dir_path = os.path.dirname(os.path.realpath(__file__)) + +def get_ssl_context(): + """Construct an SSL context factory from the user's privatekey/cert. + Here just hardcoded for tests. + Note this is off by default since the cert needs setting up. + """ + return ssl.DefaultOpenSSLContextFactory(os.path.join(dir_path, "key.pem"), + os.path.join(dir_path, "cert.pem")) + +class PayjoinServer(Resource): + def __init__(self, wallet_service): + self.wallet_service = wallet_service + super().__init__() + isLeaf = True + def render_GET(self, request): + return "Only for testing.".encode("utf-8") + def render_POST(self, request): + """ The sender will use POST to send the initial + payment transaction. + """ + jmprint("The server got this POST request: ") + print(request) + print(request.method) + print(request.uri) + print(request.args) + print(request.path) + print(request.content) + proposed_tx = request.content + assert isinstance(proposed_tx, BytesIO) + payment_psbt_base64 = proposed_tx.read() + payment_psbt = btc.PartiallySignedTransaction.from_base64( + payment_psbt_base64) + all_receiver_utxos = self.wallet_service.get_all_utxos() + # TODO is there a less verbose way to get any 2 utxos from the dict? + receiver_utxos_keys = list(all_receiver_utxos.keys())[:2] + receiver_utxos = {k: v for k, v in all_receiver_utxos.items( + ) if k in receiver_utxos_keys} + + # receiver will do other checks as discussed above, including payment + # amount; as discussed above, this is out of the scope of this PSBT test. + + # construct unsigned tx for payjoin-psbt: + payjoin_tx_inputs = [(x.prevout.hash[::-1], + x.prevout.n) for x in payment_psbt.unsigned_tx.vin] + payjoin_tx_inputs.extend(receiver_utxos.keys()) + # find payment output and change output + pay_out = None + change_out = None + for o in payment_psbt.unsigned_tx.vout: + jm_out_fmt = {"value": o.nValue, + "address": str(btc.CCoinAddress.from_scriptPubKey( + o.scriptPubKey))} + if o.nValue == payment_amt: + assert pay_out is None + pay_out = jm_out_fmt + else: + assert change_out is None + change_out = jm_out_fmt + + # we now know there were two outputs and know which is payment. + # bump payment output with our input: + outs = [pay_out, change_out] + our_inputs_val = sum([v["value"] for _, v in receiver_utxos.items()]) + pay_out["value"] += our_inputs_val + print("we bumped the payment output value by: ", our_inputs_val) + print("It is now: ", pay_out["value"]) + unsigned_payjoin_tx = btc.make_shuffled_tx(payjoin_tx_inputs, outs, + version=payment_psbt.unsigned_tx.nVersion, + locktime=payment_psbt.unsigned_tx.nLockTime) + print("we created this unsigned tx: ") + print(btc.hrt(unsigned_payjoin_tx)) + # to create the PSBT we need the spent_outs for each input, + # in the right order: + spent_outs = [] + for i, inp in enumerate(unsigned_payjoin_tx.vin): + input_found = False + for j, inp2 in enumerate(payment_psbt.unsigned_tx.vin): + if inp.prevout == inp2.prevout: + spent_outs.append(payment_psbt.inputs[j].utxo) + input_found = True + break + if input_found: + continue + # if we got here this input is ours, we must find + # it from our original utxo choice list: + for ru in receiver_utxos.keys(): + if (inp.prevout.hash[::-1], inp.prevout.n) == ru: + spent_outs.append( + self.wallet_service.witness_utxos_to_psbt_utxos( + {ru: receiver_utxos[ru]})[0]) + input_found = True + break + # there should be no other inputs: + assert input_found + + r_payjoin_psbt = self.wallet_service.create_psbt_from_tx(unsigned_payjoin_tx, + spent_outs=spent_outs) + print("Receiver created payjoin PSBT:\n{}".format( + self.wallet_service.hr_psbt(r_payjoin_psbt))) + + signresultandpsbt, err = self.wallet_service.sign_psbt(r_payjoin_psbt.serialize(), + with_sign_result=True) + assert not err, err + signresult, receiver_signed_psbt = signresultandpsbt + assert signresult.num_inputs_final == len(receiver_utxos) + assert not signresult.is_final + + print("Receiver signing successful. Payjoin PSBT is now:\n{}".format( + self.wallet_service.hr_psbt(receiver_signed_psbt))) + content = receiver_signed_psbt.to_base64() + request.setHeader(b"content-length", ("%d" % len(content)).encode("ascii")) + return content.encode("ascii") + + + +def test_start_payjoin_server(setup_payjoin_server): + # set up the wallet that the server owns, and the wallet for + # the sender too (print the seed): + if jm_single().config.get("POLICY", "native") == "true": + walletclass = SegwitWallet + else: + walletclass = SegwitLegacyWallet + + wallet_services = make_wallets(2, + wallet_structures=[[1, 3, 0, 0, 0]] * 2, + mean_amt=2, + walletclass=walletclass) + #the server bot uses the first wallet, the sender the second + server_wallet_service = wallet_services[0]['wallet'] + jmprint("\n\nTaker wallet seed : " + wallet_services[1]['seed']) + jmprint("\n") + server_wallet_service.sync_wallet(fast=True) + + site = Site(PayjoinServer(server_wallet_service)) + # TODO for now, just sticking with TLS test as non-encrypted + # is unlikely to be used, but add that option. + reactor.listenSSL(8080, site, contextFactory=get_ssl_context()) + #endpoint = endpoints.TCP4ServerEndpoint(reactor, 8080) + #endpoint.listen(site) + reactor.run() + +@pytest.fixture(scope="module") +def setup_payjoin_server(): + load_test_config() + jm_single().bc_interface.tick_forward_chain_interval = 10 + jm_single().bc_interface.simulate_blocks() + # handles the custom regtest hrp for bech32 + cryptoengine.BTC_P2WPKH.VBYTE = 100