diff --git a/jmbase/jmbase/__init__.py b/jmbase/jmbase/__init__.py index 0a7d926..8bf80a9 100644 --- a/jmbase/jmbase/__init__.py +++ b/jmbase/jmbase/__init__.py @@ -7,9 +7,12 @@ from .support import (get_log, chunks, debug_silence, jmprint, utxo_to_utxostr, EXIT_ARGERROR, EXIT_FAILURE, EXIT_SUCCESS, hexbin, dictchanger, listchanger, JM_WALLET_NAME_PREFIX, JM_APP_NAME, - IndentedHelpFormatterWithNL, wrapped_urlparse) + IndentedHelpFormatterWithNL, wrapped_urlparse, + bdict_sdict_convert) from .proof_of_work import get_pow, verify_pow -from .twisted_utils import stop_reactor, is_hs_uri, get_tor_agent, get_nontor_agent +from .twisted_utils import (stop_reactor, is_hs_uri, get_tor_agent, + get_nontor_agent, JMHiddenService, + JMHTTPResource) from .bytesprod import BytesProducer from .commands import * diff --git a/jmbase/jmbase/commands.py b/jmbase/jmbase/commands.py index 6c4a9f8..b7138e3 100644 --- a/jmbase/jmbase/commands.py +++ b/jmbase/jmbase/commands.py @@ -284,6 +284,10 @@ class SNICKERReceivePowTarget(JMCommand): """ Payjoin-related commands """ + +""" Sender-specific commands. +""" + class BIP78SenderInit(JMCommand): """ Initialization data for a BIP78 service. See documentation of `netconfig` in @@ -309,10 +313,67 @@ class BIP78SenderReceiveProposal(JMCommand): """ arguments = [(b'psbt', BigUnicode())] -class BIP78ReceiverError(JMCommand): +class BIP78SenderReceiveError(JMCommand): """ Sends a message from daemon to client indicating that the BIP78 receiver did not accept the request, or there was a network error. """ arguments = [(b'errormsg', Unicode()), (b'errorcode', Integer())] + +""" Receiver-specific commands +""" + +class BIP78ReceiverInit(JMCommand): + """ Initialization data for a BIP78 hidden service. + """ + arguments = [(b'netconfig', Unicode())] + +class BIP78ReceiverUp(JMCommand): + """ Returns onion hostname to client when + the hidden service has been brought up, indicating + readiness. + """ + arguments = [(b'hostname', Unicode())] + +class BIP78ReceiverOriginalPSBT(JMCommand): + """ Sends the sender's original + payment PSBT, base64 encoded, and the request + parameters in the url, from the daemon to the client. + """ + arguments = [(b'body', BigUnicode()), + (b'params', Unicode())] + +class BIP78ReceiverSendProposal(JMCommand): + """ Receives a payjoin proposal PSBT from + the client, sent to the daemon. + """ + arguments = [(b'psbt', BigUnicode())] + +class BIP78ReceiverSendError(JMCommand): + """ Sends a message from client to daemon + indicating that the BIP78 receiver did not + accept the request, to be forwarded to the sender. + """ + arguments = [(b'errormsg', Unicode()), + (b'errorcode', Unicode())] + +class BIP78ReceiverHiddenServiceShutdown(JMCommand): + """ Sends a message from the daemon to the + client when the hidden service has shut down. + """ + arguments = [] + +class BIP78ReceiverOnionSetupFailed(JMCommand): + """ Sends a message from the daemon to the + client when the hidden service setup failed + for the given reason. + """ + arguments = [(b'reason', Unicode())] + +class BIP78InfoMsg(JMCommand): + """ Sends an info message to the client + from the daemon about current status at + network level. + """ + arguments = [(b'infomsg', Unicode())] \ No newline at end of file diff --git a/jmbase/jmbase/support.py b/jmbase/jmbase/support.py index 3660337..90ad7b4 100644 --- a/jmbase/jmbase/support.py +++ b/jmbase/jmbase/support.py @@ -306,3 +306,21 @@ def wrapped_urlparse(url): if url.endswith(a) and not url.startswith(b): url = b + url return urlparse.urlparse(url) + +def bdict_sdict_convert(d, output_binary=False): + """ Useful for converting dicts from url parameter sets + to a form that can be handled by json dumps/loads. + This code only works if *all* keys in the dict + are binary strings, and all values are lists of same. + This code could be extended if needed. + If output_binary is True, the reverse operation is performed. + """ + newd = {} + for k, v in d.items(): + if output_binary: + newv = [a.encode("utf-8") for a in v] + newd[k.encode("utf-8")] = newv + else: + newv = [a.decode("utf-8") for a in v] + newd[k.decode("utf-8")] = newv + return newd diff --git a/jmbase/jmbase/twisted_utils.py b/jmbase/jmbase/twisted_utils.py index 67e974c..8424d24 100644 --- a/jmbase/jmbase/twisted_utils.py +++ b/jmbase/jmbase/twisted_utils.py @@ -4,7 +4,10 @@ from twisted.internet.error import ReactorNotRunning from twisted.internet import reactor from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.web.client import Agent, BrowserLikePolicyForHTTPS +import txtorcon from txtorcon.web import tor_agent +from twisted.web.server import Site +from twisted.web.resource import Resource from twisted.web.iweb import IPolicyForHTTPS from twisted.internet.ssl import CertificateOptions from .support import wrapped_urlparse @@ -73,4 +76,95 @@ def get_nontor_agent(tls_whitelist=[]): else: agent = Agent(reactor, contextFactory=WhitelistContextFactory(tls_whitelist)) - return agent \ No newline at end of file + return agent + +class JMHiddenService(object): + """ Wrapper class around the actions needed to + create and serve on a hidden service; an object of + type Resource must be provided in the constructor, + which does the HTTP serving actions (GET, POST serving). + """ + def __init__(self, resource, info_callback, error_callback, + onion_hostname_callback, tor_control_host, + tor_control_port, serving_port = None, + shutdown_callback = None): + self.site = Site(resource) + self.site.displayTracebacks = False + self.info_callback = info_callback + self.error_callback = error_callback + # this has a separate callback for convenience, it should + # be passed the literal *.onion string (port is already + # known and is 80 by default) + self.onion_hostname_callback = onion_hostname_callback + self.shutdown_callback = shutdown_callback + if not serving_port: + self.port = 80 + else: + self.port = serving_port + self.tor_control_host = tor_control_host + self.tor_control_port = tor_control_port + print("got these settings: ", self.port, self.site, self.tor_control_host, self.tor_control_port) + + def start_tor(self): + """ This function executes the workflow + of starting the hidden service and returning its hostname + """ + self.info_callback("Attempting to start onion service on port: {} " + "...".format(self.port)) + if str(self.tor_control_host).startswith('unix:'): + control_endpoint = UNIXClientEndpoint(reactor, + self.tor_control_host[5:]) + else: + control_endpoint = TCP4ClientEndpoint(reactor, + self.tor_control_host,self.tor_control_port) + d = txtorcon.connect(reactor, control_endpoint) + d.addCallback(self.create_onion_ep) + d.addErrback(self.setup_failed) + # TODO: add errbacks to the next two calls in + # the chain: + d.addCallback(self.onion_listen) + d.addCallback(self.print_host) + + def setup_failed(self, arg): + # Note that actions based on this failure are deferred to callers: + self.error_callback("Setup failed: " + str(arg)) + + def create_onion_ep(self, t): + self.tor_connection = t + return t.create_onion_endpoint(self.port) + + def onion_listen(self, onion_ep): + return onion_ep.listen(self.site) + + def print_host(self, ep): + """ Callback fired once the HS is available; + we let the caller know the hidden service onion hostname, + which is not otherwise available to them: + """ + # Note that ep,getHost().onion_port must return the same + # port as we chose in self.port; if not there is an error. + assert ep.getHost().onion_port == self.port + self.onion_hostname_callback(ep.getHost().onion_uri) + + def shutdown(self): + self.tor_connection.protocol.transport.loseConnection() + self.info_callback("Hidden service shutdown complete") + if self.shutdown_callback: + self.shutdown_callback() + +class JMHTTPResource(Resource): + """ Object acting as HTTP serving resource + """ + def __init__(self, info_callback, shutdown_callback): + self.info_callback = info_callback + self.shutdown_callback = shutdown_callback + super().__init__() + + isLeaf = True + + def render_GET(self, request): + """ by default we serve a simple string which can be used e.g. + to check if an ephemeral HS is upon Tor Browser; child classes + may override. + """ + return "Only for testing.".encode("utf-8") diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index 27ff309..5d8ddac 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -57,7 +57,7 @@ from .wallet_utils import ( from .wallet_service import WalletService from .maker import Maker from .yieldgenerator import YieldGenerator, YieldGeneratorBasic, ygmain -from .payjoin import (parse_payjoin_setup, send_payjoin, PayjoinServer, +from .payjoin import (parse_payjoin_setup, send_payjoin, JMBIP78ReceiverManager) # Set default logging handler to avoid "No handler found" warnings. diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index 80c08ff..dabdd3f 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -14,7 +14,7 @@ import hashlib import os import sys from jmbase import (get_log, EXIT_FAILURE, hextobin, bintohex, - utxo_to_utxostr) + utxo_to_utxostr, bdict_sdict_convert) from jmclient import (jm_single, get_irc_mchannels, RegtestBitcoinCoreInterface, SNICKERReceiver, process_shutdown) @@ -48,11 +48,17 @@ class BIP78ClientProtocol(BaseClientProtocol): def __init__(self, manager, params, success_callback, failure_callback, - tls_whitelist=[]): + tls_whitelist=[], mode="sender"): self.manager = manager + # can be "sender" or "receiver" + self.mode = mode self.success_callback = success_callback self.failure_callback = failure_callback - self.params = params + if self.mode == "sender": + self.params = params + else: + # receiver only learns params from request + self.params = None if len(tls_whitelist) == 0: if isinstance(jm_single().bc_interface, RegtestBitcoinCoreInterface): @@ -60,13 +66,56 @@ class BIP78ClientProtocol(BaseClientProtocol): self.tls_whitelist = tls_whitelist def connectionMade(self): - netconfig = {"socks5_host": jm_single().config.get("PAYJOIN", "onion_socks5_host"), - "socks5_port": jm_single().config.get("PAYJOIN", "onion_socks5_port"), - "tls_whitelist": ",".join(self.tls_whitelist), - "servers": [self.manager.server]} - d = self.callRemote(commands.BIP78SenderInit, - netconfig=json.dumps(netconfig)) + jcg = jm_single().config.get + if self.mode == "sender": + netconfig = {"socks5_host": jcg("PAYJOIN", "onion_socks5_host"), + "socks5_port": jcg("PAYJOIN", "onion_socks5_port"), + "tls_whitelist": ",".join(self.tls_whitelist), + "servers": [self.manager.server]} + d = self.callRemote(commands.BIP78SenderInit, + netconfig=json.dumps(netconfig)) + else: + netconfig = {"port": 80, + "tor_control_host": jcg("PAYJOIN", "tor_control_host"), + "tor_control_port": jcg("PAYJOIN", "tor_control_port")} + d = self.callRemote(commands.BIP78ReceiverInit, + netconfig=json.dumps(netconfig)) + self.defaultCallbacks(d) + + @commands.BIP78ReceiverUp.responder + def on_BIP78_RECEIVER_UP(self, hostname): + self.manager.bip21_uri_from_onion_hostname(hostname) + return {"accepted": True} + + @commands.BIP78ReceiverOriginalPSBT.responder + def on_BIP78_RECEIVER_ORIGINAL_PSBT(self, body, params): + params = json.loads(params) + # TODO: we don't need binary key/vals client side, but will have to edit + # PayjoinConverter for that: + retval = self.success_callback(body.encode("utf-8"), bdict_sdict_convert( + params, output_binary=True)) + if not retval[0]: + d = self.callRemote(commands.BIP78ReceiverSendError, errormsg=retval[1], + errorcode=retval[2]) + else: + d = self.callRemote(commands.BIP78ReceiverSendProposal, psbt=retval[1]) self.defaultCallbacks(d) + return {"accepted": True} + + @commands.BIP78ReceiverHiddenServiceShutdown.responder + def on_BIP78_RECEIVER_HIDDEN_SERVICE_SHUTDOWN(self): + """ This is called when the daemon has shut down the HS + because of an invalid message/error. An earlier message + will have conveyed the reason for the error. + """ + self.manager.shutdown() + return {"accepted": True} + + @commands.BIP78ReceiverOnionSetupFailed.responder + def on_BIP78_RECEIVER_ONION_SETUP_FAILED(self, reason): + self.manager.info_callback(reason) + self.manager.shutdown() + return {"accepted": True} @commands.BIP78SenderUp.responder def on_BIP78_SENDER_UP(self): @@ -81,11 +130,16 @@ class BIP78ClientProtocol(BaseClientProtocol): self.success_callback(psbt, self.manager) return {"accepted": True} - @commands.BIP78ReceiverError.responder - def on_BIP78_RECEIVER_ERROR(self, errormsg, errorcode): + @commands.BIP78SenderReceiveError.responder + def on_BIP78_SENDER_RECEIVER_ERROR(self, errormsg, errorcode): self.failure_callback(errormsg, errorcode, self.manager) return {"accepted": True} + @commands.BIP78InfoMsg.responder + def on_BIP78_INFO_MSG(self, infomsg): + self.manager.info_callback(infomsg) + return {"accepted": True} + class SNICKERClientProtocol(BaseClientProtocol): def __init__(self, client, servers, tls_whitelist=[], oneshot=False): @@ -682,13 +736,18 @@ class BIP78ClientProtocolFactory(protocol.ClientFactory): def buildProtocol(self, addr): return self.protocol(self.manager, self.params, self.success_callback, - self.failure_callback) + self.failure_callback, + tls_whitelist=self.tls_whitelist, + mode=self.mode) def __init__(self, manager, params, success_callback, - failure_callback): + failure_callback, tls_whitelist=[], + mode="sender"): self.manager = manager self.params = params self.success_callback = success_callback self.failure_callback = failure_callback + self.tls_whitelist = tls_whitelist + self.mode = mode class JMClientProtocolFactory(protocol.ClientFactory): protocol = JMTakerClientProtocol @@ -766,6 +825,13 @@ def start_reactor(host, port, factory=None, snickerfactory=None, if bip78: bip78port = start_daemon_on_port(port_a, bip78factory, "BIP78", 2000) + # if the port had to be incremented due to conflict above, we should update + # it in the config var so e.g. bip78 connections choose the port we actually + # used. + # This is specific to the daemon-in-same-process case; for the external daemon + # the user must just set the right value. + jm_single().config.set("DAEMON", "daemon_port", str(port_a[0])) + # Note the reactor.connect*** entries do not include BIP78 which # starts in jmclient.payjoin: if usessl: diff --git a/jmclient/jmclient/payjoin.py b/jmclient/jmclient/payjoin.py index 9d6e520..59c8324 100644 --- a/jmclient/jmclient/payjoin.py +++ b/jmclient/jmclient/payjoin.py @@ -1,11 +1,10 @@ -from twisted.internet import reactor, task -from twisted.web.server import Site -from twisted.web.resource import Resource -from twisted.internet.endpoints import TCP4ClientEndpoint, UNIXClientEndpoint -import txtorcon +from twisted.internet import reactor +try: + from twisted.internet.ssl import ClientContextFactory +except ImportError: + pass import json import random -from io import BytesIO from jmbase import bintohex, jmprint from .configure import get_log, jm_single import jmbitcoin as btc @@ -393,7 +392,6 @@ class JMPayjoinManager(object): self.user_info_callback("Choosing one coin at random") try: - print('attempting to select from mixdepth: ', str(self.mixdepth)) my_utxos = self.wallet_service.select_utxos( self.mixdepth, jm_single().DUST_THRESHOLD, select_fn=select_one_utxo) @@ -558,10 +556,12 @@ def send_payjoin(manager, accept_callback=None, params = make_payjoin_request_params(manager) factory = BIP78ClientProtocolFactory(manager, params, process_payjoin_proposal_from_server, process_error_from_server) - # TODO add SSL option as for other protocol instances: - reactor.connectTCP(jm_single().config.get("DAEMON", "daemon_host"), - jm_single().config.getint("DAEMON", "daemon_port")-2000, - factory) + h = jm_single().config.get("DAEMON", "daemon_host") + p = jm_single().config.getint("DAEMON", "daemon_port")-2000 + if jm_single().config.get("DAEMON", "use_ssl") != 'false': + reactor.connectSSL(h, p, factory, ClientContextFactory()) + else: + reactor.connectTCP(h, p, factory) return (True, None) def fallback_nonpayjoin_broadcast(err, manager): @@ -662,99 +662,49 @@ def process_payjoin_proposal_from_server(response_body, manager): """ Receiver-specific code """ -class PayjoinServer(Resource): - def __init__(self, wallet_service, mixdepth, destination, amount, - shutdown_callback, info_callback , mode="command-line", +class PayjoinConverter(object): + """ This class is used to encapsulate the objects and operations + needed to convert a given payment psbt from a sender, to a payjoin psbt + proposal. + """ + def __init__(self, manager, shutdown_callback, info_callback, pj_version = 1): + assert isinstance(manager, JMPayjoinManager) + self.manager = manager self.pj_version = pj_version - self.wallet_service = wallet_service + self.wallet_service = manager.wallet_service # a callback with no arguments and no return value, # to take whatever actions are needed when the payment has # been received: self.shutdown_callback = shutdown_callback self.info_callback = info_callback - self.manager = JMPayjoinManager(self.wallet_service, mixdepth, - destination, amount, mode=mode, - user_info_callback=self.info_callback) super().__init__() - isLeaf = True - - def bip78_error(self, request, error_meaning, - error_code="unavailable", http_code=400): - """ - See https://github.com/bitcoin/bips/blob/master/bip-0078.mediawiki#receivers-well-known-errors - - We return, to the sender, stringified json in the body as per the above. - In case the error code is "original-psbt-rejected", we do not have - any valid payment to broadcast, so we shut down with "not paid". - for other cases, we schedule the fallback for 60s from now. - """ - - request.setResponseCode(http_code) - request.setHeader(b"content-type", b"text/html; charset=utf-8") - log.debug("Returning an error: " + str( - error_code) + ": " + str(error_meaning)) - if error_code in ["original-psbt-rejected", "version-unsupported"]: - # if there is a negotiation failure in the first step, we cannot - # know whether the sender client sent a valid non-payjoin or not, - # hence the warning below is somewhat ambiguous: - log.warn("Negotiation failure. Payment has not yet been made," - " check wallet.") - # shutdown now but wait until response is sent. - task.deferLater(reactor, 2.0, self.end_failure) - else: - reactor.callLater(60.0, fallback_nonpayjoin_broadcast, - error_meaning.encode("utf-8"), self.manager) - return json.dumps({"errorCode": error_code, - "message": error_meaning}).encode("utf-8") - - def render_GET(self, request): - # can be used e.g. to check if an ephemeral HS is up - # on Tor Browser: - return "Only for testing.".encode("utf-8") - - def render_POST(self, request): - """ The sender will use POST to send the initial - payment transaction. + def request_to_psbt(self, payment_psbt_base64, sender_parameters): + """ Takes a payment psbt from a sender and their url parameters, + and returns a new payment PSBT proposal, assuming all conditions + are met. + Returns: + (False, errormsg, errortype) in case of failure. + or: + (True, base64_payjoin_psbt) in case of success. """ - log.debug("The server got this POST request: ") - # unfortunately the twisted Request object is not - # easily serialized: - log.debug(request) - log.debug(request.method) - log.debug(request.uri) - log.debug(request.args) - sender_parameters = request.args - log.debug(request.path) - # defer logging of raw request content: - proposed_tx = request.content - # we only support version 1; reject others: if not self.pj_version == int(sender_parameters[b'v'][0]): - return self.bip78_error(request, - "This version of payjoin is not supported. ", + return (False, "This version of payjoin is not supported. ", "version-unsupported") - if not isinstance(proposed_tx, BytesIO): - return self.bip78_error(request, "invalid psbt format", - "original-psbt-rejected") - payment_psbt_base64 = proposed_tx.read() - log.debug("request content: " + bintohex(payment_psbt_base64)) try: payment_psbt = btc.PartiallySignedTransaction.from_base64( payment_psbt_base64) except: - return self.bip78_error(request, - "invalid psbt format", - "original-psbt-rejected") + return (False, "invalid psbt format", "original-psbt-rejected") try: self.manager.set_payment_tx_and_psbt(payment_psbt) except Exception: # note that Assert errors, Value errors and CheckTransaction errors # are all possible, so we catch all exceptions to avoid a crash. - return self.bip78_error(request, - "Proposed initial PSBT does not pass sanity checks.", + return (False, "Proposed initial PSBT does not pass sanity checks.", "original-psbt-rejected") # if the sender set the additionalfeeoutputindex and maxadditionalfeecontribution @@ -773,7 +723,7 @@ class PayjoinServer(Resource): else: minfeerate = None except Exception as e: - return self.bip78_error(request, "Invalid request parameters.", + return (False, "Invalid request parameters.", "original-psbt-rejected") # if sender chose a fee output it must be the change output, @@ -783,11 +733,11 @@ class PayjoinServer(Resource): # reduction being not too much, which is checked against minfeerate; if # it is too big a reduction, again we fail payjoin. if (afoi is not None and mafc is None) or (mafc is not None and afoi is None): - return self.bip78_error(request, "Invalid request parameters.", + return (False, "Invalid request parameters.", "original-psbt-rejected") if afoi and not (self.manager.change_out_index == afoi): - return self.bip78_error(request, "additionalfeeoutputindex is " + return (False, "additionalfeeoutputindex is " "not the change output. Joinmarket does " "not currently support this.", "original-psbt-rejected") @@ -799,7 +749,7 @@ class PayjoinServer(Resource): res = jm_single().bc_interface.testmempoolaccept(bintohex( self.manager.payment_tx.serialize())) if not res[0]["allowed"]: - return self.bip78_error(request, "Proposed transaction was " + return (False, "Proposed transaction was " "rejected from mempool.", "original-psbt-rejected") @@ -811,9 +761,8 @@ class PayjoinServer(Resource): receiver_utxos = self.manager.select_receiver_utxos() if not receiver_utxos: - return self.bip78_error(request, - "Could not select coins for payjoin", - "unavailable") + return (False, "Could not select coins for payjoin", + "unavailable") # construct unsigned tx for payjoin-psbt: payjoin_tx_inputs = [(x.prevout.hash[::-1], @@ -861,7 +810,7 @@ class PayjoinServer(Resource): # First, let's check that the user's requested minfeerate is not higher # than the feerate they already chose: if minfeerate and minfeerate > self.manager.get_payment_psbt_feerate(): - return self.bip78_error(request, "Bad request: minfeerate " + return (False, "Bad request: minfeerate " "bigger than original psbt feerate.", "original-psbt-rejected") # set the intended virtual size of our input: @@ -889,7 +838,7 @@ class PayjoinServer(Resource): expected_new_fee_rate = self.manager.initial_psbt.get_fee()/( expected_new_tx_size + vsize) if expected_new_fee_rate < minfeerate: - return self.bip78_error(request, "Bad request: we cannot " + return (False, "Bad request: we cannot " "achieve minfeerate requested.", "original-psbt-rejected") @@ -965,15 +914,14 @@ class PayjoinServer(Resource): log.debug("Receiver signing successful. Payjoin PSBT is now:\n{}".format( self.wallet_service.human_readable_psbt(receiver_signed_psbt))) # construct txoutset for the wallet service callback; we cannot use - # txid as we don't have all signatures. + # txid as we don't have all signatures (TODO: ? but segwit only? even so, + # works anyway). txinfo = tuple(( x.scriptPubKey, x.nValue) for x in receiver_signed_psbt.unsigned_tx.vout) self.wallet_service.register_callbacks([self.end_receipt], txinfo =txinfo, cb_type="unconfirmed") - content = receiver_signed_psbt.to_base64() - request.setHeader(b"content-length", ("%d" % len(content)).encode("ascii")) - return content.encode("ascii") + return (True, receiver_signed_psbt.to_base64(), None) def end_receipt(self, txd, txid): if self.manager.mode == "gui": @@ -981,20 +929,12 @@ class PayjoinServer(Resource): "view wallet tab for update.:FINAL") else: self.info_callback("Transaction seen on network: " + txid) - # do end processing of calling object (e.g. Tor disconnect) + # in some cases (GUI) a notification of HS end is needed: self.shutdown_callback() # informs the wallet service transaction monitor # that the transaction has been processed: return True - def end_failure(self): - shutdown_msg = "Shutting down, payjoin negotiation failed." - if self.manager.mode == "gui": - shutdown_msg += "\nCheck wallet tab for payment." - shutdown_msg += ":FINAL" - self.info_callback(shutdown_msg) - self.shutdown_callback() - class JMBIP78ReceiverManager(object): """ A class to encapsulate receiver construction """ @@ -1030,6 +970,29 @@ class JMBIP78ReceiverManager(object): self.shutdown_callback = shutdown_callback self.receiving_address = None self.mode = mode + self.get_receiving_address() + self.manager = JMPayjoinManager(wallet_service, mixdepth, + self.receiving_address, amount, + mode=mode, + user_info_callback=self.info_callback) + + def initiate(self): + """ Called at reactor start to start up hidden service + and provide uri string to sender. + """ + # Note that we don't pass a "failure_callback" to the BIP78 + # Protocol; because the only failure is that the payment + # HTTP request simply doesn't arrive. Note also that the + # "params" argument is None as this is only learnt from request. + factory = BIP78ClientProtocolFactory(self, None, + self.receive_proposal_from_sender, None, + mode="receiver") + h = jm_single().config.get("DAEMON", "daemon_host") + p = jm_single().config.getint("DAEMON", "daemon_port")-2000 + if jm_single().config.get("DAEMON", "use_ssl") != 'false': + reactor.connectSSL(h, p, factory, ClientContextFactory()) + else: + reactor.connectTCP(h, p, factory) def default_info_callback(self, msg): jmprint(msg) @@ -1042,48 +1005,17 @@ class JMBIP78ReceiverManager(object): self.receiving_address = btc.CCoinAddress( self.wallet_service.get_internal_addr(next_mixdepth)) - def start_pj_server_and_tor(self): - """ Packages the startup of the receiver side. - """ - self.get_receiving_address() - self.pj_server = PayjoinServer(self.wallet_service, self.mixdepth, - self.receiving_address, self.amount, - self.shutdown, self.info_callback, mode=self.mode) - self.site = Site(self.pj_server) - self.site.displayTracebacks = False - self.info_callback("Attempting to start onion service on port: " + str( - self.port) + " ...") - self.start_tor() - - def setup_failed(self, arg): - errmsg = "Setup failed: " + str(arg) - log.error(errmsg) - self.info_callback(errmsg) - process_shutdown() - - def create_onion_ep(self, t): - self.tor_connection = t - return t.create_onion_endpoint(self.port) - - def onion_listen(self, onion_ep): - return onion_ep.listen(self.site) - - def print_host(self, ep): - """ Callback fired once the HS is available; - receiver user needs a BIP21 URI to pass to - the sender: + def receive_proposal_from_sender(self, body, params): + """ Accepts the contents of the HTTP request from the sender + and returns a payjoin proposal, or an error. """ - self.info_callback("Your hidden service is available. Please\n" - "now pass this URI string to the sender to\n" - "effect the payjoin payment:") - # Note that ep,getHost().onion_port must return the same - # port as we chose in self.port; if not there is an error. - assert ep.getHost().onion_port == self.port - self.uri_created_callback(self.bip21_uri_from_onion_hostname( - str(ep.getHost().onion_uri))) - if self.mode == "command-line": - self.info_callback("Keep this process running until the payment " - "is received.") + self.pj_converter = PayjoinConverter(self.manager, + self.shutdown, self.info_callback) + success, a, b = self.pj_converter.request_to_psbt(body, params) + if not success: + return (False, a, b) + else: + return (True, a) def bip21_uri_from_onion_hostname(self, host): """ Encoding the BIP21 URI according to BIP78 specifications, @@ -1096,38 +1028,28 @@ class JMBIP78ReceiverManager(object): full_pj_string = "http://" + host + port_str bip78_btc_amount = btc.amount_to_btc(btc.amount_to_sat(self.amount)) # "safe" option is required to encode url in url unmolested: - return btc.encode_bip21_uri(str(self.receiving_address), + bip21_uri = btc.encode_bip21_uri(str(self.receiving_address), {"amount": bip78_btc_amount, "pj": full_pj_string.encode("utf-8")}, safe=":/") - - def start_tor(self): - """ This function executes the workflow - of starting the hidden service and returning/ - printing the BIP21 URI: - """ - control_host = jm_single().config.get("PAYJOIN", "tor_control_host") - control_port = int(jm_single().config.get("PAYJOIN", "tor_control_port")) - if str(control_host).startswith('unix:'): - control_endpoint = UNIXClientEndpoint(reactor, control_host[5:]) - else: - control_endpoint = TCP4ClientEndpoint(reactor, control_host, control_port) - d = txtorcon.connect(reactor, control_endpoint) - d.addCallback(self.create_onion_ep) - d.addErrback(self.setup_failed) - # TODO: add errbacks to the next two calls in - # the chain: - d.addCallback(self.onion_listen) - d.addCallback(self.print_host) + self.info_callback("Your hidden service is available. Please\n" + "now pass this URI string to the sender to\n" + "effect the payjoin payment:") + self.uri_created_callback(bip21_uri) + if self.mode == "command-line": + self.info_callback("Keep this process running until the payment " + "is received.") def shutdown(self): - self.tor_connection.protocol.transport.loseConnection() + """ Triggered when processing has completed successfully + or failed, receiver side. + """ process_shutdown(self.mode) # on receiver side, if we are part of a long running # process (meaning above process_shutdown is a no-op), # we need to abandon the delayed call (this is the normal # success case): - tfdc = self.pj_server.manager.timeout_fallback_dc + tfdc = self.manager.timeout_fallback_dc if tfdc and tfdc.active(): tfdc.cancel() self.info_callback("Hidden service shutdown complete") diff --git a/jmclient/test/test_payjoin.py b/jmclient/test/test_payjoin.py index 076c356..2738bdb 100644 --- a/jmclient/test/test_payjoin.py +++ b/jmclient/test/test_payjoin.py @@ -12,12 +12,15 @@ from twisted.trial import unittest import urllib.parse as urlparse from urllib.parse import urlencode -from jmbase import get_log, jmprint, BytesProducer -from jmbitcoin import (CCoinAddress, encode_bip21_uri, +from jmbase import (get_log, jmprint, BytesProducer, + JMHTTPResource, get_nontor_agent, + wrapped_urlparse) +from jmbitcoin import (encode_bip21_uri, amount_to_btc, amount_to_sat) from jmclient import (load_test_config, jm_single, - SegwitLegacyWallet, - PayjoinServer, parse_payjoin_setup) + SegwitLegacyWallet, SegwitWallet, + parse_payjoin_setup, + JMBIP78ReceiverManager) from jmclient.payjoin import make_payjoin_request_params, make_payment_psbt from jmclient.payjoin import process_payjoin_proposal_from_server from commontest import make_wallets @@ -26,17 +29,43 @@ from test_coinjoin import make_wallets_to_list, sync_wallets testdir = os.path.dirname(os.path.realpath(__file__)) log = get_log() -class TrialTestPayjoinServer(unittest.TestCase): - +class DummyBIP78ReceiverResource(JMHTTPResource): + """ A simplified version of the BIP78Resource object created + to serve requests in jmdaemon. + """ + def __init__(self, info_callback, shutdown_callback, bip78receivermanager): + assert isinstance(bip78receivermanager, JMBIP78ReceiverManager) + self.bip78_receiver_manager = bip78receivermanager + self.info_callback = info_callback + self.shutdown_callback = shutdown_callback + super().__init__(info_callback, shutdown_callback) + + def render_POST(self, request): + proposed_tx = request.content + payment_psbt_base64 = proposed_tx.read().decode("utf-8") + retval = self.bip78_receiver_manager.receive_proposal_from_sender( + payment_psbt_base64, request.args) + assert retval[0] + content = retval[1].encode("utf-8") + request.setHeader(b"content-length", ("%d" % len(content))) + return content + +class PayjoinTestBase(object): + """ This tests that a payjoin invoice and + then payment of the invoice, results in the correct changes + in balance in the sender and receiver wallets, while also + implicitly testing all the BIP78 rules (failures are caught + by the JMPayjoinManager and PayjoinConverter rules). + """ def setUp(self): load_test_config() jm_single().bc_interface.tick_forward_chain_interval = 5 jm_single().bc_interface.simulate_blocks() - def test_payment(self): + def do_test_payment(self, wc1, wc2): wallet_structures = [[1, 3, 0, 0, 0]] * 2 mean_amt = 2.0 - wallet_cls = (SegwitLegacyWallet, SegwitLegacyWallet) + wallet_cls = (wc1, wc2) self.wallet_services = [] self.wallet_services.append(make_wallets_to_list(make_wallets( 1, wallet_structures=[wallet_structures[0]], @@ -53,37 +82,31 @@ class TrialTestPayjoinServer(unittest.TestCase): self.ssb = getbals(self.wallet_services[1], 0) self.cj_amount = int(1.1 * 10**8) - # destination address is in 2nd mixdepth of receiver - # (note: not first because sourcing from first) - bip78_receiving_address = self.wallet_services[0].get_internal_addr(1) def cbStopListening(): return self.port.stopListening() - pjs = PayjoinServer(self.wallet_services[0], 0, - CCoinAddress(bip78_receiving_address), - self.cj_amount, cbStopListening, jmprint) - site = Site(pjs) - - # NB The connectivity aspects of the BIP78 tests are in - # test/payjoin[client/server].py as they are time heavy - # and require extra setup. This server is TCP only. - self.port = reactor.listenTCP(47083, site) + b78rm = JMBIP78ReceiverManager(self.wallet_services[0], 0, + self.cj_amount, 47083) + resource = DummyBIP78ReceiverResource(jmprint, cbStopListening, b78rm) + self.site = Site(resource) + self.site.displayTracebacks = False + # NB The connectivity aspects of the onion-based BIP78 setup + # are time heavy. This server is TCP only. + self.port = reactor.listenTCP(47083, self.site) self.addCleanup(cbStopListening) # setup of spender bip78_btc_amount = amount_to_btc(amount_to_sat(self.cj_amount)) - bip78_uri = encode_bip21_uri(bip78_receiving_address, + bip78_uri = encode_bip21_uri(str(b78rm.receiving_address), {"amount": bip78_btc_amount, "pj": b"http://127.0.0.1:47083"}, safe=":/") self.manager = parse_payjoin_setup(bip78_uri, self.wallet_services[1], 0) self.manager.mode = "testing" - self.site = site success, msg = make_payment_psbt(self.manager) assert success, msg params = make_payjoin_request_params(self.manager) # avoiding backend daemon (testing only jmclient code here), # we send the http request manually: - from jmbase import get_nontor_agent, wrapped_urlparse serv = b"http://127.0.0.1:47083" agent = get_nontor_agent() body = BytesProducer(self.manager.initial_psbt.to_base64().encode("utf-8")) @@ -98,12 +121,21 @@ class TrialTestPayjoinServer(unittest.TestCase): def tearDown(self): for dc in reactor.getDelayedCalls(): - dc.cancel() + dc.cancel() res = final_checks(self.wallet_services, self.cj_amount, self.manager.final_psbt.get_fee(), self.ssb, self.rsb) assert res, "final checks failed" +class TrialTestPayjoin1(PayjoinTestBase, unittest.TestCase): + def test_payment(self): + return self.do_test_payment(SegwitLegacyWallet, SegwitLegacyWallet) + +class TrialTestPayjoin2(PayjoinTestBase, unittest.TestCase): + def test_bech32_payment(self): + return self.do_test_payment(SegwitWallet, SegwitWallet) + + def bip78_receiver_response(response, manager): d = readBody(response) # if the response code is not 200 OK, we must assume payjoin @@ -115,6 +147,7 @@ def bip78_receiver_response(response, manager): def process_receiver_errormsg(r, c): print("Failed: r, c: ", r, c) + assert False def process_receiver_psbt(response, manager): process_payjoin_proposal_from_server(response.decode("utf-8"), manager) diff --git a/jmdaemon/jmdaemon/daemon_protocol.py b/jmdaemon/jmdaemon/daemon_protocol.py index a884e44..3bc9268 100644 --- a/jmdaemon/jmdaemon/daemon_protocol.py +++ b/jmdaemon/jmdaemon/daemon_protocol.py @@ -9,17 +9,19 @@ from .protocol import (COMMAND_PREFIX, ORDER_KEYS, NICK_HASH_LENGTH, COMMITMENT_PREFIXES) from .irc import IRCMessageChannel -from jmbase import (hextobin, is_hs_uri, get_tor_agent, - get_nontor_agent, BytesProducer, wrapped_urlparse) +from jmbase import (hextobin, is_hs_uri, get_tor_agent, JMHiddenService, + get_nontor_agent, BytesProducer, wrapped_urlparse, + bintohex, bdict_sdict_convert, JMHTTPResource) from jmbase.commands import * from twisted.protocols import amp -from twisted.internet import reactor, ssl +from twisted.internet import reactor, ssl, task from twisted.internet.protocol import ServerFactory from twisted.internet.error import (ConnectionLost, ConnectionAborted, ConnectionClosed, ConnectionDone, ConnectionRefusedError) from twisted.web.http_headers import Headers from twisted.web.client import ResponseFailed, readBody +from twisted.web import server from txtorcon.socks import HostUnreachableError from twisted.python import log import urllib.parse as urlparse @@ -27,6 +29,7 @@ from urllib.parse import urlencode import json import threading import os +from io import BytesIO import copy from functools import wraps from numbers import Integral @@ -94,6 +97,67 @@ def check_utxo_blacklist(commitment, persist=False): class JMProtocolError(Exception): pass +class BIP78ReceiverResource(JMHTTPResource): + + def __init__(self, info_callback, shutdown_callback, post_request_handler): + """ The POST request handling callback has function signature: + args: (request-body-content-in-bytes,) + returns: (errormsg, errcode, httpcode, response-in-bytes) + If the request was successful, errormsg should be true and response + should be in bytes, to be sent in the return value of render_POST(). + """ + self.post_request_handler = post_request_handler + super().__init__(info_callback, shutdown_callback) + + def bip78_error(self, request, error_meaning, + error_code="unavailable", http_code=400): + """ + See https://github.com/bitcoin/bips/blob/master/bip-0078.mediawiki#receivers-well-known-errors + + We return, to the sender, stringified json in the body as per the above. + """ + request.setResponseCode(http_code) + request.setHeader(b"content-type", b"text/html; charset=utf-8") + print("Returning an error: " + str( + error_code) + ": " + str(error_meaning)) + if error_code in ["original-psbt-rejected", "version-unsupported"]: + # if there is a negotiation failure in the first step, we cannot + # know whether the sender client sent a valid non-payjoin or not, + # hence the warning below is somewhat ambiguous: + print("Negotiation failure. Payment has not yet been made," + " check wallet.") + # shutdown now but wait until response is sent. + task.deferLater(reactor, 2.0, self.end_failure) + return json.dumps({"errorCode": error_code, + "message": error_meaning}).encode("utf-8") + + def render_POST(self, request): + """ The sender will use POST to send the initial + payment transaction. + """ + print("The server got this POST request: ") + # unfortunately the twisted Request object is not + # easily serialized: + print(request) + print(request.method) + print(request.uri) + print(request.args) + sender_parameters = request.args + print(request.path) + # defer logging of raw request content: + proposed_tx = request.content + if not isinstance(proposed_tx, BytesIO): + return self.bip78_error(request, "invalid psbt format", + "original-psbt-rejected") + payment_psbt_base64 = proposed_tx.read().decode("utf-8") + reactor.callLater(0.0, self.post_request_handler, request, + payment_psbt_base64, sender_parameters) + return server.NOT_DONE_YET + + def end_failure(self): + self.info_callback("Shutting down, payjoin negotiation failed.") + self.shutdown_callback() + class HTTPPassThrough(amp.AMP): """ This class supports passing through requests over HTTPS or over a socks proxy to a remote @@ -183,7 +247,7 @@ class HTTPPassThrough(amp.AMP): failure.trap(ResponseFailed, ConnectionRefusedError, HostUnreachableError, ConnectionLost) log.msg(failure.value) - self.callRemote(BIP78ReceiverError, + self.callRemote(BIP78SenderReceiveError, errormsg="failure to connect", errorcode=10000) d.addErrback(noResponse) @@ -206,6 +270,80 @@ class HTTPPassThrough(amp.AMP): d.addErrback(self.defaultErrback) class BIP78ServerProtocol(HTTPPassThrough): + @BIP78ReceiverInit.responder + def on_BIP78_RECEIVER_INIT(self, netconfig): + netconfig = json.loads(netconfig) + self.serving_port = int(netconfig["port"]) + self.tor_control_host = netconfig["tor_control_host"] + self.tor_control_port = int(netconfig["tor_control_port"]) + self.bip78_rr = BIP78ReceiverResource(self.info_callback, + self.shutdown_callback, + self.post_request_handler) + self.hs = JMHiddenService(self.bip78_rr, + self.info_callback, + self.setup_error_callback, + self.onion_hostname_callback, + self.tor_control_host, + self.tor_control_port, + self.serving_port, + self.shutdown_callback) + # this call will start bringing up the HS; when it's finished, + # it will fire the `onion_hostname_callback`, or if it fails, + # it'll fire the `setup_error_callback`. + self.hs.start_tor() + return {"accepted": True} + + def setup_error_callback(self, errormsg): + d = self.callRemote(BIP78ReceiverOnionSetupFailed, + reason=errormsg) + self.defaultCallbacks(d) + + def shutdown_callback(self): + d = self.callRemote(BIP78ReceiverHiddenServiceShutdown) + self.defaultCallbacks(d) + + def info_callback(self, msg): + """ Informational messages are all passed + to the client. TODO makes sense to log locally + too, in case daemon is isolated?. + """ + d = self.callRemote(BIP78InfoMsg, infomsg=msg) + self.defaultCallbacks(d) + + def onion_hostname_callback(self, hostname): + """ On successful start of HS, we pass hostname + to client, who can use this to build the full URI. + """ + d = self.callRemote(BIP78ReceiverUp, + hostname=hostname) + self.defaultCallbacks(d) + + def post_request_handler(self, request, body, params): + """ Fired when a sender has sent a POST request + to our hidden service. Argument `body` should be a base64 + string and params should be a dict. + """ + self.post_request = request + d = self.callRemote(BIP78ReceiverOriginalPSBT, body=body, + params=json.dumps(bdict_sdict_convert(params))) + self.defaultCallbacks(d) + + @BIP78ReceiverSendProposal.responder + def on_BIP78_RECEIVER_SEND_PROPOSAL(self, psbt): + content = psbt.encode("utf-8") + self.post_request.setHeader(b"content-length", + ("%d" % len(content))) + self.post_request.write(content) + self.post_request.finish() + return {"accepted": True} + + @BIP78ReceiverSendError.responder + def on_BIP78_RECEIVER_SEND_ERROR(self, errormsg, errorcode): + self.post_request.write(self.bip78_rr.bip78_error( + self.post_request, errormsg, errorcode)) + self.post_request.finish() + return {"accepted": True} + @BIP78SenderInit.responder def on_BIP78_SENDER_INIT(self, netconfig): self.on_INIT(netconfig) @@ -231,7 +369,7 @@ class BIP78ServerProtocol(HTTPPassThrough): d.addCallback(self.process_receiver_psbt) def process_receiver_errormsg(self, response, errorcode): - d = self.callRemote(BIP78ReceiverError, + d = self.callRemote(BIP78SenderReceiveError, errormsg=response.decode("utf-8"), errorcode=errorcode) self.defaultCallbacks(d) diff --git a/scripts/joinmarket-qt.py b/scripts/joinmarket-qt.py index 43596c0..f862237 100755 --- a/scripts/joinmarket-qt.py +++ b/scripts/joinmarket-qt.py @@ -849,7 +849,7 @@ class SpendTab(QWidget): self.clientfactory = JMClientProtocolFactory(self.taker) daemon = jm_single().config.getint("DAEMON", "no_daemon") daemon = True if daemon == 1 else False - start_reactor("localhost", + start_reactor(jm_single().config.get("DAEMON", "daemon_host"), jm_single().config.getint("DAEMON", "daemon_port"), self.clientfactory, ish=False, @@ -1471,6 +1471,9 @@ class JMMainWindow(QMainWindow): # BIP 78 Receiver manager object, only # created when user starts a payjoin event: self.backend_receiver = None + # Keep track of whether the BIP78 daemon has + # been started, to avoid unnecessary duplication: + self.bip78daemon = False self.reactor = reactor self.initUI() @@ -1572,7 +1575,19 @@ class JMMainWindow(QMainWindow): uri_created_callback=self.receiver_bip78_dialog.update_uri, shutdown_callback=self.receiver_bip78_dialog.process_complete, mode="gui") - self.backend_receiver.start_pj_server_and_tor() + if not self.bip78daemon: + #First run means we need to start: create daemon; + # the client and its connection are created in the .initiate() + # call. + daemon = jm_single().config.getint("DAEMON", "no_daemon") + if daemon: + # this call not needed if daemon is external. + start_reactor(jm_single().config.get("DAEMON", "daemon_host"), + jm_single().config.getint("DAEMON", "daemon_port"), + jm_coinjoin=False, bip78=True, daemon=True, + gui=True, rs=False) + self.bip78daemon = True + self.backend_receiver.initiate() return True def stopReceiver(self): diff --git a/scripts/receive-payjoin.py b/scripts/receive-payjoin.py index 8a3bc99..c1e2df7 100755 --- a/scripts/receive-payjoin.py +++ b/scripts/receive-payjoin.py @@ -8,7 +8,7 @@ from twisted.internet import reactor from jmbase import get_log, set_logging_level, jmprint from jmclient import jm_single, load_program_config, \ WalletService, open_test_wallet_maybe, get_wallet_path, check_regtest, \ - add_base_options, JMBIP78ReceiverManager + add_base_options, JMBIP78ReceiverManager, start_reactor from jmbase.support import EXIT_FAILURE, EXIT_ARGERROR from jmbitcoin import amount_to_sat jlog = get_log() @@ -70,8 +70,13 @@ def receive_payjoin_main(): sys.exit(EXIT_ARGERROR) receiver_manager = JMBIP78ReceiverManager(wallet_service, options.mixdepth, bip78_amount, options.hsport) - receiver_manager.start_pj_server_and_tor() - reactor.run() + reactor.callWhenRunning(receiver_manager.initiate) + nodaemon = jm_single().config.getint("DAEMON", "no_daemon") + daemon = True if nodaemon == 1 else False + dhost = jm_single().config.get("DAEMON", "daemon_host") + dport = jm_single().config.getint("DAEMON", "daemon_port") + # JM is default, so must be switched off explicitly in this call: + start_reactor(dhost, dport, bip78=True, jm_coinjoin=False, daemon=daemon) if __name__ == "__main__": receive_payjoin_main()