From 7dcd3f3b5a165f2410a8d2b0ddb6e4e7deeb4999 Mon Sep 17 00:00:00 2001 From: undeath Date: Sun, 27 Jun 2021 18:14:07 +0200 Subject: [PATCH 1/4] move client/server data encoding to twisted --- jmbase/jmbase/arguments.py | 10 +++ jmbase/jmbase/commands.py | 46 +++++----- jmbase/test/test_commands.py | 22 ++--- jmclient/jmclient/client_protocol.py | 61 ++++++-------- jmclient/jmclient/maker.py | 6 +- jmclient/test/test_client_protocol.py | 20 +++-- jmdaemon/jmdaemon/daemon_protocol.py | 116 +++++++++++--------------- jmdaemon/jmdaemon/message_channel.py | 8 +- jmdaemon/test/test_daemon_protocol.py | 27 +++--- test/ygrunner.py | 8 +- 10 files changed, 153 insertions(+), 171 deletions(-) create mode 100644 jmbase/jmbase/arguments.py diff --git a/jmbase/jmbase/arguments.py b/jmbase/jmbase/arguments.py new file mode 100644 index 0000000..e6f68b5 --- /dev/null +++ b/jmbase/jmbase/arguments.py @@ -0,0 +1,10 @@ +import json +from twisted.protocols.amp import String + + +class JsonEncodable(String): + def toString(self, inObject): + return super().toString(json.dumps(inObject).encode('ascii')) + + def fromString(self, inString): + return super().fromString(json.loads(inString)) diff --git a/jmbase/jmbase/commands.py b/jmbase/jmbase/commands.py index 935d30e..2eed32e 100644 --- a/jmbase/jmbase/commands.py +++ b/jmbase/jmbase/commands.py @@ -3,8 +3,10 @@ Commands defining client-server (daemon) messaging protocol (*not* Joinmarket p2p protocol). Used for AMP asynchronous messages. """ -from twisted.protocols.amp import Boolean, Command, Integer, Unicode +from twisted.protocols.amp import Boolean, Command, Integer, Unicode, ListOf,\ + String from .bigstring import BigUnicode +from .arguments import JsonEncodable class DaemonNotReady(Exception): @@ -29,7 +31,7 @@ class JMInit(JMCommand): """ arguments = [(b'bcsource', Unicode()), (b'network', Unicode()), - (b'irc_configs', Unicode()), + (b'irc_configs', JsonEncodable()), (b'minmakers', Integer()), (b'maker_timeout_sec', Integer()), (b'dust_threshold', Integer())] @@ -47,7 +49,7 @@ class JMSetup(JMCommand): role, passes initial offers for announcement (for TAKER, this data is "none") """ arguments = [(b'role', Unicode()), - (b'offers', Unicode()), + (b'initdata', JsonEncodable()), (b'use_fidelity_bond', Boolean())] class JMMsgSignature(JMCommand): @@ -82,13 +84,13 @@ class JMFill(JMCommand): arguments = [(b'amount', Integer()), (b'commitment', Unicode()), (b'revelation', Unicode()), - (b'filled_offers', Unicode())] + (b'filled_offers', JsonEncodable())] class JMMakeTx(JMCommand): """Send a hex encoded raw bitcoin transaction to a set of counterparties """ - arguments = [(b'nick_list', Unicode()), + arguments = [(b'nick_list', ListOf(Unicode())), (b'txhex', Unicode())] class JMPushTx(JMCommand): @@ -106,9 +108,9 @@ class JMAnnounceOffers(JMCommand): to the daemon, along with new announcement and cancellation lists (deltas). """ - arguments = [(b'to_announce', Unicode()), - (b'to_cancel', Unicode()), - (b'offerlist', Unicode())] + arguments = [(b'to_announce', JsonEncodable()), + (b'to_cancel', JsonEncodable()), + (b'offerlist', JsonEncodable())] class JMFidelityBondProof(JMCommand): """Send requested fidelity bond proof message""" @@ -120,7 +122,7 @@ class JMIOAuth(JMCommand): verifying Taker's auth message """ arguments = [(b'nick', Unicode()), - (b'utxolist', Unicode()), + (b'utxolist', JsonEncodable()), (b'pubkey', Unicode()), (b'cjaddr', Unicode()), (b'changeaddr', Unicode()), @@ -131,7 +133,7 @@ class JMTXSigs(JMCommand): sent by TAKER """ arguments = [(b'nick', Unicode()), - (b'sigs', Unicode())] + (b'sigs', ListOf(Unicode()))] """COMMANDS FROM DAEMON TO CLIENT ================================= @@ -197,7 +199,7 @@ class JMFillResponse(JMCommand): """Returns ioauth data from MAKER if successful. """ arguments = [(b'success', Boolean()), - (b'ioauth_data', Unicode())] + (b'ioauth_data', JsonEncodable())] class JMSigReceived(JMCommand): """Returns an individual bitcoin transaction signature @@ -221,9 +223,9 @@ class JMAuthReceived(JMCommand): before setting up encryption and continuing. """ arguments = [(b'nick', Unicode()), - (b'offer', Unicode()), + (b'offer', JsonEncodable()), (b'commitment', Unicode()), - (b'revelation', Unicode()), + (b'revelation', JsonEncodable()), (b'amount', Integer()), (b'kphex', Unicode())] @@ -232,8 +234,8 @@ class JMTXReceived(JMCommand): by TAKER, along with offerdata to verify fees. """ arguments = [(b'nick', Unicode()), - (b'txhex', Unicode()), - (b'offer', Unicode())] + (b'tx', String()), + (b'offer', JsonEncodable())] class JMTXBroadcast(JMCommand): """ Accept a bitcoin transaction @@ -241,7 +243,7 @@ class JMTXBroadcast(JMCommand): and relay it to the client for network broadcast. """ - arguments = [(b'txhex', Unicode())] + arguments = [(b'tx', String())] """SNICKER related commands. """ @@ -251,12 +253,12 @@ class SNICKERReceiverInit(JMCommand): See documentation of `netconfig` in jmdaemon.HTTPPassThrough.on_INIT """ - arguments = [(b'netconfig', Unicode())] + arguments = [(b'netconfig', JsonEncodable())] class SNICKERProposerInit(JMCommand): """ As for receiver. """ - arguments = [(b'netconfig', Unicode())] + arguments = [(b'netconfig', JsonEncodable())] class SNICKERReceiverUp(JMCommand): arguments = [] @@ -307,7 +309,7 @@ class BIP78SenderInit(JMCommand): See documentation of `netconfig` in jmdaemon.HTTPPassThrough.on_INIT """ - arguments = [(b'netconfig', Unicode())] + arguments = [(b'netconfig', JsonEncodable())] class BIP78SenderUp(JMCommand): arguments = [] @@ -319,7 +321,7 @@ class BIP78SenderOriginalPSBT(JMCommand): to be sent as an http request to the receiver. """ arguments = [(b'body', BigUnicode()), - (b'params', Unicode())] + (b'params', JsonEncodable())] class BIP78SenderReceiveProposal(JMCommand): """ Sends the payjoin proposal PSBT, received @@ -341,7 +343,7 @@ class BIP78SenderReceiveError(JMCommand): class BIP78ReceiverInit(JMCommand): """ Initialization data for a BIP78 hidden service. """ - arguments = [(b'netconfig', Unicode())] + arguments = [(b'netconfig', JsonEncodable())] class BIP78ReceiverUp(JMCommand): """ Returns onion hostname to client when @@ -356,7 +358,7 @@ class BIP78ReceiverOriginalPSBT(JMCommand): parameters in the url, from the daemon to the client. """ arguments = [(b'body', BigUnicode()), - (b'params', Unicode())] + (b'params', JsonEncodable())] class BIP78ReceiverSendProposal(JMCommand): """ Receives a payjoin proposal PSBT from diff --git a/jmbase/test/test_commands.py b/jmbase/test/test_commands.py index 419a61f..dafda0f 100644 --- a/jmbase/test/test_commands.py +++ b/jmbase/test/test_commands.py @@ -63,8 +63,8 @@ class JMTestServerProtocol(JMBaseProtocol): return {'accepted': True} @JMSetup.responder - def on_JM_SETUP(self, role, offers, use_fidelity_bond): - show_receipt("JMSETUP", role, offers, use_fidelity_bond) + def on_JM_SETUP(self, role, initdata, use_fidelity_bond): + show_receipt("JMSETUP", role, initdata, use_fidelity_bond) d = self.callRemote(JMSetupDone) self.defaultCallbacks(d) return {'accepted': True} @@ -84,8 +84,8 @@ class JMTestServerProtocol(JMBaseProtocol): def on_JM_FILL(self, amount, commitment, revelation, filled_offers): show_receipt("JMFILL", amount, commitment, revelation, filled_offers) d = self.callRemote(JMFillResponse, - success=True, - ioauth_data = json.dumps(['dummy', 'list'])) + success=True, + ioauth_data=['dummy', 'list']) return {'accepted': True} @JMMakeTx.responder @@ -113,7 +113,7 @@ class JMTestServerProtocol(JMBaseProtocol): max_encoded=5, hostid="hostid2") self.defaultCallbacks(d3) - d4 = self.callRemote(JMTXBroadcast, txhex="deadbeef") + d4 = self.callRemote(JMTXBroadcast, tx=b"deadbeef") self.defaultCallbacks(d4) return {'accepted': True} @@ -137,7 +137,7 @@ class JMTestClientProtocol(JMBaseProtocol): d = self.callRemote(JMInit, bcsource="dummyblockchain", network="dummynetwork", - irc_configs=json.dumps(['dummy', 'irc', 'config']), + irc_configs=['dummy', 'irc', 'config'], minmakers=7, maker_timeout_sec=8, dust_threshold=1500) @@ -158,7 +158,7 @@ class JMTestClientProtocol(JMBaseProtocol): show_receipt("JMUP") d = self.callRemote(JMSetup, role="TAKER", - offers="{}", + initdata=None, use_fidelity_bond=False) self.defaultCallbacks(d) return {'accepted': True} @@ -174,7 +174,7 @@ class JMTestClientProtocol(JMBaseProtocol): def on_JM_FILL_RESPONSE(self, success, ioauth_data): show_receipt("JMFILLRESPONSE", success, ioauth_data) d = self.callRemote(JMMakeTx, - nick_list= json.dumps(['nick1', 'nick2', 'nick3']), + nick_list=['nick1', 'nick2', 'nick3'], txhex="deadbeef") self.defaultCallbacks(d) return {'accepted': True} @@ -186,7 +186,7 @@ class JMTestClientProtocol(JMBaseProtocol): amount=100, commitment="dummycommitment", revelation="dummyrevelation", - filled_offers=json.dumps(['list', 'of', 'filled', 'offers'])) + filled_offers=['list', 'of', 'filled', 'offers']) self.defaultCallbacks(d) return {'accepted': True} @@ -222,8 +222,8 @@ class JMTestClientProtocol(JMBaseProtocol): return {'accepted': True} @JMTXBroadcast.responder - def on_JM_TX_BROADCAST(self, txhex): - show_receipt("JMTXBROADCAST", txhex) + def on_JM_TX_BROADCAST(self, tx): + show_receipt("JMTXBROADCAST", tx) return {"accepted": True} class JMTestClientProtocolFactory(protocol.ClientFactory): diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index d19b32b..dd50184 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -73,13 +73,13 @@ class BIP78ClientProtocol(BaseClientProtocol): "tls_whitelist": ",".join(self.tls_whitelist), "servers": [self.manager.server]} d = self.callRemote(commands.BIP78SenderInit, - netconfig=json.dumps(netconfig)) + netconfig=netconfig) else: netconfig = {"port": 80, "tor_control_host": jcg("PAYJOIN", "tor_control_host"), "tor_control_port": jcg("PAYJOIN", "tor_control_port")} d = self.callRemote(commands.BIP78ReceiverInit, - netconfig=json.dumps(netconfig)) + netconfig=netconfig) self.defaultCallbacks(d) @commands.BIP78ReceiverUp.responder @@ -89,7 +89,6 @@ class BIP78ClientProtocol(BaseClientProtocol): @commands.BIP78ReceiverOriginalPSBT.responder def on_BIP78_RECEIVER_ORIGINAL_PSBT(self, body, params): - params = json.loads(params) # TODO: we don't need binary key/vals client side, but will have to edit # PayjoinConverter for that: retval = self.success_callback(body.encode("utf-8"), bdict_sdict_convert( @@ -121,7 +120,7 @@ class BIP78ClientProtocol(BaseClientProtocol): def on_BIP78_SENDER_UP(self): d = self.callRemote(commands.BIP78SenderOriginalPSBT, body=self.manager.initial_psbt.to_base64(), - params=json.dumps(self.params)) + params=self.params) self.defaultCallbacks(d) return {"accepted": True} @@ -168,10 +167,10 @@ class SNICKERClientProtocol(BaseClientProtocol): if isinstance(self.client, SNICKERReceiver): d = self.callRemote(commands.SNICKERReceiverInit, - netconfig=json.dumps(netconfig)) + netconfig=netconfig) else: d = self.callRemote(commands.SNICKERProposerInit, - netconfig=json.dumps(netconfig)) + netconfig=netconfig) self.defaultCallbacks(d) def shutdown(self): @@ -363,7 +362,7 @@ class JMClientProtocol(BaseClientProtocol): def make_tx(self, nick_list, txhex): d = self.callRemote(commands.JMMakeTx, - nick_list= json.dumps(nick_list), + nick_list=nick_list, txhex=txhex) self.defaultCallbacks(d) @@ -394,7 +393,7 @@ class JMMakerClientProtocol(JMClientProtocol): self.offers_ready_loop.stop() d = self.callRemote(commands.JMSetup, role="MAKER", - offers=json.dumps(self.client.offerlist), + initdata=self.client.offerlist, use_fidelity_bond=(self.client.fidelity_bond is not None)) self.defaultCallbacks(d) @@ -423,7 +422,7 @@ class JMMakerClientProtocol(JMClientProtocol): d = self.callRemote(commands.JMInit, bcsource=blockchain_source, network=network, - irc_configs=json.dumps(irc_configs), + irc_configs=irc_configs, minmakers=minmakers, maker_timeout_sec=maker_timeout_sec, dust_threshold=jm_single().DUST_THRESHOLD) @@ -443,8 +442,6 @@ class JMMakerClientProtocol(JMClientProtocol): @commands.JMAuthReceived.responder def on_JM_AUTH_RECEIVED(self, nick, offer, commitment, revelation, amount, kphex): - offer = json.loads(offer) - revelation = json.loads(revelation) retval = self.client.on_auth_received(nick, offer, commitment, revelation, amount, kphex) if not retval[0]: @@ -461,7 +458,7 @@ class JMMakerClientProtocol(JMClientProtocol): auth_pub_hex = bintohex(auth_pub) d = self.callRemote(commands.JMIOAuth, nick=nick, - utxolist=json.dumps(utxos_strkeyed), + utxolist=utxos_strkeyed, pubkey=auth_pub_hex, cjaddr=cj_addr, changeaddr=change_addr, @@ -470,17 +467,15 @@ class JMMakerClientProtocol(JMClientProtocol): return {"accepted": True} @commands.JMTXReceived.responder - def on_JM_TX_RECEIVED(self, nick, txhex, offer): - offer = json.loads(offer) - retval = self.client.on_tx_received(nick, txhex, offer) + def on_JM_TX_RECEIVED(self, nick, tx, offer): + retval = self.client.on_tx_received(nick, tx, offer) if not retval[0]: jlog.info("Maker refuses to continue on receipt of tx") else: sigs = retval[1] self.finalized_offers[nick] = offer - tx = btc.CMutableTransaction.deserialize(hextobin(txhex)) + tx = btc.CMutableTransaction.deserialize(tx) self.finalized_offers[nick]["txd"] = tx - txid = tx.GetTxid()[::-1] # we index the callback by the out-set of the transaction, # because the txid is not known until all scriptSigs collected # (hence this is required for Makers, but not Takers). @@ -497,14 +492,12 @@ class JMMakerClientProtocol(JMClientProtocol): txinfo, self.unconfirm_callback, "unconfirmed", "transaction with outputs: " + str(txinfo) + " not broadcast.") - d = self.callRemote(commands.JMTXSigs, - nick=nick, - sigs=json.dumps(sigs)) + d = self.callRemote(commands.JMTXSigs, nick=nick, sigs=sigs) self.defaultCallbacks(d) return {"accepted": True} @commands.JMTXBroadcast.responder - def on_JM_TX_BROADCAST(self, txhex): + def on_JM_TX_BROADCAST(self, tx): """ Makers have no issue broadcasting anything, so only need to prevent crashes. Note in particular we don't check the return value, @@ -512,11 +505,10 @@ class JMMakerClientProtocol(JMClientProtocol): our (maker)'s concern. """ try: - txbin = hextobin(txhex) - jm_single().bc_interface.pushtx(txbin) + jm_single().bc_interface.pushtx(tx) except: jlog.info("We received an invalid transaction broadcast " - "request: " + txhex) + "request: " + tx.hex()) return {"accepted": True} def tx_match(self, txd): @@ -547,9 +539,9 @@ class JMMakerClientProtocol(JMClientProtocol): "transaction with outputs " + str(txinfo) + " not confirmed.") d = self.callRemote(commands.JMAnnounceOffers, - to_announce=json.dumps(to_announce), - to_cancel=json.dumps(to_cancel), - offerlist=json.dumps(self.client.offerlist)) + to_announce=to_announce, + to_cancel=to_cancel, + offerlist=self.client.offerlist) self.defaultCallbacks(d) return True @@ -564,9 +556,9 @@ class JMMakerClientProtocol(JMClientProtocol): txid, confirms) self.client.modify_orders(to_cancel, to_announce) d = self.callRemote(commands.JMAnnounceOffers, - to_announce=json.dumps(to_announce), - to_cancel=json.dumps(to_cancel), - offerlist=json.dumps(self.client.offerlist)) + to_announce=to_announce, + to_cancel=to_cancel, + offerlist=self.client.offerlist) self.defaultCallbacks(d) return True @@ -601,7 +593,7 @@ class JMTakerClientProtocol(JMClientProtocol): d = self.callRemote(commands.JMInit, bcsource=blockchain_source, network=network, - irc_configs=json.dumps(irc_configs), + irc_configs=irc_configs, minmakers=minmakers, maker_timeout_sec=maker_timeout_sec, dust_threshold=jm_single().DUST_THRESHOLD) @@ -641,7 +633,7 @@ class JMTakerClientProtocol(JMClientProtocol): def on_JM_UP(self): d = self.callRemote(commands.JMSetup, role="TAKER", - offers="{}", + initdata=None, use_fidelity_bond=False) self.defaultCallbacks(d) return {'accepted': True} @@ -669,13 +661,12 @@ class JMTakerClientProtocol(JMClientProtocol): the ioauth data and returns the proposed transaction, passes the phase 2 initiating data to the daemon. """ - ioauth_data = json.loads(ioauth_data) if not success: jlog.info("Makers who didnt respond: " + str(ioauth_data)) self.client.add_ignored_makers(ioauth_data) return {'accepted': True} else: - jlog.info("Makers responded with: " + json.dumps(ioauth_data)) + jlog.info("Makers responded with: " + str(ioauth_data)) retval = self.client.receive_utxos(ioauth_data) if not retval[0]: jlog.info("Taker is not continuing, phase 2 abandoned.") @@ -717,7 +708,7 @@ class JMTakerClientProtocol(JMClientProtocol): amount=amt, commitment=str(cmt), revelation=str(rev), - filled_offers=json.dumps(foffers)) + filled_offers=foffers) self.defaultCallbacks(d) return {'accepted': True} diff --git a/jmclient/jmclient/maker.py b/jmclient/jmclient/maker.py index 2213598..3fd9066 100644 --- a/jmclient/jmclient/maker.py +++ b/jmclient/jmclient/maker.py @@ -118,7 +118,7 @@ class Maker(object): return (True, utxos, auth_pub, cj_addr, change_addr, btc_sig) @hexbin - def on_tx_received(self, nick, tx_from_taker, offerinfo): + def on_tx_received(self, nick, tx, offerinfo): """Called when the counterparty has sent an unsigned transaction. Sigs are created and returned if and only if the transaction passes verification checks (see @@ -129,9 +129,9 @@ class Maker(object): if not isinstance(offerinfo["offer"]["cjfee"], str): offerinfo["offer"]["cjfee"] = bintohex(offerinfo["offer"]["cjfee"]) try: - tx = btc.CMutableTransaction.deserialize(tx_from_taker) + tx = btc.CMutableTransaction.deserialize(tx) except Exception as e: - return (False, 'malformed txhex. ' + repr(e)) + return (False, 'malformed tx. ' + repr(e)) # if the above deserialization was successful, the human readable # parsing will be also: jlog.info('obtained tx\n' + btc.human_readable_transaction(tx)) diff --git a/jmclient/test/test_client_protocol.py b/jmclient/test/test_client_protocol.py index bf8ceb8..0d4a6ba 100644 --- a/jmclient/test/test_client_protocol.py +++ b/jmclient/test/test_client_protocol.py @@ -20,6 +20,7 @@ from commontest import default_max_cj_fee import json import jmbitcoin as bitcoin import twisted +import base64 twisted.internet.base.DelayedCall.debug = True test_completed = False @@ -99,7 +100,7 @@ class DummyMaker(Maker): # success, utxos, auth_pub, cj_addr, change_addr, btc_sig return True, [], b"", '', '', '' - def on_tx_received(self, nick, txhex, offerinfo): + def on_tx_received(self, nick, tx, offerinfo): # success, sigs return True, [] @@ -185,8 +186,8 @@ class JMTestServerProtocol(JMBaseProtocol): return {'accepted': True} @JMSetup.responder - def on_JM_SETUP(self, role, offers, use_fidelity_bond): - show_receipt("JMSETUP", role, offers, use_fidelity_bond) + def on_JM_SETUP(self, role, initdata, use_fidelity_bond): + show_receipt("JMSETUP", role, initdata, use_fidelity_bond) d = self.callRemote(JMSetupDone) self.defaultCallbacks(d) return {'accepted': True} @@ -208,8 +209,8 @@ class JMTestServerProtocol(JMBaseProtocol): success = False if amount == -1 else True show_receipt("JMFILL", amount, commitment, revelation, filled_offers) d = self.callRemote(JMFillResponse, - success=success, - ioauth_data = json.dumps(['dummy', 'list'])) + success=success, + ioauth_data=['dummy', 'list']) return {'accepted': True} @JMMakeTx.responder @@ -390,13 +391,14 @@ class TestMakerClientProtocol(unittest.TestCase): def test_JMAuthReceived(self): yield self.init_client() yield self.callClient( - JMAuthReceived, nick='testnick', offer='{}', - commitment='testcommitment', revelation='{}', amount=100000, + JMAuthReceived, nick='testnick', offer={}, + commitment='testcommitment', revelation={}, amount=100000, kphex='testkphex') @inlineCallbacks def test_JMTXReceived(self): yield self.init_client() yield self.callClient( - JMTXReceived, nick='testnick', txhex=t_raw_signed_tx, - offer='{"cjaddr":"2MwfecDHsQTm4Gg3RekQdpqAMR15BJrjfRF"}') + JMTXReceived, nick='testnick', + tx=base64.b16decode(t_raw_signed_tx, casefold=True), + offer={"cjaddr":"2MwfecDHsQTm4Gg3RekQdpqAMR15BJrjfRF"}) diff --git a/jmdaemon/jmdaemon/daemon_protocol.py b/jmdaemon/jmdaemon/daemon_protocol.py index 44280c9..7ae1868 100644 --- a/jmdaemon/jmdaemon/daemon_protocol.py +++ b/jmdaemon/jmdaemon/daemon_protocol.py @@ -9,7 +9,7 @@ from .protocol import (COMMAND_PREFIX, ORDER_KEYS, NICK_HASH_LENGTH, COMMITMENT_PREFIXES) from .irc import IRCMessageChannel -from jmbase import (hextobin, is_hs_uri, get_tor_agent, JMHiddenService, +from jmbase import (is_hs_uri, get_tor_agent, JMHiddenService, get_nontor_agent, BytesProducer, wrapped_urlparse, bdict_sdict_convert, JMHTTPResource) from jmbase.commands import * @@ -32,7 +32,6 @@ import os from io import BytesIO import copy from functools import wraps -from numbers import Integral """Joinmarket application protocol control flow. For documentation on protocol (formats, message sequence) see @@ -166,15 +165,14 @@ class HTTPPassThrough(amp.AMP): def on_INIT(self, netconfig): """ The network config must be passed in json - and contains these fields: - socks5_host - socks5_proxy - servers (comma separated list) - tls_whitelist (comma separated list) - filterconfig (not yet defined) - credentials (not yet defined) - """ - netconfig = json.loads(netconfig) + and contains these fields: + socks5_host + socks5_proxy + servers (comma separated list) + tls_whitelist (comma separated list) + filterconfig (not yet defined) + credentials (not yet defined) + """ self.socks5_host = netconfig["socks5_host"] self.socks5_port = int(netconfig["socks5_port"]) self.servers = [a for a in netconfig["servers"] if a != ""] @@ -272,7 +270,6 @@ class HTTPPassThrough(amp.AMP): class BIP78ServerProtocol(HTTPPassThrough): @BIP78ReceiverInit.responder def on_BIP78_RECEIVER_INIT(self, netconfig): - netconfig = json.loads(netconfig) self.serving_port = int(netconfig["port"]) self.tor_control_host = netconfig["tor_control_host"] self.tor_control_port = int(netconfig["tor_control_port"]) @@ -325,7 +322,7 @@ class BIP78ServerProtocol(HTTPPassThrough): """ self.post_request = request d = self.callRemote(BIP78ReceiverOriginalPSBT, body=body, - params=json.dumps(bdict_sdict_convert(params))) + params=bdict_sdict_convert(params)) self.defaultCallbacks(d) @BIP78ReceiverSendProposal.responder @@ -354,9 +351,9 @@ class BIP78ServerProtocol(HTTPPassThrough): @BIP78SenderOriginalPSBT.responder def on_BIP78_SENDER_ORIGINAL_PSBT(self, body, params): self.postRequest(body, self.servers[0], - self.bip78_receiver_response, - params=json.loads(params), - headers=Headers({"Content-Type": ["text/plain"]})) + self.bip78_receiver_response, + params=params, + headers=Headers({"Content-Type": ["text/plain"]})) return {"accepted": True} def bip78_receiver_response(self, response, server): @@ -475,6 +472,8 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): self.sig_lock = threading.Lock() self.active_orders = {} self.use_fidelity_bond = False + self.offerlist = None + self.kp = None def checkClientResponse(self, response): """A generic check of client acceptance; any failure @@ -502,11 +501,9 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): If a new message channel configuration is required, the current one is shutdown in preparation. """ - self.maker_timeout_sec = int(maker_timeout_sec) - # used in OrderbookWatch: + self.maker_timeout_sec = maker_timeout_sec + self.minmakers = minmakers self.dust_threshold = int(dust_threshold) - self.minmakers = int(minmakers) - irc_configs = json.loads(irc_configs) #(bitcoin) network only referenced in channel name construction self.network = network if irc_configs == self.irc_configs: @@ -554,7 +551,7 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): return {'accepted': True} @JMSetup.responder - def on_JM_SETUP(self, role, offers, use_fidelity_bond): + def on_JM_SETUP(self, role, initdata, use_fidelity_bond): assert self.jm_state == 0 self.role = role self.crypto_boxes = {} @@ -568,7 +565,7 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): if self.role == "TAKER": self.mcc.pubmsg(COMMAND_PREFIX + "orderbook") elif self.role == "MAKER": - self.offerlist = json.loads(offers) + self.offerlist = initdata self.use_fidelity_bond = use_fidelity_bond self.mcc.announce_orders(self.offerlist, None, None, None) self.jm_state = 1 @@ -614,15 +611,14 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): """Takes the necessary data from the Taker and initiates the Stage 1 interaction with the Makers. """ - if not (self.jm_state == 1 and isinstance(amount, Integral) - and amount >= 0): + if self.jm_state != 1 or amount < 0: return {'accepted': False} self.cjamount = amount self.commitment = commitment self.revelation = revelation #Reset utxo data to null for this new transaction self.ioauth_data = {} - self.active_orders = json.loads(filled_offers) + self.active_orders = filled_offers for nick, offer_dict in self.active_orders.items(): offer_fill_msg = " ".join([str(offer_dict["oid"]), str(amount), self.kp.hex_pk().decode('ascii'), str(commitment)]) @@ -639,7 +635,6 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): if not self.jm_state == 4: log.msg("Make tx was called in wrong state, rejecting") return {'accepted': False} - nick_list = json.loads(nick_list) self.mcc.send_tx(nick_list, txhex) return {'accepted': True} @@ -659,9 +654,7 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): """ if self.role != "MAKER": return - to_announce = json.loads(to_announce) - to_cancel = json.loads(to_cancel) - self.offerlist = json.loads(offerlist) + self.offerlist = offerlist if len(to_cancel) > 0: self.mcc.cancel_orders(to_cancel) if len(to_announce) > 0: @@ -682,14 +675,13 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): """ if not self.role == "MAKER": return - if not nick in self.active_orders: + if nick not in self.active_orders: return - utxos= json.loads(utxolist) #completed population of order/offer object self.active_orders[nick]["cjaddr"] = cjaddr self.active_orders[nick]["changeaddr"] = changeaddr - self.active_orders[nick]["utxos"] = utxos - msg = str(",".join(utxos.keys())) + " " + " ".join( + self.active_orders[nick]["utxos"] = utxolist + msg = str(",".join(utxolist)) + " " + " ".join( [pubkey, cjaddr, changeaddr, pubkeysig]) self.mcc.prepare_privmsg(nick, "ioauth", msg) #In case of *blacklisted (ie already used) commitments, we already @@ -705,11 +697,10 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): @JMTXSigs.responder def on_JM_TX_SIGS(self, nick, sigs): """Signatures that the Maker has produced - are passed here to the daemon as a list and - broadcast one by one. TODO: could shorten this, - have more than one sig per message. - """ - sigs = json.loads(sigs) + are passed here to the daemon as a list and + broadcast one by one. TODO: could shorten this, + have more than one sig per message. + """ for sig in sigs: self.mcc.prepare_privmsg(nick, "sig", sig) return {"accepted": True} @@ -794,19 +785,19 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): @maker_only def on_seen_auth(self, nick, commitment_revelation): """Passes to Maker the !auth message from the Taker, - for processing. This will include validating the PoDLE - commitment revelation against the existing commitment, - which was already stored in active_orders[nick]. - """ - if not nick in self.active_orders: + for processing. This will include validating the PoDLE + commitment revelation against the existing commitment, + which was already stored in active_orders[nick]. + """ + if nick not in self.active_orders: return ao =self.active_orders[nick] #ask the client to validate the commitment and prepare the utxo data d = self.callRemote(JMAuthReceived, nick=nick, - offer=json.dumps(ao["offer"]), + offer=ao["offer"], commitment=ao["commit"], - revelation=json.dumps(commitment_revelation), + revelation=commitment_revelation, amount=ao["amount"], kphex=ao["kp"].hex_pk().decode('ascii')) self.defaultCallbacks(d) @@ -831,20 +822,14 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): self.mcc.pubmsg("!hp2 " + commitment) @maker_only - def on_push_tx(self, nick, txhex): - """Broadcast unquestioningly, except checking - hex format. - """ - try: - dummy = hextobin(txhex) - except: - return - d = self.callRemote(JMTXBroadcast, - txhex=txhex) + def on_push_tx(self, nick, tx): + """Broadcast unquestioningly + """ + d = self.callRemote(JMTXBroadcast, tx=tx) self.defaultCallbacks(d) @maker_only - def on_seen_tx(self, nick, txhex): + def on_seen_tx(self, nick, tx): """Passes the txhex to the Maker for verification and signing. Note the security checks occur in Maker. """ @@ -855,10 +840,7 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): ao = copy.deepcopy(self.active_orders[nick]) del ao["crypto_box"] del ao["kp"] - d = self.callRemote(JMTXReceived, - nick=nick, - txhex=txhex, - offer=json.dumps(ao)) + d = self.callRemote(JMTXReceived, nick=nick, tx=tx, offer=ao) self.defaultCallbacks(d) @taker_only @@ -898,9 +880,7 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): def on_sig(self, nick, sig): """Pass signature through to Taker. """ - d = self.callRemote(JMSigReceived, - nick=nick, - sig=sig) + d = self.callRemote(JMSigReceived, nick=nick, sig=sig) self.defaultCallbacks(d) def on_error(self, msg): @@ -990,12 +970,12 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): self.jm_state = 3 if not accepted: #use ioauth data field to return the list of non-responsive makers - nonresponders = [x for x in self.active_orders.keys() if x not - in self.ioauth_data.keys()] + nonresponders = [x for x in self.active_orders + if x not in self.ioauth_data] ioauth_data = self.ioauth_data if accepted else nonresponders d = self.callRemote(JMFillResponse, - success=accepted, - ioauth_data = json.dumps(ioauth_data)) + success=accepted, + ioauth_data=ioauth_data) if not accepted: #Client simply accepts failure TODO self.defaultCallbacks(d) @@ -1010,7 +990,7 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): either send success + ioauth data if enough makers, else send failure to client. """ - response = True if len(self.ioauth_data.keys()) >= self.minmakers else False + response = True if len(self.ioauth_data) >= self.minmakers else False self.respondToIoauths(response) def checkUtxosAccepted(self, accepted): diff --git a/jmdaemon/jmdaemon/message_channel.py b/jmdaemon/jmdaemon/message_channel.py index 1abdee4..aa786f2 100644 --- a/jmdaemon/jmdaemon/message_channel.py +++ b/jmdaemon/jmdaemon/message_channel.py @@ -1028,21 +1028,21 @@ class MessageChannel(object): elif _chunks[0] == 'tx': b64tx = _chunks[1] try: - txhex = binascii.hexlify(base64.b64decode(b64tx)).decode('ascii') + tx = base64.b64decode(b64tx) except TypeError as e: self.send_error(nick, 'bad base64 tx. ' + repr(e)) return if self.on_seen_tx: - self.on_seen_tx(nick, txhex) + self.on_seen_tx(nick, tx) elif _chunks[0] == 'push': b64tx = _chunks[1] try: - txhex = binascii.hexlify(base64.b64decode(b64tx)).decode('ascii') + tx = base64.b64decode(b64tx) except TypeError as e: self.send_error(nick, 'bad base64 tx. ' + repr(e)) return if self.on_push_tx: - self.on_push_tx(nick, txhex) + self.on_push_tx(nick, tx) except (IndexError, ValueError): # TODO proper error handling log.debug('cj peer error TODO handle') diff --git a/jmdaemon/test/test_daemon_protocol.py b/jmdaemon/test/test_daemon_protocol.py index 4d19413..22a5298 100644 --- a/jmdaemon/test/test_daemon_protocol.py +++ b/jmdaemon/test/test_daemon_protocol.py @@ -19,7 +19,6 @@ from twisted.protocols import amp from twisted.trial import unittest from jmbase.commands import * from msgdata import * -import json import base64 import sys from dummy_mc import DummyMessageChannel @@ -64,7 +63,7 @@ class JMTestClientProtocol(JMBaseProtocol): d = self.callRemote(JMInit, bcsource="dummyblockchain", network="dummynetwork", - irc_configs=json.dumps(irc), + irc_configs=irc, minmakers=2, maker_timeout_sec=3, dust_threshold=27300) @@ -85,7 +84,7 @@ class JMTestClientProtocol(JMBaseProtocol): show_receipt("JMUP") d = self.callRemote(JMSetup, role="TAKER", - offers="{}", + initdata=None, use_fidelity_bond=False) self.defaultCallbacks(d) return {'accepted': True} @@ -104,10 +103,9 @@ class JMTestClientProtocol(JMBaseProtocol): return {'accepted': True} def maketx(self, ioauth_data): - ioauth_data = json.loads(ioauth_data) - nl = list(ioauth_data.keys()) + nl = list(ioauth_data) d = self.callRemote(JMMakeTx, - nick_list= json.dumps(nl), + nick_list=nl, txhex="deadbeef") self.defaultCallbacks(d) @@ -136,7 +134,7 @@ class JMTestClientProtocol(JMBaseProtocol): amount=100, commitment="dummycommitment", revelation="dummyrevelation", - filled_offers=json.dumps(t_chosen_orders)) + filled_offers=t_chosen_orders) self.defaultCallbacks(d) return {'accepted': True} @@ -215,9 +213,9 @@ class JMDaemonTestServerProtocol(JMDaemonServerProtocol): @JMInit.responder def on_JM_INIT(self, bcsource, network, irc_configs, minmakers, maker_timeout_sec, dust_threshold): - self.maker_timeout_sec = int(maker_timeout_sec) + self.maker_timeout_sec = maker_timeout_sec self.dust_threshold = int(dust_threshold) - self.minmakers = int(minmakers) + self.minmakers = minmakers mcs = [DummyMC(None)] self.mcc = MessageChannelCollection(mcs) #The following is a hack to get the counterparties marked seen/active; @@ -240,21 +238,20 @@ class JMDaemonTestServerProtocol(JMDaemonServerProtocol): return {'accepted': True} @JMFill.responder - def on_JM_FILL(self, amount, commitment, revelation, filled_offers): - tmpfo = json.loads(filled_offers) + def on_JM_FILL(self, amount, commitment, revelation, filled_offers): dummypub = "073732a7ca60470f709f23c602b2b8a6b1ba62ee8f3f83a61e5484ab5cbf9c3d" #trigger invalid on_pubkey conditions reactor.callLater(1, self.on_pubkey, "notrealcp", dummypub) - reactor.callLater(2, self.on_pubkey, list(tmpfo.keys())[0], dummypub + "deadbeef") + reactor.callLater(2, self.on_pubkey, list(filled_offers)[0], dummypub + "deadbeef") #trigger invalid on_ioauth condition reactor.callLater(2, self.on_ioauth, "notrealcp", 1, 2, 3, 4, 5) #trigger msg sig verify request operation for a dummy message #currently a pass-through reactor.callLater(1, self.request_signature_verify, "1", "!push abcd abc def", "3", "4", - str(list(tmpfo.keys())[0]), 6, 7, self.mcc.mchannels[0].hostid) + str(list(filled_offers)[0]), 6, 7, self.mcc.mchannels[0].hostid) #send "valid" onpubkey, onioauth messages - for k, v in tmpfo.items(): + for k, v in filled_offers.items(): reactor.callLater(1, self.on_pubkey, k, dummypub) reactor.callLater(2, self.on_ioauth, k, ['a', 'b'], "auth_pub", "cj_addr", "change_addr", "btc_sig") @@ -262,7 +259,7 @@ class JMDaemonTestServerProtocol(JMDaemonServerProtocol): @JMMakeTx.responder def on_JM_MAKE_TX(self, nick_list, txhex): - for n in json.loads(nick_list): + for n in nick_list: reactor.callLater(1, self.on_sig, n, "dummytxsig") return super().on_JM_MAKE_TX(nick_list, txhex) diff --git a/test/ygrunner.py b/test/ygrunner.py index b3a888e..75411dc 100644 --- a/test/ygrunner.py +++ b/test/ygrunner.py @@ -54,12 +54,12 @@ class MaliciousYieldGenerator(YieldGeneratorBasic): jmprint("Counterparty commitment rejected maliciously", "debug") return (False,) return super().on_auth_received(nick, offer, commitment, cr, amount, kphex) - def on_tx_received(self, nick, txhex, offerinfo): + def on_tx_received(self, nick, tx, offerinfo): if self.txmal: if random.randint(1, 100) < self.mfrac: jmprint("Counterparty tx rejected maliciously", "debug") return (False, "malicious tx rejection") - return super().on_tx_received(nick, txhex, offerinfo) + return super().on_tx_received(nick, tx, offerinfo) class DeterministicMaliciousYieldGenerator(YieldGeneratorBasic): """Overrides, randomly chosen persistently, some maker functions @@ -85,11 +85,11 @@ class DeterministicMaliciousYieldGenerator(YieldGeneratorBasic): jmprint("Counterparty commitment rejected maliciously", "debug") return (False,) return super().on_auth_received(nick, offer, commitment, cr, amount, kphex) - def on_tx_received(self, nick, txhex, offerinfo): + def on_tx_received(self, nick, tx, offerinfo): if self.txmal: jmprint("Counterparty tx rejected maliciously", "debug") return (False, "malicious tx rejection") - return super().on_tx_received(nick, txhex, offerinfo) + return super().on_tx_received(nick, tx, offerinfo) @pytest.mark.parametrize( From dfc82ab130157f1f65a05c16c8c4dbe71d14d571 Mon Sep 17 00:00:00 2001 From: undeath Date: Sun, 27 Jun 2021 18:16:19 +0200 Subject: [PATCH 2/4] various whitespace fixes --- jmbase/test/test_commands.py | 30 +++++----- jmclient/jmclient/client_protocol.py | 4 +- jmclient/test/test_client_protocol.py | 38 ++++++------ jmdaemon/jmdaemon/daemon_protocol.py | 83 ++++++++++++++------------- jmdaemon/test/test_daemon_protocol.py | 36 ++++++------ 5 files changed, 96 insertions(+), 95 deletions(-) diff --git a/jmbase/test/test_commands.py b/jmbase/test/test_commands.py index dafda0f..d3d2863 100644 --- a/jmbase/test/test_commands.py +++ b/jmbase/test/test_commands.py @@ -92,26 +92,26 @@ class JMTestServerProtocol(JMBaseProtocol): def on_JM_MAKE_TX(self, nick_list, txhex): show_receipt("JMMAKETX", nick_list, txhex) d = self.callRemote(JMSigReceived, - nick="dummynick", - sig="xxxsig") + nick="dummynick", + sig="xxxsig") self.defaultCallbacks(d) #add dummy calls to check message sign and message verify d2 = self.callRemote(JMRequestMsgSig, - nick="dummynickforsign", - cmd="command1", - msg="msgforsign", - msg_to_be_signed="fullmsgforsign", - hostid="hostid1") + nick="dummynickforsign", + cmd="command1", + msg="msgforsign", + msg_to_be_signed="fullmsgforsign", + hostid="hostid1") self.defaultCallbacks(d2) d3 = self.callRemote(JMRequestMsgSigVerify, - msg="msgforverify", - fullmsg="fullmsgforverify", - sig="xxxsigforverify", - pubkey="pubkey1", - nick="dummynickforverify", - hashlen=4, - max_encoded=5, - hostid="hostid2") + msg="msgforverify", + fullmsg="fullmsgforverify", + sig="xxxsigforverify", + pubkey="pubkey1", + nick="dummynickforverify", + hashlen=4, + max_encoded=5, + hostid="hostid2") self.defaultCallbacks(d3) d4 = self.callRemote(JMTXBroadcast, tx=b"deadbeef") self.defaultCallbacks(d4) diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index dd50184..9969b73 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -443,7 +443,7 @@ class JMMakerClientProtocol(JMClientProtocol): def on_JM_AUTH_RECEIVED(self, nick, offer, commitment, revelation, amount, kphex): retval = self.client.on_auth_received(nick, offer, - commitment, revelation, amount, kphex) + commitment, revelation, amount, kphex) if not retval[0]: jlog.info("Maker refuses to continue on receiving auth.") else: @@ -512,7 +512,7 @@ class JMMakerClientProtocol(JMClientProtocol): return {"accepted": True} def tx_match(self, txd): - for k,v in self.finalized_offers.items(): + for k, v in self.finalized_offers.items(): # Tx considered defined by its output set if v["txd"].vout == txd.vout: offerinfo = v diff --git a/jmclient/test/test_client_protocol.py b/jmclient/test/test_client_protocol.py index 0d4a6ba..cb37a16 100644 --- a/jmclient/test/test_client_protocol.py +++ b/jmclient/test/test_client_protocol.py @@ -199,8 +199,8 @@ class JMTestServerProtocol(JMBaseProtocol): orderbook = ["aaaa" for _ in range(15)] fidelitybonds = ["bbbb" for _ in range(15)] d = self.callRemote(JMOffers, - orderbook=json.dumps(orderbook), - fidelitybonds=json.dumps(fidelitybonds)) + orderbook=json.dumps(orderbook), + fidelitybonds=json.dumps(fidelitybonds)) self.defaultCallbacks(d) return {'accepted': True} @@ -217,16 +217,16 @@ class JMTestServerProtocol(JMBaseProtocol): def on_JM_MAKE_TX(self, nick_list, txhex): show_receipt("JMMAKETX", nick_list, txhex) d = self.callRemote(JMSigReceived, - nick="dummynick", - sig="xxxsig") + nick="dummynick", + sig="xxxsig") self.defaultCallbacks(d) #add dummy calls to check message sign and message verify d2 = self.callRemote(JMRequestMsgSig, - nick="dummynickforsign", - cmd="command1", - msg="msgforsign", - msg_to_be_signed="fullmsgforsign", - hostid="hostid1") + nick="dummynickforsign", + cmd="command1", + msg="msgforsign", + msg_to_be_signed="fullmsgforsign", + hostid="hostid1") self.defaultCallbacks(d2) #To test, this must include a valid ecdsa sig fullmsg = "fullmsgforverify" @@ -234,18 +234,18 @@ class JMTestServerProtocol(JMBaseProtocol): pub = bintohex(bitcoin.privkey_to_pubkey(priv)) sig = bitcoin.ecdsa_sign(fullmsg, priv) d3 = self.callRemote(JMRequestMsgSigVerify, - msg="msgforverify", - fullmsg=fullmsg, - sig=sig, - pubkey=pub, - nick="dummynickforverify", - hashlen=4, - max_encoded=5, - hostid="hostid2") + msg="msgforverify", + fullmsg=fullmsg, + sig=sig, + pubkey=pub, + nick="dummynickforverify", + hashlen=4, + max_encoded=5, + hostid="hostid2") self.defaultCallbacks(d3) d4 = self.callRemote(JMSigReceived, - nick="dummynick", - sig="dummysig") + nick="dummynick", + sig="dummysig") self.defaultCallbacks(d4) return {'accepted': True} diff --git a/jmdaemon/jmdaemon/daemon_protocol.py b/jmdaemon/jmdaemon/daemon_protocol.py index 7ae1868..64d3f1c 100644 --- a/jmdaemon/jmdaemon/daemon_protocol.py +++ b/jmdaemon/jmdaemon/daemon_protocol.py @@ -100,11 +100,11 @@ class BIP78ReceiverResource(JMHTTPResource): def __init__(self, info_callback, shutdown_callback, post_request_handler): """ The POST request handling callback has function signature: - args: (request-body-content-in-bytes,) - returns: (errormsg, errcode, httpcode, response-in-bytes) - If the request was successful, errormsg should be true and response - should be in bytes, to be sent in the return value of render_POST(). - """ + args: (request-body-content-in-bytes,) + returns: (errormsg, errcode, httpcode, response-in-bytes) + If the request was successful, errormsg should be true and response + should be in bytes, to be sent in the return value of render_POST(). + """ self.post_request_handler = post_request_handler super().__init__(info_callback, shutdown_callback) @@ -157,6 +157,7 @@ class BIP78ReceiverResource(JMHTTPResource): self.info_callback("Shutting down, payjoin negotiation failed.") self.shutdown_callback() + class HTTPPassThrough(amp.AMP): """ This class supports passing through requests over HTTPS or over a socks proxy to a remote @@ -204,9 +205,9 @@ class HTTPPassThrough(amp.AMP): def getRequest(self, server, success_callback, url=None, headers=None): """ Make GET request to server server, if response received OK, - passed to success_callback, which must have function signature + passed to success_callback, which must have function signature (response, server). - """ + """ agent, destination_url = self.getAgentDestination(server) if url: destination_url = destination_url + url @@ -228,7 +229,7 @@ class HTTPPassThrough(amp.AMP): def postRequest(self, body, server, success_callback, url=None, params=None, headers=None): """ Pass body of post request as string, will be encoded here. - """ + """ agent, destination_url = self.getAgentDestination(server, params=params) if url: @@ -301,25 +302,25 @@ class BIP78ServerProtocol(HTTPPassThrough): def info_callback(self, msg): """ Informational messages are all passed - to the client. TODO makes sense to log locally - too, in case daemon is isolated?. - """ + to the client. TODO makes sense to log locally + too, in case daemon is isolated?. + """ d = self.callRemote(BIP78InfoMsg, infomsg=msg) self.defaultCallbacks(d) def onion_hostname_callback(self, hostname): """ On successful start of HS, we pass hostname - to client, who can use this to build the full URI. - """ + to client, who can use this to build the full URI. + """ d = self.callRemote(BIP78ReceiverUp, hostname=hostname) self.defaultCallbacks(d) def post_request_handler(self, request, body, params): """ Fired when a sender has sent a POST request - to our hidden service. Argument `body` should be a base64 + to our hidden service. Argument `body` should be a base64 string and params should be a dict. - """ + """ self.post_request = request d = self.callRemote(BIP78ReceiverOriginalPSBT, body=body, params=bdict_sdict_convert(params)) @@ -381,8 +382,8 @@ class SNICKERDaemonServerProtocol(HTTPPassThrough): @SNICKERProposerPostProposals.responder def on_SNICKER_PROPOSER_POST_PROPOSALS(self, proposals, server): """ Receives a list of proposals to be posted to a specific - server. - """ + server. + """ self.postRequest(proposals, server, self.receive_proposals_response) return {"accepted": True} @@ -418,7 +419,7 @@ class SNICKERDaemonServerProtocol(HTTPPassThrough): def receive_proposals_from_server(self, response, server): """ Parses the response from one server. - """ + """ # if the response code is not 200 OK, we must let the client # know that this server is not responding as expected. if int(response.code) != 200: @@ -484,7 +485,7 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): def defaultErrback(self, failure): """TODO better network error handling. - """ + """ failure.trap(ConnectionAborted, ConnectionClosed, ConnectionDone, ConnectionLost) @@ -630,8 +631,8 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): @JMMakeTx.responder def on_JM_MAKE_TX(self, nick_list, txhex): """Taker sends the prepared unsigned transaction - to all the Makers in nick_list - """ + to all the Makers in nick_list + """ if not self.jm_state == 4: log.msg("Make tx was called in wrong state, rejecting") return {'accepted': False} @@ -649,9 +650,9 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): @JMAnnounceOffers.responder def on_JM_ANNOUNCE_OFFERS(self, to_announce, to_cancel, offerlist): """Called by Maker to reset his current offerlist; - Daemon decides what messages (cancel, announce) to - send to the message channel. - """ + Daemon decides what messages (cancel, announce) to + send to the message channel. + """ if self.role != "MAKER": return self.offerlist = offerlist @@ -670,9 +671,9 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): @JMIOAuth.responder def on_JM_IOAUTH(self, nick, utxolist, pubkey, cjaddr, changeaddr, pubkeysig): """Daemon constructs full !ioauth message to be sent on message - channel based on data from Maker. Relevant data (utxos, addresses) - are stored in the active_orders dict keyed by the nick of the Taker. - """ + channel based on data from Maker. Relevant data (utxos, addresses) + are stored in the active_orders dict keyed by the nick of the Taker. + """ if not self.role == "MAKER": return if nick not in self.active_orders: @@ -791,7 +792,7 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): """ if nick not in self.active_orders: return - ao =self.active_orders[nick] + ao = self.active_orders[nick] #ask the client to validate the commitment and prepare the utxo data d = self.callRemote(JMAuthReceived, nick=nick, @@ -805,8 +806,8 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): @maker_only def on_commitment_seen(self, nick, commitment): """Triggered when we see a commitment for blacklisting - appear in the public pit channel. - """ + appear in the public pit channel. + """ #just add if necessary, ignore return value. check_utxo_blacklist(commitment, persist=True) log.msg("Received commitment broadcast by other maker: " + str( @@ -815,10 +816,10 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): @maker_only def on_commitment_transferred(self, nick, commitment): """Triggered when a privmsg is received from another maker - with a commitment to announce in public (obfuscation of source). + with a commitment to announce in public (obfuscation of source). We simply post it in public (not affected by whether we ourselves are *accepting* commitment broadcasts. - """ + """ self.mcc.pubmsg("!hp2 " + commitment) @maker_only @@ -831,8 +832,8 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): @maker_only def on_seen_tx(self, nick, tx): """Passes the txhex to the Maker for verification - and signing. Note the security checks occur in Maker. - """ + and signing. Note the security checks occur in Maker. + """ if nick not in self.active_orders: return #we send a copy of the entire "active_orders" entry except the cryptobox, @@ -900,11 +901,11 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): """ with self.sig_lock: d = self.callRemote(JMRequestMsgSig, - nick=str(nick), - cmd=str(cmd), - msg=str(msg), - msg_to_be_signed=str(msg_to_be_signed), - hostid=str(hostid)) + nick=str(nick), + cmd=str(cmd), + msg=str(msg), + msg_to_be_signed=str(msg_to_be_signed), + hostid=str(hostid)) self.defaultCallbacks(d) def request_signature_verify(self, msg, fullmsg, sig, pubkey, nick, hashlen, @@ -939,8 +940,8 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): def transfer_commitment(self, commit): """Send this commitment via privmsg to one (random) - other maker. - """ + other maker. + """ crow = self.db.execute( 'SELECT DISTINCT counterparty FROM orderbook ORDER BY ' + 'RANDOM() LIMIT 1;' diff --git a/jmdaemon/test/test_daemon_protocol.py b/jmdaemon/test/test_daemon_protocol.py index 22a5298..059cb4a 100644 --- a/jmdaemon/test/test_daemon_protocol.py +++ b/jmdaemon/test/test_daemon_protocol.py @@ -101,7 +101,7 @@ class JMTestClientProtocol(JMBaseProtocol): show_receipt("JMFILLRESPONSE", success, ioauth_data) reactor.callLater(1, self.maketx, ioauth_data) return {'accepted': True} - + def maketx(self, ioauth_data): nl = list(ioauth_data) d = self.callRemote(JMMakeTx, @@ -118,18 +118,18 @@ class JMTestClientProtocol(JMBaseProtocol): nick = str(list(t_chosen_orders.keys())[0]) b64tx = base64.b64encode(b"deadbeef").decode('ascii') d1 = self.callRemote(JMMsgSignatureVerify, - verif_result=True, - nick=nick, - fullmsg="!push " + b64tx + " abc def", - hostid="dummy") + verif_result=True, + nick=nick, + fullmsg="!push " + b64tx + " abc def", + hostid="dummy") self.defaultCallbacks(d1) #unverified d2 = self.callRemote(JMMsgSignatureVerify, - verif_result=False, - nick=nick, - fullmsg="!push " + b64tx + " abc def", - hostid="dummy") - self.defaultCallbacks(d2) + verif_result=False, + nick=nick, + fullmsg="!push " + b64tx + " abc def", + hostid="dummy") + self.defaultCallbacks(d2) d = self.callRemote(JMFill, amount=100, commitment="dummycommitment", @@ -173,7 +173,7 @@ class JMTestClientProtocol(JMBaseProtocol): class JMTestClientProtocolFactory(protocol.ClientFactory): protocol = JMTestClientProtocol - + def show_receipt(name, *args): tmsg("Received msgtype: " + name + ", args: " + ",".join([str(x) for x in args])) @@ -209,7 +209,7 @@ class JMDaemonTestServerProtocol(JMDaemonServerProtocol): o["minsize"], o["maxsize"], o["txfee"], o["cjfee"]) return super().on_JM_REQUEST_OFFERS() - + @JMInit.responder def on_JM_INIT(self, bcsource, network, irc_configs, minmakers, maker_timeout_sec, dust_threshold): @@ -226,14 +226,14 @@ class JMDaemonTestServerProtocol(JMDaemonServerProtocol): OrderbookWatch.set_msgchan(self, self.mcc) #register taker-specific msgchan callbacks here self.mcc.register_taker_callbacks(self.on_error, self.on_pubkey, - self.on_ioauth, self.on_sig) + self.on_ioauth, self.on_sig) self.mcc.set_daemon(self) self.restart_mc_required = True d = self.callRemote(JMInitProto, - nick_hash_length=NICK_HASH_LENGTH, - nick_max_encoded=NICK_MAX_ENCODED, - joinmarket_nick_header=JOINMARKET_NICK_HEADER, - joinmarket_version=JM_VERSION) + nick_hash_length=NICK_HASH_LENGTH, + nick_max_encoded=NICK_MAX_ENCODED, + joinmarket_nick_header=JOINMARKET_NICK_HEADER, + joinmarket_version=JM_VERSION) self.defaultCallbacks(d) return {'accepted': True} @@ -267,7 +267,7 @@ class JMDaemonTestServerProtocol(JMDaemonServerProtocol): class JMDaemonTestServerProtocolFactory(ServerFactory): protocol = JMDaemonTestServerProtocol - + def buildProtocol(self, addr): return JMDaemonTestServerProtocol(self) From e082c3c45135620c1f84703acdbe35abe728ed70 Mon Sep 17 00:00:00 2001 From: undeath Date: Sun, 27 Jun 2021 18:21:32 +0200 Subject: [PATCH 3/4] remove unneeded hex encoding/decoding from sent_tx + push_tx --- jmbase/jmbase/commands.py | 4 ++-- jmbase/test/test_commands.py | 6 +++--- jmclient/jmclient/client_protocol.py | 17 ++++++++--------- jmclient/jmclient/taker.py | 4 ++-- jmclient/test/test_client_protocol.py | 7 ++++--- jmclient/test/test_coinjoin.py | 6 +++--- jmdaemon/jmdaemon/daemon_protocol.py | 8 ++++---- jmdaemon/jmdaemon/message_channel.py | 17 ++++++++--------- jmdaemon/test/test_daemon_protocol.py | 6 +++--- jmdaemon/test/test_message_channel.py | 8 ++++---- 10 files changed, 41 insertions(+), 42 deletions(-) diff --git a/jmbase/jmbase/commands.py b/jmbase/jmbase/commands.py index 2eed32e..2d8eced 100644 --- a/jmbase/jmbase/commands.py +++ b/jmbase/jmbase/commands.py @@ -91,14 +91,14 @@ class JMMakeTx(JMCommand): to a set of counterparties """ arguments = [(b'nick_list', ListOf(Unicode())), - (b'txhex', Unicode())] + (b'tx', String())] class JMPushTx(JMCommand): """Pass a raw hex transaction to a specific counterparty (maker) for pushing (anonymity feature in JM) """ arguments = [(b'nick', Unicode()), - (b'txhex', Unicode())] + (b'tx', String())] """MAKER specific commands """ diff --git a/jmbase/test/test_commands.py b/jmbase/test/test_commands.py index d3d2863..83a5423 100644 --- a/jmbase/test/test_commands.py +++ b/jmbase/test/test_commands.py @@ -89,8 +89,8 @@ class JMTestServerProtocol(JMBaseProtocol): return {'accepted': True} @JMMakeTx.responder - def on_JM_MAKE_TX(self, nick_list, txhex): - show_receipt("JMMAKETX", nick_list, txhex) + def on_JM_MAKE_TX(self, nick_list, tx): + show_receipt("JMMAKETX", nick_list, tx) d = self.callRemote(JMSigReceived, nick="dummynick", sig="xxxsig") @@ -175,7 +175,7 @@ class JMTestClientProtocol(JMBaseProtocol): show_receipt("JMFILLRESPONSE", success, ioauth_data) d = self.callRemote(JMMakeTx, nick_list=['nick1', 'nick2', 'nick3'], - txhex="deadbeef") + tx=b"deadbeef") self.defaultCallbacks(d) return {'accepted': True} diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index 9969b73..10062b9 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -360,10 +360,10 @@ class JMClientProtocol(BaseClientProtocol): self.defaultCallbacks(d) return {'accepted': True} - def make_tx(self, nick_list, txhex): + def make_tx(self, nick_list, tx): d = self.callRemote(commands.JMMakeTx, nick_list=nick_list, - txhex=txhex) + tx=tx) self.defaultCallbacks(d) class JMMakerClientProtocol(JMClientProtocol): @@ -677,8 +677,8 @@ class JMTakerClientProtocol(JMClientProtocol): self.client.on_finished_callback(False, False, 0.0) return {'accepted': False} else: - nick_list, txhex = retval[1:] - reactor.callLater(0, self.make_tx, nick_list, txhex) + nick_list, tx = retval[1:] + reactor.callLater(0, self.make_tx, nick_list, tx) return {'accepted': True} @commands.JMOffers.responder @@ -716,17 +716,16 @@ class JMTakerClientProtocol(JMClientProtocol): def on_JM_SIG_RECEIVED(self, nick, sig): retval = self.client.on_sig(nick, sig) if retval: - nick_to_use, txhex = retval - self.push_tx(nick_to_use, txhex) + nick_to_use, tx = retval + self.push_tx(nick_to_use, tx) return {'accepted': True} def get_offers(self): d = self.callRemote(commands.JMRequestOffers) self.defaultCallbacks(d) - def push_tx(self, nick_to_push, txhex_to_push): - d = self.callRemote(commands.JMPushTx, nick=str(nick_to_push), - txhex=str(txhex_to_push)) + def push_tx(self, nick_to_push, tx): + d = self.callRemote(commands.JMPushTx, nick=str(nick_to_push), tx=tx) self.defaultCallbacks(d) class SNICKERClientProtocolFactory(protocol.ClientFactory): diff --git a/jmclient/jmclient/taker.py b/jmclient/jmclient/taker.py index a4eb2df..67af570 100644 --- a/jmclient/jmclient/taker.py +++ b/jmclient/jmclient/taker.py @@ -524,7 +524,7 @@ class Taker(object): self.taker_info_callback("INFO", "Built tx, sending to counterparties.") return (True, list(self.maker_utxo_data.keys()), - bintohex(self.latest_tx.serialize())) + self.latest_tx.serialize()) def _verify_ioauth_data(self, ioauth_data): verified_data = [] @@ -943,7 +943,7 @@ class Taker(object): self.on_finished_callback(False, fromtx=True) else: if nick_to_use: - return (nick_to_use, bintohex(self.latest_tx.serialize())) + return (nick_to_use, self.latest_tx.serialize()) #if push was not successful, return None def self_sign_and_push(self): diff --git a/jmclient/test/test_client_protocol.py b/jmclient/test/test_client_protocol.py index cb37a16..f77f446 100644 --- a/jmclient/test/test_client_protocol.py +++ b/jmclient/test/test_client_protocol.py @@ -65,7 +65,8 @@ class DummyTaker(Taker): if self.failutxos: return (False, "dummyreason") else: - return (True, [x*64 + ":01" for x in ["a", "b", "c"]], t_raw_signed_tx) + return (True, [x*64 + ":01" for x in ["a", "b", "c"]], + base64.b16decode(t_raw_signed_tx, casefold=True)) def on_sig(self, nick, sigb64): @@ -214,8 +215,8 @@ class JMTestServerProtocol(JMBaseProtocol): return {'accepted': True} @JMMakeTx.responder - def on_JM_MAKE_TX(self, nick_list, txhex): - show_receipt("JMMAKETX", nick_list, txhex) + def on_JM_MAKE_TX(self, nick_list, tx): + show_receipt("JMMAKETX", nick_list, tx) d = self.callRemote(JMSigReceived, nick="dummynick", sig="xxxsig") diff --git a/jmclient/test/test_coinjoin.py b/jmclient/test/test_coinjoin.py index 431699f..516c68b 100644 --- a/jmclient/test/test_coinjoin.py +++ b/jmclient/test/test_coinjoin.py @@ -9,7 +9,7 @@ import pytest import copy from twisted.internet import reactor -from jmbase import get_log, hextobin +from jmbase import get_log from jmclient import load_test_config, jm_single,\ YieldGeneratorBasic, Taker, LegacyWallet, SegwitLegacyWallet, SegwitWallet,\ NO_ROUNDING @@ -206,7 +206,7 @@ def test_coinjoin_mixdepth_wrap_taker(monkeypatch, tmpdir, setup_cj): taker_final_result = do_tx_signing(taker, makers, active_orders, txdata) assert taker_final_result is not False - tx = btc.CMutableTransaction.deserialize(hextobin(txdata[2])) + tx = btc.CMutableTransaction.deserialize(txdata[2]) wallet_service = wallet_services[-1] # TODO change for new tx monitoring: @@ -261,7 +261,7 @@ def test_coinjoin_mixdepth_wrap_maker(monkeypatch, tmpdir, setup_cj): taker_final_result = do_tx_signing(taker, makers, active_orders, txdata) assert taker_final_result is not False - tx = btc.CMutableTransaction.deserialize(hextobin(txdata[2])) + tx = btc.CMutableTransaction.deserialize(txdata[2]) for i in range(MAKER_NUM): wallet_service = wallet_services[i] diff --git a/jmdaemon/jmdaemon/daemon_protocol.py b/jmdaemon/jmdaemon/daemon_protocol.py index 64d3f1c..2430cd3 100644 --- a/jmdaemon/jmdaemon/daemon_protocol.py +++ b/jmdaemon/jmdaemon/daemon_protocol.py @@ -629,19 +629,19 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): return {'accepted': True} @JMMakeTx.responder - def on_JM_MAKE_TX(self, nick_list, txhex): + def on_JM_MAKE_TX(self, nick_list, tx): """Taker sends the prepared unsigned transaction to all the Makers in nick_list """ if not self.jm_state == 4: log.msg("Make tx was called in wrong state, rejecting") return {'accepted': False} - self.mcc.send_tx(nick_list, txhex) + self.mcc.send_tx(nick_list, tx) return {'accepted': True} @JMPushTx.responder - def on_JM_PushTx(self, nick, txhex): - self.mcc.push_tx(nick, txhex) + def on_JM_PushTx(self, nick, tx): + self.mcc.push_tx(nick, tx) return {'accepted': True} """Maker specific responders diff --git a/jmdaemon/jmdaemon/message_channel.py b/jmdaemon/jmdaemon/message_channel.py index aa786f2..277abc0 100644 --- a/jmdaemon/jmdaemon/message_channel.py +++ b/jmdaemon/jmdaemon/message_channel.py @@ -1,7 +1,6 @@ #! /usr/bin/env python import abc import base64 -import binascii import threading from twisted.internet import reactor from jmdaemon import encrypt_encode, decode_decrypt, COMMAND_PREFIX,\ @@ -351,14 +350,14 @@ class MessageChannelCollection(object): self.prepare_privmsg(nick, "error", errormsg) @check_privmsg - def push_tx(self, nick, txhex): + def push_tx(self, nick, tx): #TODO supporting sending to arbitrary nicks #adds quite a bit of complexity, not supported #initially; will fail if nick is not part of TX - txb64 = base64.b64encode(binascii.unhexlify(txhex)).decode('ascii') + txb64 = base64.b64encode(tx).decode('ascii') self.prepare_privmsg(nick, "push", txb64) - def send_tx(self, nick_list, txhex): + def send_tx(self, nick_list, tx): """Push out the transaction to nicks in groups by their message channel. """ @@ -376,10 +375,10 @@ class MessageChannelCollection(object): else: tx_nick_sets[self.active_channels[nick]].append(nick) for mc, nl in tx_nick_sets.items(): - self.prepare_send_tx(mc, nl, txhex) + self.prepare_send_tx(mc, nl, tx) - def prepare_send_tx(self, mc, nick_list, txhex): - txb64 = base64.b64encode(binascii.unhexlify(txhex)).decode('ascii') + def prepare_send_tx(self, mc, nick_list, tx): + txb64 = base64.b64encode(tx).decode('ascii') for nick in nick_list: self.prepare_privmsg(nick, "tx", txb64, mc=mc) @@ -853,10 +852,10 @@ class MessageChannel(object): msg += ' ' + commitment self.privmsg(c, 'fill', msg) - def push_tx(self, nick, txhex): + def push_tx(self, nick, tx): #Note: not currently used; will require prepare_privmsg call so #not in this class (see send_error) - txb64 = base64.b64encode(binascii.unhexlify(txhex)).decode('ascii') + txb64 = base64.b64encode(tx).decode('ascii') self.privmsg(nick, 'push', txb64) def send_error(self, nick, errormsg): diff --git a/jmdaemon/test/test_daemon_protocol.py b/jmdaemon/test/test_daemon_protocol.py index 059cb4a..002e517 100644 --- a/jmdaemon/test/test_daemon_protocol.py +++ b/jmdaemon/test/test_daemon_protocol.py @@ -106,7 +106,7 @@ class JMTestClientProtocol(JMBaseProtocol): nl = list(ioauth_data) d = self.callRemote(JMMakeTx, nick_list=nl, - txhex="deadbeef") + tx=b"deadbeef") self.defaultCallbacks(d) @JMOffers.responder @@ -258,10 +258,10 @@ class JMDaemonTestServerProtocol(JMDaemonServerProtocol): return super().on_JM_FILL(amount, commitment, revelation, filled_offers) @JMMakeTx.responder - def on_JM_MAKE_TX(self, nick_list, txhex): + def on_JM_MAKE_TX(self, nick_list, tx): for n in nick_list: reactor.callLater(1, self.on_sig, n, "dummytxsig") - return super().on_JM_MAKE_TX(nick_list, txhex) + return super().on_JM_MAKE_TX(nick_list, tx) diff --git a/jmdaemon/test/test_message_channel.py b/jmdaemon/test/test_message_channel.py index 684128e..a9a8bfb 100644 --- a/jmdaemon/test/test_message_channel.py +++ b/jmdaemon/test/test_message_channel.py @@ -160,7 +160,7 @@ def test_setup_mc(): mcc.privmsg(cp1+"XXX", "fill", "0") #trigger check_privmsg decorator mcc.send_error(cp1, "errormsg") - mcc.push_tx(cp1, "deadbeef") + mcc.push_tx(cp1, b"deadbeef") #kill the chan on which the cp is marked active; #note dummychannel has no actual shutdown (call it anyway), #so change its status manually. @@ -215,13 +215,13 @@ def test_setup_mc(): mcc.fill_orders(new_offers, 1000, "dummypubkey", "dummycommit") #now send a dummy transaction to this same set. #first fails with no crypto box. - mcc.send_tx(cps, "deadbeef") + mcc.send_tx(cps, b"deadbeef") #Now initialize the boxes for c in cps: dummydaemon.crypto_boxes[c] = ["a", DummyBox()] - mcc.send_tx(cps, "deadbeef") + mcc.send_tx(cps, b"deadbeef") #try to send the transaction to a wrong cp: - mcc.send_tx(["notrealcp"], "deadbeef") + mcc.send_tx(["notrealcp"], b"deadbeef") #At this stage, dmcs0,2 should be "up" and 1 should be "down": assert mcc.mc_status[dmcs[0]] == 1 From 2fdebb88c28a9a183269d1eb00ee81b17ddc1897 Mon Sep 17 00:00:00 2001 From: undeath Date: Mon, 2 Aug 2021 21:05:59 +0200 Subject: [PATCH 4/4] do not call reactor.stop() in test_commands.py --- jmbase/test/test_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jmbase/test/test_commands.py b/jmbase/test/test_commands.py index 83a5423..3971670 100644 --- a/jmbase/test/test_commands.py +++ b/jmbase/test/test_commands.py @@ -21,12 +21,12 @@ class JMBaseProtocol(amp.AMP): is considered criticial. """ if 'accepted' not in response or not response['accepted']: - reactor.stop() + raise Exception(response) def defaultErrback(self, failure): failure.trap(ConnectionAborted, ConnectionClosed, ConnectionDone, ConnectionLost, UnknownRemoteError) - reactor.stop() + raise Exception(failure) def defaultCallbacks(self, d): d.addCallback(self.checkClientResponse)