From 7dcd3f3b5a165f2410a8d2b0ddb6e4e7deeb4999 Mon Sep 17 00:00:00 2001 From: undeath Date: Sun, 27 Jun 2021 18:14:07 +0200 Subject: [PATCH] move client/server data encoding to twisted --- jmbase/jmbase/arguments.py | 10 +++ jmbase/jmbase/commands.py | 46 +++++----- jmbase/test/test_commands.py | 22 ++--- jmclient/jmclient/client_protocol.py | 61 ++++++-------- jmclient/jmclient/maker.py | 6 +- jmclient/test/test_client_protocol.py | 20 +++-- jmdaemon/jmdaemon/daemon_protocol.py | 116 +++++++++++--------------- jmdaemon/jmdaemon/message_channel.py | 8 +- jmdaemon/test/test_daemon_protocol.py | 27 +++--- test/ygrunner.py | 8 +- 10 files changed, 153 insertions(+), 171 deletions(-) create mode 100644 jmbase/jmbase/arguments.py diff --git a/jmbase/jmbase/arguments.py b/jmbase/jmbase/arguments.py new file mode 100644 index 0000000..e6f68b5 --- /dev/null +++ b/jmbase/jmbase/arguments.py @@ -0,0 +1,10 @@ +import json +from twisted.protocols.amp import String + + +class JsonEncodable(String): + def toString(self, inObject): + return super().toString(json.dumps(inObject).encode('ascii')) + + def fromString(self, inString): + return super().fromString(json.loads(inString)) diff --git a/jmbase/jmbase/commands.py b/jmbase/jmbase/commands.py index 935d30e..2eed32e 100644 --- a/jmbase/jmbase/commands.py +++ b/jmbase/jmbase/commands.py @@ -3,8 +3,10 @@ Commands defining client-server (daemon) messaging protocol (*not* Joinmarket p2p protocol). Used for AMP asynchronous messages. """ -from twisted.protocols.amp import Boolean, Command, Integer, Unicode +from twisted.protocols.amp import Boolean, Command, Integer, Unicode, ListOf,\ + String from .bigstring import BigUnicode +from .arguments import JsonEncodable class DaemonNotReady(Exception): @@ -29,7 +31,7 @@ class JMInit(JMCommand): """ arguments = [(b'bcsource', Unicode()), (b'network', Unicode()), - (b'irc_configs', Unicode()), + (b'irc_configs', JsonEncodable()), (b'minmakers', Integer()), (b'maker_timeout_sec', Integer()), (b'dust_threshold', Integer())] @@ -47,7 +49,7 @@ class JMSetup(JMCommand): role, passes initial offers for announcement (for TAKER, this data is "none") """ arguments = [(b'role', Unicode()), - (b'offers', Unicode()), + (b'initdata', JsonEncodable()), (b'use_fidelity_bond', Boolean())] class JMMsgSignature(JMCommand): @@ -82,13 +84,13 @@ class JMFill(JMCommand): arguments = [(b'amount', Integer()), (b'commitment', Unicode()), (b'revelation', Unicode()), - (b'filled_offers', Unicode())] + (b'filled_offers', JsonEncodable())] class JMMakeTx(JMCommand): """Send a hex encoded raw bitcoin transaction to a set of counterparties """ - arguments = [(b'nick_list', Unicode()), + arguments = [(b'nick_list', ListOf(Unicode())), (b'txhex', Unicode())] class JMPushTx(JMCommand): @@ -106,9 +108,9 @@ class JMAnnounceOffers(JMCommand): to the daemon, along with new announcement and cancellation lists (deltas). """ - arguments = [(b'to_announce', Unicode()), - (b'to_cancel', Unicode()), - (b'offerlist', Unicode())] + arguments = [(b'to_announce', JsonEncodable()), + (b'to_cancel', JsonEncodable()), + (b'offerlist', JsonEncodable())] class JMFidelityBondProof(JMCommand): """Send requested fidelity bond proof message""" @@ -120,7 +122,7 @@ class JMIOAuth(JMCommand): verifying Taker's auth message """ arguments = [(b'nick', Unicode()), - (b'utxolist', Unicode()), + (b'utxolist', JsonEncodable()), (b'pubkey', Unicode()), (b'cjaddr', Unicode()), (b'changeaddr', Unicode()), @@ -131,7 +133,7 @@ class JMTXSigs(JMCommand): sent by TAKER """ arguments = [(b'nick', Unicode()), - (b'sigs', Unicode())] + (b'sigs', ListOf(Unicode()))] """COMMANDS FROM DAEMON TO CLIENT ================================= @@ -197,7 +199,7 @@ class JMFillResponse(JMCommand): """Returns ioauth data from MAKER if successful. """ arguments = [(b'success', Boolean()), - (b'ioauth_data', Unicode())] + (b'ioauth_data', JsonEncodable())] class JMSigReceived(JMCommand): """Returns an individual bitcoin transaction signature @@ -221,9 +223,9 @@ class JMAuthReceived(JMCommand): before setting up encryption and continuing. """ arguments = [(b'nick', Unicode()), - (b'offer', Unicode()), + (b'offer', JsonEncodable()), (b'commitment', Unicode()), - (b'revelation', Unicode()), + (b'revelation', JsonEncodable()), (b'amount', Integer()), (b'kphex', Unicode())] @@ -232,8 +234,8 @@ class JMTXReceived(JMCommand): by TAKER, along with offerdata to verify fees. """ arguments = [(b'nick', Unicode()), - (b'txhex', Unicode()), - (b'offer', Unicode())] + (b'tx', String()), + (b'offer', JsonEncodable())] class JMTXBroadcast(JMCommand): """ Accept a bitcoin transaction @@ -241,7 +243,7 @@ class JMTXBroadcast(JMCommand): and relay it to the client for network broadcast. """ - arguments = [(b'txhex', Unicode())] + arguments = [(b'tx', String())] """SNICKER related commands. """ @@ -251,12 +253,12 @@ class SNICKERReceiverInit(JMCommand): See documentation of `netconfig` in jmdaemon.HTTPPassThrough.on_INIT """ - arguments = [(b'netconfig', Unicode())] + arguments = [(b'netconfig', JsonEncodable())] class SNICKERProposerInit(JMCommand): """ As for receiver. """ - arguments = [(b'netconfig', Unicode())] + arguments = [(b'netconfig', JsonEncodable())] class SNICKERReceiverUp(JMCommand): arguments = [] @@ -307,7 +309,7 @@ class BIP78SenderInit(JMCommand): See documentation of `netconfig` in jmdaemon.HTTPPassThrough.on_INIT """ - arguments = [(b'netconfig', Unicode())] + arguments = [(b'netconfig', JsonEncodable())] class BIP78SenderUp(JMCommand): arguments = [] @@ -319,7 +321,7 @@ class BIP78SenderOriginalPSBT(JMCommand): to be sent as an http request to the receiver. """ arguments = [(b'body', BigUnicode()), - (b'params', Unicode())] + (b'params', JsonEncodable())] class BIP78SenderReceiveProposal(JMCommand): """ Sends the payjoin proposal PSBT, received @@ -341,7 +343,7 @@ class BIP78SenderReceiveError(JMCommand): class BIP78ReceiverInit(JMCommand): """ Initialization data for a BIP78 hidden service. """ - arguments = [(b'netconfig', Unicode())] + arguments = [(b'netconfig', JsonEncodable())] class BIP78ReceiverUp(JMCommand): """ Returns onion hostname to client when @@ -356,7 +358,7 @@ class BIP78ReceiverOriginalPSBT(JMCommand): parameters in the url, from the daemon to the client. """ arguments = [(b'body', BigUnicode()), - (b'params', Unicode())] + (b'params', JsonEncodable())] class BIP78ReceiverSendProposal(JMCommand): """ Receives a payjoin proposal PSBT from diff --git a/jmbase/test/test_commands.py b/jmbase/test/test_commands.py index 419a61f..dafda0f 100644 --- a/jmbase/test/test_commands.py +++ b/jmbase/test/test_commands.py @@ -63,8 +63,8 @@ class JMTestServerProtocol(JMBaseProtocol): return {'accepted': True} @JMSetup.responder - def on_JM_SETUP(self, role, offers, use_fidelity_bond): - show_receipt("JMSETUP", role, offers, use_fidelity_bond) + def on_JM_SETUP(self, role, initdata, use_fidelity_bond): + show_receipt("JMSETUP", role, initdata, use_fidelity_bond) d = self.callRemote(JMSetupDone) self.defaultCallbacks(d) return {'accepted': True} @@ -84,8 +84,8 @@ class JMTestServerProtocol(JMBaseProtocol): def on_JM_FILL(self, amount, commitment, revelation, filled_offers): show_receipt("JMFILL", amount, commitment, revelation, filled_offers) d = self.callRemote(JMFillResponse, - success=True, - ioauth_data = json.dumps(['dummy', 'list'])) + success=True, + ioauth_data=['dummy', 'list']) return {'accepted': True} @JMMakeTx.responder @@ -113,7 +113,7 @@ class JMTestServerProtocol(JMBaseProtocol): max_encoded=5, hostid="hostid2") self.defaultCallbacks(d3) - d4 = self.callRemote(JMTXBroadcast, txhex="deadbeef") + d4 = self.callRemote(JMTXBroadcast, tx=b"deadbeef") self.defaultCallbacks(d4) return {'accepted': True} @@ -137,7 +137,7 @@ class JMTestClientProtocol(JMBaseProtocol): d = self.callRemote(JMInit, bcsource="dummyblockchain", network="dummynetwork", - irc_configs=json.dumps(['dummy', 'irc', 'config']), + irc_configs=['dummy', 'irc', 'config'], minmakers=7, maker_timeout_sec=8, dust_threshold=1500) @@ -158,7 +158,7 @@ class JMTestClientProtocol(JMBaseProtocol): show_receipt("JMUP") d = self.callRemote(JMSetup, role="TAKER", - offers="{}", + initdata=None, use_fidelity_bond=False) self.defaultCallbacks(d) return {'accepted': True} @@ -174,7 +174,7 @@ class JMTestClientProtocol(JMBaseProtocol): def on_JM_FILL_RESPONSE(self, success, ioauth_data): show_receipt("JMFILLRESPONSE", success, ioauth_data) d = self.callRemote(JMMakeTx, - nick_list= json.dumps(['nick1', 'nick2', 'nick3']), + nick_list=['nick1', 'nick2', 'nick3'], txhex="deadbeef") self.defaultCallbacks(d) return {'accepted': True} @@ -186,7 +186,7 @@ class JMTestClientProtocol(JMBaseProtocol): amount=100, commitment="dummycommitment", revelation="dummyrevelation", - filled_offers=json.dumps(['list', 'of', 'filled', 'offers'])) + filled_offers=['list', 'of', 'filled', 'offers']) self.defaultCallbacks(d) return {'accepted': True} @@ -222,8 +222,8 @@ class JMTestClientProtocol(JMBaseProtocol): return {'accepted': True} @JMTXBroadcast.responder - def on_JM_TX_BROADCAST(self, txhex): - show_receipt("JMTXBROADCAST", txhex) + def on_JM_TX_BROADCAST(self, tx): + show_receipt("JMTXBROADCAST", tx) return {"accepted": True} class JMTestClientProtocolFactory(protocol.ClientFactory): diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index d19b32b..dd50184 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -73,13 +73,13 @@ class BIP78ClientProtocol(BaseClientProtocol): "tls_whitelist": ",".join(self.tls_whitelist), "servers": [self.manager.server]} d = self.callRemote(commands.BIP78SenderInit, - netconfig=json.dumps(netconfig)) + netconfig=netconfig) else: netconfig = {"port": 80, "tor_control_host": jcg("PAYJOIN", "tor_control_host"), "tor_control_port": jcg("PAYJOIN", "tor_control_port")} d = self.callRemote(commands.BIP78ReceiverInit, - netconfig=json.dumps(netconfig)) + netconfig=netconfig) self.defaultCallbacks(d) @commands.BIP78ReceiverUp.responder @@ -89,7 +89,6 @@ class BIP78ClientProtocol(BaseClientProtocol): @commands.BIP78ReceiverOriginalPSBT.responder def on_BIP78_RECEIVER_ORIGINAL_PSBT(self, body, params): - params = json.loads(params) # TODO: we don't need binary key/vals client side, but will have to edit # PayjoinConverter for that: retval = self.success_callback(body.encode("utf-8"), bdict_sdict_convert( @@ -121,7 +120,7 @@ class BIP78ClientProtocol(BaseClientProtocol): def on_BIP78_SENDER_UP(self): d = self.callRemote(commands.BIP78SenderOriginalPSBT, body=self.manager.initial_psbt.to_base64(), - params=json.dumps(self.params)) + params=self.params) self.defaultCallbacks(d) return {"accepted": True} @@ -168,10 +167,10 @@ class SNICKERClientProtocol(BaseClientProtocol): if isinstance(self.client, SNICKERReceiver): d = self.callRemote(commands.SNICKERReceiverInit, - netconfig=json.dumps(netconfig)) + netconfig=netconfig) else: d = self.callRemote(commands.SNICKERProposerInit, - netconfig=json.dumps(netconfig)) + netconfig=netconfig) self.defaultCallbacks(d) def shutdown(self): @@ -363,7 +362,7 @@ class JMClientProtocol(BaseClientProtocol): def make_tx(self, nick_list, txhex): d = self.callRemote(commands.JMMakeTx, - nick_list= json.dumps(nick_list), + nick_list=nick_list, txhex=txhex) self.defaultCallbacks(d) @@ -394,7 +393,7 @@ class JMMakerClientProtocol(JMClientProtocol): self.offers_ready_loop.stop() d = self.callRemote(commands.JMSetup, role="MAKER", - offers=json.dumps(self.client.offerlist), + initdata=self.client.offerlist, use_fidelity_bond=(self.client.fidelity_bond is not None)) self.defaultCallbacks(d) @@ -423,7 +422,7 @@ class JMMakerClientProtocol(JMClientProtocol): d = self.callRemote(commands.JMInit, bcsource=blockchain_source, network=network, - irc_configs=json.dumps(irc_configs), + irc_configs=irc_configs, minmakers=minmakers, maker_timeout_sec=maker_timeout_sec, dust_threshold=jm_single().DUST_THRESHOLD) @@ -443,8 +442,6 @@ class JMMakerClientProtocol(JMClientProtocol): @commands.JMAuthReceived.responder def on_JM_AUTH_RECEIVED(self, nick, offer, commitment, revelation, amount, kphex): - offer = json.loads(offer) - revelation = json.loads(revelation) retval = self.client.on_auth_received(nick, offer, commitment, revelation, amount, kphex) if not retval[0]: @@ -461,7 +458,7 @@ class JMMakerClientProtocol(JMClientProtocol): auth_pub_hex = bintohex(auth_pub) d = self.callRemote(commands.JMIOAuth, nick=nick, - utxolist=json.dumps(utxos_strkeyed), + utxolist=utxos_strkeyed, pubkey=auth_pub_hex, cjaddr=cj_addr, changeaddr=change_addr, @@ -470,17 +467,15 @@ class JMMakerClientProtocol(JMClientProtocol): return {"accepted": True} @commands.JMTXReceived.responder - def on_JM_TX_RECEIVED(self, nick, txhex, offer): - offer = json.loads(offer) - retval = self.client.on_tx_received(nick, txhex, offer) + def on_JM_TX_RECEIVED(self, nick, tx, offer): + retval = self.client.on_tx_received(nick, tx, offer) if not retval[0]: jlog.info("Maker refuses to continue on receipt of tx") else: sigs = retval[1] self.finalized_offers[nick] = offer - tx = btc.CMutableTransaction.deserialize(hextobin(txhex)) + tx = btc.CMutableTransaction.deserialize(tx) self.finalized_offers[nick]["txd"] = tx - txid = tx.GetTxid()[::-1] # we index the callback by the out-set of the transaction, # because the txid is not known until all scriptSigs collected # (hence this is required for Makers, but not Takers). @@ -497,14 +492,12 @@ class JMMakerClientProtocol(JMClientProtocol): txinfo, self.unconfirm_callback, "unconfirmed", "transaction with outputs: " + str(txinfo) + " not broadcast.") - d = self.callRemote(commands.JMTXSigs, - nick=nick, - sigs=json.dumps(sigs)) + d = self.callRemote(commands.JMTXSigs, nick=nick, sigs=sigs) self.defaultCallbacks(d) return {"accepted": True} @commands.JMTXBroadcast.responder - def on_JM_TX_BROADCAST(self, txhex): + def on_JM_TX_BROADCAST(self, tx): """ Makers have no issue broadcasting anything, so only need to prevent crashes. Note in particular we don't check the return value, @@ -512,11 +505,10 @@ class JMMakerClientProtocol(JMClientProtocol): our (maker)'s concern. """ try: - txbin = hextobin(txhex) - jm_single().bc_interface.pushtx(txbin) + jm_single().bc_interface.pushtx(tx) except: jlog.info("We received an invalid transaction broadcast " - "request: " + txhex) + "request: " + tx.hex()) return {"accepted": True} def tx_match(self, txd): @@ -547,9 +539,9 @@ class JMMakerClientProtocol(JMClientProtocol): "transaction with outputs " + str(txinfo) + " not confirmed.") d = self.callRemote(commands.JMAnnounceOffers, - to_announce=json.dumps(to_announce), - to_cancel=json.dumps(to_cancel), - offerlist=json.dumps(self.client.offerlist)) + to_announce=to_announce, + to_cancel=to_cancel, + offerlist=self.client.offerlist) self.defaultCallbacks(d) return True @@ -564,9 +556,9 @@ class JMMakerClientProtocol(JMClientProtocol): txid, confirms) self.client.modify_orders(to_cancel, to_announce) d = self.callRemote(commands.JMAnnounceOffers, - to_announce=json.dumps(to_announce), - to_cancel=json.dumps(to_cancel), - offerlist=json.dumps(self.client.offerlist)) + to_announce=to_announce, + to_cancel=to_cancel, + offerlist=self.client.offerlist) self.defaultCallbacks(d) return True @@ -601,7 +593,7 @@ class JMTakerClientProtocol(JMClientProtocol): d = self.callRemote(commands.JMInit, bcsource=blockchain_source, network=network, - irc_configs=json.dumps(irc_configs), + irc_configs=irc_configs, minmakers=minmakers, maker_timeout_sec=maker_timeout_sec, dust_threshold=jm_single().DUST_THRESHOLD) @@ -641,7 +633,7 @@ class JMTakerClientProtocol(JMClientProtocol): def on_JM_UP(self): d = self.callRemote(commands.JMSetup, role="TAKER", - offers="{}", + initdata=None, use_fidelity_bond=False) self.defaultCallbacks(d) return {'accepted': True} @@ -669,13 +661,12 @@ class JMTakerClientProtocol(JMClientProtocol): the ioauth data and returns the proposed transaction, passes the phase 2 initiating data to the daemon. """ - ioauth_data = json.loads(ioauth_data) if not success: jlog.info("Makers who didnt respond: " + str(ioauth_data)) self.client.add_ignored_makers(ioauth_data) return {'accepted': True} else: - jlog.info("Makers responded with: " + json.dumps(ioauth_data)) + jlog.info("Makers responded with: " + str(ioauth_data)) retval = self.client.receive_utxos(ioauth_data) if not retval[0]: jlog.info("Taker is not continuing, phase 2 abandoned.") @@ -717,7 +708,7 @@ class JMTakerClientProtocol(JMClientProtocol): amount=amt, commitment=str(cmt), revelation=str(rev), - filled_offers=json.dumps(foffers)) + filled_offers=foffers) self.defaultCallbacks(d) return {'accepted': True} diff --git a/jmclient/jmclient/maker.py b/jmclient/jmclient/maker.py index 2213598..3fd9066 100644 --- a/jmclient/jmclient/maker.py +++ b/jmclient/jmclient/maker.py @@ -118,7 +118,7 @@ class Maker(object): return (True, utxos, auth_pub, cj_addr, change_addr, btc_sig) @hexbin - def on_tx_received(self, nick, tx_from_taker, offerinfo): + def on_tx_received(self, nick, tx, offerinfo): """Called when the counterparty has sent an unsigned transaction. Sigs are created and returned if and only if the transaction passes verification checks (see @@ -129,9 +129,9 @@ class Maker(object): if not isinstance(offerinfo["offer"]["cjfee"], str): offerinfo["offer"]["cjfee"] = bintohex(offerinfo["offer"]["cjfee"]) try: - tx = btc.CMutableTransaction.deserialize(tx_from_taker) + tx = btc.CMutableTransaction.deserialize(tx) except Exception as e: - return (False, 'malformed txhex. ' + repr(e)) + return (False, 'malformed tx. ' + repr(e)) # if the above deserialization was successful, the human readable # parsing will be also: jlog.info('obtained tx\n' + btc.human_readable_transaction(tx)) diff --git a/jmclient/test/test_client_protocol.py b/jmclient/test/test_client_protocol.py index bf8ceb8..0d4a6ba 100644 --- a/jmclient/test/test_client_protocol.py +++ b/jmclient/test/test_client_protocol.py @@ -20,6 +20,7 @@ from commontest import default_max_cj_fee import json import jmbitcoin as bitcoin import twisted +import base64 twisted.internet.base.DelayedCall.debug = True test_completed = False @@ -99,7 +100,7 @@ class DummyMaker(Maker): # success, utxos, auth_pub, cj_addr, change_addr, btc_sig return True, [], b"", '', '', '' - def on_tx_received(self, nick, txhex, offerinfo): + def on_tx_received(self, nick, tx, offerinfo): # success, sigs return True, [] @@ -185,8 +186,8 @@ class JMTestServerProtocol(JMBaseProtocol): return {'accepted': True} @JMSetup.responder - def on_JM_SETUP(self, role, offers, use_fidelity_bond): - show_receipt("JMSETUP", role, offers, use_fidelity_bond) + def on_JM_SETUP(self, role, initdata, use_fidelity_bond): + show_receipt("JMSETUP", role, initdata, use_fidelity_bond) d = self.callRemote(JMSetupDone) self.defaultCallbacks(d) return {'accepted': True} @@ -208,8 +209,8 @@ class JMTestServerProtocol(JMBaseProtocol): success = False if amount == -1 else True show_receipt("JMFILL", amount, commitment, revelation, filled_offers) d = self.callRemote(JMFillResponse, - success=success, - ioauth_data = json.dumps(['dummy', 'list'])) + success=success, + ioauth_data=['dummy', 'list']) return {'accepted': True} @JMMakeTx.responder @@ -390,13 +391,14 @@ class TestMakerClientProtocol(unittest.TestCase): def test_JMAuthReceived(self): yield self.init_client() yield self.callClient( - JMAuthReceived, nick='testnick', offer='{}', - commitment='testcommitment', revelation='{}', amount=100000, + JMAuthReceived, nick='testnick', offer={}, + commitment='testcommitment', revelation={}, amount=100000, kphex='testkphex') @inlineCallbacks def test_JMTXReceived(self): yield self.init_client() yield self.callClient( - JMTXReceived, nick='testnick', txhex=t_raw_signed_tx, - offer='{"cjaddr":"2MwfecDHsQTm4Gg3RekQdpqAMR15BJrjfRF"}') + JMTXReceived, nick='testnick', + tx=base64.b16decode(t_raw_signed_tx, casefold=True), + offer={"cjaddr":"2MwfecDHsQTm4Gg3RekQdpqAMR15BJrjfRF"}) diff --git a/jmdaemon/jmdaemon/daemon_protocol.py b/jmdaemon/jmdaemon/daemon_protocol.py index 44280c9..7ae1868 100644 --- a/jmdaemon/jmdaemon/daemon_protocol.py +++ b/jmdaemon/jmdaemon/daemon_protocol.py @@ -9,7 +9,7 @@ from .protocol import (COMMAND_PREFIX, ORDER_KEYS, NICK_HASH_LENGTH, COMMITMENT_PREFIXES) from .irc import IRCMessageChannel -from jmbase import (hextobin, is_hs_uri, get_tor_agent, JMHiddenService, +from jmbase import (is_hs_uri, get_tor_agent, JMHiddenService, get_nontor_agent, BytesProducer, wrapped_urlparse, bdict_sdict_convert, JMHTTPResource) from jmbase.commands import * @@ -32,7 +32,6 @@ import os from io import BytesIO import copy from functools import wraps -from numbers import Integral """Joinmarket application protocol control flow. For documentation on protocol (formats, message sequence) see @@ -166,15 +165,14 @@ class HTTPPassThrough(amp.AMP): def on_INIT(self, netconfig): """ The network config must be passed in json - and contains these fields: - socks5_host - socks5_proxy - servers (comma separated list) - tls_whitelist (comma separated list) - filterconfig (not yet defined) - credentials (not yet defined) - """ - netconfig = json.loads(netconfig) + and contains these fields: + socks5_host + socks5_proxy + servers (comma separated list) + tls_whitelist (comma separated list) + filterconfig (not yet defined) + credentials (not yet defined) + """ self.socks5_host = netconfig["socks5_host"] self.socks5_port = int(netconfig["socks5_port"]) self.servers = [a for a in netconfig["servers"] if a != ""] @@ -272,7 +270,6 @@ class HTTPPassThrough(amp.AMP): class BIP78ServerProtocol(HTTPPassThrough): @BIP78ReceiverInit.responder def on_BIP78_RECEIVER_INIT(self, netconfig): - netconfig = json.loads(netconfig) self.serving_port = int(netconfig["port"]) self.tor_control_host = netconfig["tor_control_host"] self.tor_control_port = int(netconfig["tor_control_port"]) @@ -325,7 +322,7 @@ class BIP78ServerProtocol(HTTPPassThrough): """ self.post_request = request d = self.callRemote(BIP78ReceiverOriginalPSBT, body=body, - params=json.dumps(bdict_sdict_convert(params))) + params=bdict_sdict_convert(params)) self.defaultCallbacks(d) @BIP78ReceiverSendProposal.responder @@ -354,9 +351,9 @@ class BIP78ServerProtocol(HTTPPassThrough): @BIP78SenderOriginalPSBT.responder def on_BIP78_SENDER_ORIGINAL_PSBT(self, body, params): self.postRequest(body, self.servers[0], - self.bip78_receiver_response, - params=json.loads(params), - headers=Headers({"Content-Type": ["text/plain"]})) + self.bip78_receiver_response, + params=params, + headers=Headers({"Content-Type": ["text/plain"]})) return {"accepted": True} def bip78_receiver_response(self, response, server): @@ -475,6 +472,8 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): self.sig_lock = threading.Lock() self.active_orders = {} self.use_fidelity_bond = False + self.offerlist = None + self.kp = None def checkClientResponse(self, response): """A generic check of client acceptance; any failure @@ -502,11 +501,9 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): If a new message channel configuration is required, the current one is shutdown in preparation. """ - self.maker_timeout_sec = int(maker_timeout_sec) - # used in OrderbookWatch: + self.maker_timeout_sec = maker_timeout_sec + self.minmakers = minmakers self.dust_threshold = int(dust_threshold) - self.minmakers = int(minmakers) - irc_configs = json.loads(irc_configs) #(bitcoin) network only referenced in channel name construction self.network = network if irc_configs == self.irc_configs: @@ -554,7 +551,7 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): return {'accepted': True} @JMSetup.responder - def on_JM_SETUP(self, role, offers, use_fidelity_bond): + def on_JM_SETUP(self, role, initdata, use_fidelity_bond): assert self.jm_state == 0 self.role = role self.crypto_boxes = {} @@ -568,7 +565,7 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): if self.role == "TAKER": self.mcc.pubmsg(COMMAND_PREFIX + "orderbook") elif self.role == "MAKER": - self.offerlist = json.loads(offers) + self.offerlist = initdata self.use_fidelity_bond = use_fidelity_bond self.mcc.announce_orders(self.offerlist, None, None, None) self.jm_state = 1 @@ -614,15 +611,14 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): """Takes the necessary data from the Taker and initiates the Stage 1 interaction with the Makers. """ - if not (self.jm_state == 1 and isinstance(amount, Integral) - and amount >= 0): + if self.jm_state != 1 or amount < 0: return {'accepted': False} self.cjamount = amount self.commitment = commitment self.revelation = revelation #Reset utxo data to null for this new transaction self.ioauth_data = {} - self.active_orders = json.loads(filled_offers) + self.active_orders = filled_offers for nick, offer_dict in self.active_orders.items(): offer_fill_msg = " ".join([str(offer_dict["oid"]), str(amount), self.kp.hex_pk().decode('ascii'), str(commitment)]) @@ -639,7 +635,6 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): if not self.jm_state == 4: log.msg("Make tx was called in wrong state, rejecting") return {'accepted': False} - nick_list = json.loads(nick_list) self.mcc.send_tx(nick_list, txhex) return {'accepted': True} @@ -659,9 +654,7 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): """ if self.role != "MAKER": return - to_announce = json.loads(to_announce) - to_cancel = json.loads(to_cancel) - self.offerlist = json.loads(offerlist) + self.offerlist = offerlist if len(to_cancel) > 0: self.mcc.cancel_orders(to_cancel) if len(to_announce) > 0: @@ -682,14 +675,13 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): """ if not self.role == "MAKER": return - if not nick in self.active_orders: + if nick not in self.active_orders: return - utxos= json.loads(utxolist) #completed population of order/offer object self.active_orders[nick]["cjaddr"] = cjaddr self.active_orders[nick]["changeaddr"] = changeaddr - self.active_orders[nick]["utxos"] = utxos - msg = str(",".join(utxos.keys())) + " " + " ".join( + self.active_orders[nick]["utxos"] = utxolist + msg = str(",".join(utxolist)) + " " + " ".join( [pubkey, cjaddr, changeaddr, pubkeysig]) self.mcc.prepare_privmsg(nick, "ioauth", msg) #In case of *blacklisted (ie already used) commitments, we already @@ -705,11 +697,10 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): @JMTXSigs.responder def on_JM_TX_SIGS(self, nick, sigs): """Signatures that the Maker has produced - are passed here to the daemon as a list and - broadcast one by one. TODO: could shorten this, - have more than one sig per message. - """ - sigs = json.loads(sigs) + are passed here to the daemon as a list and + broadcast one by one. TODO: could shorten this, + have more than one sig per message. + """ for sig in sigs: self.mcc.prepare_privmsg(nick, "sig", sig) return {"accepted": True} @@ -794,19 +785,19 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): @maker_only def on_seen_auth(self, nick, commitment_revelation): """Passes to Maker the !auth message from the Taker, - for processing. This will include validating the PoDLE - commitment revelation against the existing commitment, - which was already stored in active_orders[nick]. - """ - if not nick in self.active_orders: + for processing. This will include validating the PoDLE + commitment revelation against the existing commitment, + which was already stored in active_orders[nick]. + """ + if nick not in self.active_orders: return ao =self.active_orders[nick] #ask the client to validate the commitment and prepare the utxo data d = self.callRemote(JMAuthReceived, nick=nick, - offer=json.dumps(ao["offer"]), + offer=ao["offer"], commitment=ao["commit"], - revelation=json.dumps(commitment_revelation), + revelation=commitment_revelation, amount=ao["amount"], kphex=ao["kp"].hex_pk().decode('ascii')) self.defaultCallbacks(d) @@ -831,20 +822,14 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): self.mcc.pubmsg("!hp2 " + commitment) @maker_only - def on_push_tx(self, nick, txhex): - """Broadcast unquestioningly, except checking - hex format. - """ - try: - dummy = hextobin(txhex) - except: - return - d = self.callRemote(JMTXBroadcast, - txhex=txhex) + def on_push_tx(self, nick, tx): + """Broadcast unquestioningly + """ + d = self.callRemote(JMTXBroadcast, tx=tx) self.defaultCallbacks(d) @maker_only - def on_seen_tx(self, nick, txhex): + def on_seen_tx(self, nick, tx): """Passes the txhex to the Maker for verification and signing. Note the security checks occur in Maker. """ @@ -855,10 +840,7 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): ao = copy.deepcopy(self.active_orders[nick]) del ao["crypto_box"] del ao["kp"] - d = self.callRemote(JMTXReceived, - nick=nick, - txhex=txhex, - offer=json.dumps(ao)) + d = self.callRemote(JMTXReceived, nick=nick, tx=tx, offer=ao) self.defaultCallbacks(d) @taker_only @@ -898,9 +880,7 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): def on_sig(self, nick, sig): """Pass signature through to Taker. """ - d = self.callRemote(JMSigReceived, - nick=nick, - sig=sig) + d = self.callRemote(JMSigReceived, nick=nick, sig=sig) self.defaultCallbacks(d) def on_error(self, msg): @@ -990,12 +970,12 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): self.jm_state = 3 if not accepted: #use ioauth data field to return the list of non-responsive makers - nonresponders = [x for x in self.active_orders.keys() if x not - in self.ioauth_data.keys()] + nonresponders = [x for x in self.active_orders + if x not in self.ioauth_data] ioauth_data = self.ioauth_data if accepted else nonresponders d = self.callRemote(JMFillResponse, - success=accepted, - ioauth_data = json.dumps(ioauth_data)) + success=accepted, + ioauth_data=ioauth_data) if not accepted: #Client simply accepts failure TODO self.defaultCallbacks(d) @@ -1010,7 +990,7 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): either send success + ioauth data if enough makers, else send failure to client. """ - response = True if len(self.ioauth_data.keys()) >= self.minmakers else False + response = True if len(self.ioauth_data) >= self.minmakers else False self.respondToIoauths(response) def checkUtxosAccepted(self, accepted): diff --git a/jmdaemon/jmdaemon/message_channel.py b/jmdaemon/jmdaemon/message_channel.py index 1abdee4..aa786f2 100644 --- a/jmdaemon/jmdaemon/message_channel.py +++ b/jmdaemon/jmdaemon/message_channel.py @@ -1028,21 +1028,21 @@ class MessageChannel(object): elif _chunks[0] == 'tx': b64tx = _chunks[1] try: - txhex = binascii.hexlify(base64.b64decode(b64tx)).decode('ascii') + tx = base64.b64decode(b64tx) except TypeError as e: self.send_error(nick, 'bad base64 tx. ' + repr(e)) return if self.on_seen_tx: - self.on_seen_tx(nick, txhex) + self.on_seen_tx(nick, tx) elif _chunks[0] == 'push': b64tx = _chunks[1] try: - txhex = binascii.hexlify(base64.b64decode(b64tx)).decode('ascii') + tx = base64.b64decode(b64tx) except TypeError as e: self.send_error(nick, 'bad base64 tx. ' + repr(e)) return if self.on_push_tx: - self.on_push_tx(nick, txhex) + self.on_push_tx(nick, tx) except (IndexError, ValueError): # TODO proper error handling log.debug('cj peer error TODO handle') diff --git a/jmdaemon/test/test_daemon_protocol.py b/jmdaemon/test/test_daemon_protocol.py index 4d19413..22a5298 100644 --- a/jmdaemon/test/test_daemon_protocol.py +++ b/jmdaemon/test/test_daemon_protocol.py @@ -19,7 +19,6 @@ from twisted.protocols import amp from twisted.trial import unittest from jmbase.commands import * from msgdata import * -import json import base64 import sys from dummy_mc import DummyMessageChannel @@ -64,7 +63,7 @@ class JMTestClientProtocol(JMBaseProtocol): d = self.callRemote(JMInit, bcsource="dummyblockchain", network="dummynetwork", - irc_configs=json.dumps(irc), + irc_configs=irc, minmakers=2, maker_timeout_sec=3, dust_threshold=27300) @@ -85,7 +84,7 @@ class JMTestClientProtocol(JMBaseProtocol): show_receipt("JMUP") d = self.callRemote(JMSetup, role="TAKER", - offers="{}", + initdata=None, use_fidelity_bond=False) self.defaultCallbacks(d) return {'accepted': True} @@ -104,10 +103,9 @@ class JMTestClientProtocol(JMBaseProtocol): return {'accepted': True} def maketx(self, ioauth_data): - ioauth_data = json.loads(ioauth_data) - nl = list(ioauth_data.keys()) + nl = list(ioauth_data) d = self.callRemote(JMMakeTx, - nick_list= json.dumps(nl), + nick_list=nl, txhex="deadbeef") self.defaultCallbacks(d) @@ -136,7 +134,7 @@ class JMTestClientProtocol(JMBaseProtocol): amount=100, commitment="dummycommitment", revelation="dummyrevelation", - filled_offers=json.dumps(t_chosen_orders)) + filled_offers=t_chosen_orders) self.defaultCallbacks(d) return {'accepted': True} @@ -215,9 +213,9 @@ class JMDaemonTestServerProtocol(JMDaemonServerProtocol): @JMInit.responder def on_JM_INIT(self, bcsource, network, irc_configs, minmakers, maker_timeout_sec, dust_threshold): - self.maker_timeout_sec = int(maker_timeout_sec) + self.maker_timeout_sec = maker_timeout_sec self.dust_threshold = int(dust_threshold) - self.minmakers = int(minmakers) + self.minmakers = minmakers mcs = [DummyMC(None)] self.mcc = MessageChannelCollection(mcs) #The following is a hack to get the counterparties marked seen/active; @@ -240,21 +238,20 @@ class JMDaemonTestServerProtocol(JMDaemonServerProtocol): return {'accepted': True} @JMFill.responder - def on_JM_FILL(self, amount, commitment, revelation, filled_offers): - tmpfo = json.loads(filled_offers) + def on_JM_FILL(self, amount, commitment, revelation, filled_offers): dummypub = "073732a7ca60470f709f23c602b2b8a6b1ba62ee8f3f83a61e5484ab5cbf9c3d" #trigger invalid on_pubkey conditions reactor.callLater(1, self.on_pubkey, "notrealcp", dummypub) - reactor.callLater(2, self.on_pubkey, list(tmpfo.keys())[0], dummypub + "deadbeef") + reactor.callLater(2, self.on_pubkey, list(filled_offers)[0], dummypub + "deadbeef") #trigger invalid on_ioauth condition reactor.callLater(2, self.on_ioauth, "notrealcp", 1, 2, 3, 4, 5) #trigger msg sig verify request operation for a dummy message #currently a pass-through reactor.callLater(1, self.request_signature_verify, "1", "!push abcd abc def", "3", "4", - str(list(tmpfo.keys())[0]), 6, 7, self.mcc.mchannels[0].hostid) + str(list(filled_offers)[0]), 6, 7, self.mcc.mchannels[0].hostid) #send "valid" onpubkey, onioauth messages - for k, v in tmpfo.items(): + for k, v in filled_offers.items(): reactor.callLater(1, self.on_pubkey, k, dummypub) reactor.callLater(2, self.on_ioauth, k, ['a', 'b'], "auth_pub", "cj_addr", "change_addr", "btc_sig") @@ -262,7 +259,7 @@ class JMDaemonTestServerProtocol(JMDaemonServerProtocol): @JMMakeTx.responder def on_JM_MAKE_TX(self, nick_list, txhex): - for n in json.loads(nick_list): + for n in nick_list: reactor.callLater(1, self.on_sig, n, "dummytxsig") return super().on_JM_MAKE_TX(nick_list, txhex) diff --git a/test/ygrunner.py b/test/ygrunner.py index b3a888e..75411dc 100644 --- a/test/ygrunner.py +++ b/test/ygrunner.py @@ -54,12 +54,12 @@ class MaliciousYieldGenerator(YieldGeneratorBasic): jmprint("Counterparty commitment rejected maliciously", "debug") return (False,) return super().on_auth_received(nick, offer, commitment, cr, amount, kphex) - def on_tx_received(self, nick, txhex, offerinfo): + def on_tx_received(self, nick, tx, offerinfo): if self.txmal: if random.randint(1, 100) < self.mfrac: jmprint("Counterparty tx rejected maliciously", "debug") return (False, "malicious tx rejection") - return super().on_tx_received(nick, txhex, offerinfo) + return super().on_tx_received(nick, tx, offerinfo) class DeterministicMaliciousYieldGenerator(YieldGeneratorBasic): """Overrides, randomly chosen persistently, some maker functions @@ -85,11 +85,11 @@ class DeterministicMaliciousYieldGenerator(YieldGeneratorBasic): jmprint("Counterparty commitment rejected maliciously", "debug") return (False,) return super().on_auth_received(nick, offer, commitment, cr, amount, kphex) - def on_tx_received(self, nick, txhex, offerinfo): + def on_tx_received(self, nick, tx, offerinfo): if self.txmal: jmprint("Counterparty tx rejected maliciously", "debug") return (False, "malicious tx rejection") - return super().on_tx_received(nick, txhex, offerinfo) + return super().on_tx_received(nick, tx, offerinfo) @pytest.mark.parametrize(