diff --git a/jmbase/jmbase/__init__.py b/jmbase/jmbase/__init__.py index 15ecebd..351cd64 100644 --- a/jmbase/jmbase/__init__.py +++ b/jmbase/jmbase/__init__.py @@ -1,6 +1,6 @@ from __future__ import print_function from .support import (get_log, chunks, debug_silence, debug_dump_object, - joinmarket_alert, core_alert, get_password) + joinmarket_alert, core_alert, get_password, _byteify) from commands import * diff --git a/jmbase/jmbase/commands.py b/jmbase/jmbase/commands.py index ed62e29..1f302f2 100644 --- a/jmbase/jmbase/commands.py +++ b/jmbase/jmbase/commands.py @@ -37,7 +37,10 @@ class JMStartMC(JMCommand): class JMSetup(JMCommand): arguments = [('role', String()), - ('n_counterparties', Integer())] + ('initdata', String())] + +"""Taker specific commands +""" class JMRequestOffers(JMCommand): arguments = [] @@ -67,6 +70,28 @@ class JMMsgSignatureVerify(JMCommand): ('nick', String()), ('fullmsg', String()), ('hostid', String())] + +"""Maker specific commands +""" + +class JMAnnounceOffers(JMCommand): + arguments = [('offerlist', String())] + +class JMMakerPubkey(JMCommand): + arguments = [('nick', String()), + ('pubkey', String())] + +class JMIOAuth(JMCommand): + arguments = [('nick', String()), + ('utxolist', String()), + ('pubkey', String()), + ('cjaddr', String()), + ('changeaddr', String()), + ('pubkeysig', String())] + +class JMTXSigs(JMCommand): + arguments = [('nick', String()), + ('sigs', String())] #commands from daemon to client @@ -93,6 +118,26 @@ class JMSigReceived(JMCommand): arguments = [('nick', String()), ('sig', String())] +"""Maker specific daemon-client messages +""" + +class JMCommitmentOpen(JMCommand): + arguments = [('nick', String()), + ('revelation', String())] + +class JMAuthReceived(JMCommand): + arguments = [('nick', String()), + ('offer', String()), + ('commitment', String()), + ('revelation', String()), + ('amount', Integer()), + ('kphex', String())] + +class JMTXReceived(JMCommand): + arguments = [('nick', String()), + ('txhex', String()), + ('offer', String())] + class JMRequestMsgSig(JMCommand): arguments = [('nick', String()), ('cmd', String()), diff --git a/jmbase/jmbase/support.py b/jmbase/jmbase/support.py index d452127..fe78b46 100644 --- a/jmbase/jmbase/support.py +++ b/jmbase/jmbase/support.py @@ -68,3 +68,20 @@ def debug_dump_object(obj, skip_fields=None): log.debug(pprint.pformat(v)) else: log.debug(str(v)) + +def _byteify(data, ignore_dicts = False): + # if this is a unicode string, return its string representation + if isinstance(data, unicode): + return data.encode('utf-8') + # if this is a list of values, return list of byteified values + if isinstance(data, list): + return [ _byteify(item, ignore_dicts=True) for item in data ] + # if this is a dictionary, return dictionary of byteified keys and values + # but only if we haven't already byteified it + if isinstance(data, dict) and not ignore_dicts: + return { + _byteify(key, ignore_dicts=True): _byteify(value, ignore_dicts=True) + for key, value in data.iteritems() + } + # if it's anything else, return it in its original form + return data diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index ca9adf7..e974e50 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -21,12 +21,11 @@ from .wallet import (AbstractWallet, BitcoinCoreInterface, Wallet, create_wallet_file, SegwitWallet, Bip39Wallet) from .configure import (load_program_config, jm_single, get_p2pk_vbyte, get_network, jm_single, get_network, validate_address, get_irc_mchannels, - check_utxo_blacklist, get_blockchain_interface_instance, get_p2sh_vbyte, - set_config) + get_blockchain_interface_instance, get_p2sh_vbyte, set_config) from .blockchaininterface import (BlockrInterface, BlockchainInterface, sync_wallet, RegtestBitcoinCoreInterface, BitcoinCoreInterface) from .electruminterface import ElectrumInterface -from .client_protocol import JMTakerClientProtocolFactory, start_reactor +from .client_protocol import (JMClientProtocolFactory, start_reactor) from .podle import (set_commitment_file, get_commitment_file, generate_podle_error_string, add_external_commitments, PoDLE, generate_podle, get_podle_commitments, @@ -39,6 +38,8 @@ from .taker_utils import (tumbler_taker_finished_update, restart_waiter, restart_wait, get_tumble_log, direct_send, tumbler_filter_orders_callback) from .wallet_utils import wallet_tool_main +from .maker import Maker +from .yieldgenerator import YieldGenerator, ygmain # Set default logging handler to avoid "No handler found" warnings. try: diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index 86945d4..61398e0 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -34,22 +34,16 @@ jlog = get_log() class JMProtocolError(Exception): pass -class JMTakerClientProtocol(amp.AMP): - - def __init__(self, factory, taker, nick_priv=None): - self.taker = taker - self.factory = factory - self.orderbook = None - self.supported_messages = ["JM_UP", "JM_SETUP_DONE", "JM_FILL_RESPONSE", - "JM_OFFERS", "JM_SIG_RECEIVED", - "JM_REQUEST_MSGSIG", - "JM_REQUEST_MSGSIG_VERIFY", "JM_INIT_PROTO"] - if not nick_priv: - self.nick_priv = hashlib.sha256(os.urandom(16)).hexdigest() + '01' - else: - self.nick_priv = nick_priv +class JMClientProtocol(amp.AMP): + def __init__(self, factory, client, nick_priv=None): + self.client = client + self.factory = factory + if not nick_priv: + self.nick_priv = hashlib.sha256(os.urandom(16)).hexdigest() + '01' + else: + self.nick_priv = nick_priv - self.shutdown_requested = False + self.shutdown_requested = False def checkClientResponse(self, response): """A generic check of client acceptance; any failure @@ -72,12 +66,173 @@ class JMTakerClientProtocol(amp.AMP): self.factory.setClient(self) self.clientStart() + def set_nick(self): + self.nick_pubkey = btc.privtopub(self.nick_priv) + self.nick_pkh_raw = hashlib.sha256(self.nick_pubkey).digest()[ + :self.nick_hashlen] + self.nick_pkh = btc.changebase(self.nick_pkh_raw, 256, 58) + #right pad to maximum possible; b58 is not fixed length. + #Use 'O' as one of the 4 not included chars in base58. + self.nick_pkh += 'O' * (self.nick_maxencoded - len(self.nick_pkh)) + #The constructed length will be 1 + 1 + NICK_MAX_ENCODED + self.nick = self.nick_header + str(self.jm_version) + self.nick_pkh + jm_single().nickname = self.nick + + @commands.JMInitProto.responder + def on_JM_INIT_PROTO(self, nick_hash_length, nick_max_encoded, + joinmarket_nick_header, joinmarket_version): + """Daemon indicates init-ed status and passes back protocol constants. + Use protocol settings to set actual nick from nick private key, + then call setup to instantiate message channel connections in the daemon. + """ + self.nick_hashlen = nick_hash_length + self.nick_maxencoded = nick_max_encoded + self.nick_header = joinmarket_nick_header + self.jm_version = joinmarket_version + self.set_nick() + d = self.callRemote(commands.JMStartMC, + nick=self.nick) + self.defaultCallbacks(d) + return {'accepted': True} + + @commands.JMRequestMsgSig.responder + def on_JM_REQUEST_MSGSIG(self, nick, cmd, msg, msg_to_be_signed, hostid): + sig = btc.ecdsa_sign(str(msg_to_be_signed), self.nick_priv) + msg_to_return = str(msg) + " " + self.nick_pubkey + " " + sig + d = self.callRemote(commands.JMMsgSignature, + nick=nick, + cmd=cmd, + msg_to_return=msg_to_return, + hostid=hostid) + self.defaultCallbacks(d) + return {'accepted': True} + + @commands.JMRequestMsgSigVerify.responder + def on_JM_REQUEST_MSGSIG_VERIFY(self, msg, fullmsg, sig, pubkey, nick, + hashlen, max_encoded, hostid): + verif_result = True + if not btc.ecdsa_verify(str(msg), sig, pubkey): + jlog.debug("nick signature verification failed, ignoring.") + verif_result = False + #check that nick matches hash of pubkey + nick_pkh_raw = hashlib.sha256(pubkey).digest()[:hashlen] + nick_stripped = nick[2:2 + max_encoded] + #strip right padding + nick_unpadded = ''.join([x for x in nick_stripped if x != 'O']) + if not nick_unpadded == btc.changebase(nick_pkh_raw, 256, 58): + jlog.debug("Nick hash check failed, expected: " + str(nick_unpadded) + + ", got: " + str(btc.changebase(nick_pkh_raw, 256, 58))) + verif_result = False + d = self.callRemote(commands.JMMsgSignatureVerify, + verif_result=verif_result, + nick=nick, + fullmsg=fullmsg, + hostid=hostid) + self.defaultCallbacks(d) + return {'accepted': True} + +class JMMakerClientProtocol(JMClientProtocol): + def __init__(self, factory, maker, nick_priv=None): + self.factory = factory + JMClientProtocol.__init__(self, factory, maker, nick_priv) + + @commands.JMUp.responder + def on_JM_UP(self): + d = self.callRemote(commands.JMSetup, + role="MAKER", + initdata=json.dumps(self.client.offerlist)) + #for as long as the maker is up, it can asynchronously pass through + #its updated offer list + offer_loop = LoopingCall(self.get_offers) + offer_loop.start(3.0) + self.defaultCallbacks(d) + return {'accepted': True} + + def get_offers(self): + """Feeds through current offers from Maker obect, to the daemon + """ + offerlist = self.client.offerlist + d = self.callRemote(commands.JMAnnounceOffers, + offerlist=json.dumps(offerlist)) + self.defaultCallbacks(d) + + @commands.JMSetupDone.responder + def on_JM_SETUP_DONE(self): + jlog.info("JM daemon setup complete") + return {'accepted': True} + def clientStart(self): """Upon confirmation of network connection to daemon, request message channel initialization with relevant config data for our message channels """ - if self.taker.aborted: + if self.client.aborted: + return + #needed only for naming convention in IRC currently + blockchain_source = jm_single().config.get("BLOCKCHAIN", + "blockchain_source") + #needed only for channel naming convention + network = jm_single().config.get("BLOCKCHAIN", "network") + irc_configs = get_irc_mchannels() + #only here because Init message uses this field; not used by makers TODO + minmakers = jm_single().config.getint("POLICY", "minimum_makers") + maker_timeout_sec = jm_single().maker_timeout_sec + + d = self.callRemote(commands.JMInit, + bcsource=blockchain_source, + network=network, + irc_configs=json.dumps(irc_configs), + minmakers=minmakers, + maker_timeout_sec=maker_timeout_sec) + self.defaultCallbacks(d) + + @commands.JMAuthReceived.responder + def on_JM_AUTH_RECEIVED(self, nick, offer, commitment, revelation, amount, + kphex): + offer = json.loads(offer) + revelation = json.loads(revelation) + retval = self.client.on_auth_received(nick, offer, + commitment, revelation, amount, kphex) + if not retval[0]: + jlog.info("Maker refuses to continue on receiving auth.") + else: + utxos, auth_pub, cj_addr, change_addr, btc_sig = retval[1:] + d = self.callRemote(commands.JMIOAuth, + nick=nick, + utxolist=json.dumps(utxos), + pubkey=auth_pub, + cjaddr=cj_addr, + changeaddr=change_addr, + pubkeysig=btc_sig) + self.defaultCallbacks(d) + return {"accepted": True} + + @commands.JMTXReceived.responder + def on_JM_TX_RECEIVED(self, nick, txhex, offer): + offer = json.loads(offer) + retval = self.client.on_tx_received(nick, txhex, offer) + if not retval[0]: + jlog.info("Maker refuses to continue on receipt of tx") + else: + sigs = retval[1] + d = self.callRemote(commands.JMTXSigs, + nick=nick, + sigs=json.dumps(sigs)) + self.defaultCallbacks(d) + return {"accepted": True} + +class JMTakerClientProtocol(JMClientProtocol): + + def __init__(self, factory, client, nick_priv=None): + self.orderbook = None + JMClientProtocol.__init__(self, factory, client, nick_priv) + + def clientStart(self): + """Upon confirmation of network connection + to daemon, request message channel initialization + with relevant config data for our message channels + """ + if self.client.aborted: return #needed only for naming convention in IRC currently blockchain_source = jm_single().config.get("BLOCKCHAIN", @@ -90,9 +245,9 @@ class JMTakerClientProtocol(amp.AMP): #To avoid creating yet another config variable, we set the timeout #to 20 * maker_timeout_sec. - if not hasattr(self.taker, 'testflag'): #pragma: no cover + if not hasattr(self.client, 'testflag'): #pragma: no cover reactor.callLater(20*maker_timeout_sec, self.stallMonitor, - self.taker.schedule_index+1) + self.client.schedule_index+1) d = self.callRemote(commands.JMInit, bcsource=blockchain_source, @@ -112,60 +267,31 @@ class JMTakerClientProtocol(amp.AMP): on to the next item before we were woken up. """ jlog.info("STALL MONITOR:") - if self.taker.aborted: + if self.client.aborted: jlog.info("Transaction was aborted.") return - if not self.taker.schedule_index == schedule_index: + if not self.client.schedule_index == schedule_index: #TODO pre-initialize() ? jlog.info("No stall detected, continuing") return - if self.taker.waiting_for_conf: + if self.client.waiting_for_conf: #Don't restart if the tx is already on the network! jlog.info("No stall detected, continuing") return - if not self.taker.txid: + if not self.client.txid: #txid is set on pushing; if it's not there, we have failed. jlog.info("Stall detected. Regenerating transactions and retrying.") - self.taker.on_finished_callback(False, True, 0.0) + self.client.on_finished_callback(False, True, 0.0) else: #This shouldn't really happen; if the tx confirmed, #the finished callback should already be called. jlog.info("Tx was already pushed; ignoring") - def set_nick(self): - self.nick_pubkey = btc.privtopub(self.nick_priv) - self.nick_pkh_raw = hashlib.sha256(self.nick_pubkey).digest()[ - :self.nick_hashlen] - self.nick_pkh = btc.changebase(self.nick_pkh_raw, 256, 58) - #right pad to maximum possible; b58 is not fixed length. - #Use 'O' as one of the 4 not included chars in base58. - self.nick_pkh += 'O' * (self.nick_maxencoded - len(self.nick_pkh)) - #The constructed length will be 1 + 1 + NICK_MAX_ENCODED - self.nick = self.nick_header + str(self.jm_version) + self.nick_pkh - jm_single().nickname = self.nick - - @commands.JMInitProto.responder - def on_JM_INIT_PROTO(self, nick_hash_length, nick_max_encoded, - joinmarket_nick_header, joinmarket_version): - """Daemon indicates init-ed status and passes back protocol constants. - Use protocol settings to set actual nick from nick private key, - then call setup to instantiate message channel connections in the daemon. - """ - self.nick_hashlen = nick_hash_length - self.nick_maxencoded = nick_max_encoded - self.nick_header = joinmarket_nick_header - self.jm_version = joinmarket_version - self.set_nick() - d = self.callRemote(commands.JMStartMC, - nick=self.nick) - self.defaultCallbacks(d) - return {'accepted': True} - @commands.JMUp.responder def on_JM_UP(self): d = self.callRemote(commands.JMSetup, role="TAKER", - n_counterparties=4) #TODO this number should be set + initdata="none") self.defaultCallbacks(d) return {'accepted': True} @@ -189,11 +315,11 @@ class JMTakerClientProtocol(amp.AMP): if not success: nonresponders = ioauth_data jlog.info("Makers didnt respond: " + str(nonresponders)) - self.taker.add_ignored_makers(nonresponders) + self.client.add_ignored_makers(nonresponders) return {'accepted': True} else: jlog.info("Makers responded with: " + json.dumps(ioauth_data)) - retval = self.taker.receive_utxos(ioauth_data) + retval = self.client.receive_utxos(ioauth_data) if not retval[0]: jlog.info("Taker is not continuing, phase 2 abandoned.") jlog.info("Reason: " + str(retval[1])) @@ -208,7 +334,7 @@ class JMTakerClientProtocol(amp.AMP): self.orderbook = json.loads(orderbook) #Removed for now, as judged too large, even for DEBUG: #jlog.debug("Got the orderbook: " + str(self.orderbook)) - retval = self.taker.initialize(self.orderbook) + retval = self.client.initialize(self.orderbook) #format of retval is: #True, self.cjamount, commitment, revelation, self.filtered_orderbook) if not retval[0]: @@ -225,48 +351,12 @@ class JMTakerClientProtocol(amp.AMP): @commands.JMSigReceived.responder def on_JM_SIG_RECEIVED(self, nick, sig): - retval = self.taker.on_sig(nick, sig) + retval = self.client.on_sig(nick, sig) if retval: nick_to_use, txhex = retval self.push_tx(nick_to_use, txhex) return {'accepted': True} - @commands.JMRequestMsgSig.responder - def on_JM_REQUEST_MSGSIG(self, nick, cmd, msg, msg_to_be_signed, hostid): - sig = btc.ecdsa_sign(str(msg_to_be_signed), self.nick_priv) - msg_to_return = str(msg) + " " + self.nick_pubkey + " " + sig - d = self.callRemote(commands.JMMsgSignature, - nick=nick, - cmd=cmd, - msg_to_return=msg_to_return, - hostid=hostid) - self.defaultCallbacks(d) - return {'accepted': True} - - @commands.JMRequestMsgSigVerify.responder - def on_JM_REQUEST_MSGSIG_VERIFY(self, msg, fullmsg, sig, pubkey, nick, - hashlen, max_encoded, hostid): - verif_result = True - if not btc.ecdsa_verify(str(msg), sig, pubkey): - jlog.debug("nick signature verification failed, ignoring.") - verif_result = False - #check that nick matches hash of pubkey - nick_pkh_raw = hashlib.sha256(pubkey).digest()[:hashlen] - nick_stripped = nick[2:2 + max_encoded] - #strip right padding - nick_unpadded = ''.join([x for x in nick_stripped if x != 'O']) - if not nick_unpadded == btc.changebase(nick_pkh_raw, 256, 58): - jlog.debug("Nick hash check failed, expected: " + str(nick_unpadded) - + ", got: " + str(btc.changebase(nick_pkh_raw, 256, 58))) - verif_result = False - d = self.callRemote(commands.JMMsgSignatureVerify, - verif_result=verif_result, - nick=nick, - fullmsg=fullmsg, - hostid=hostid) - self.defaultCallbacks(d) - return {'accepted': True} - def get_offers(self): d = self.callRemote(commands.JMRequestOffers) self.defaultCallbacks(d) @@ -282,23 +372,28 @@ class JMTakerClientProtocol(amp.AMP): txhex=str(txhex_to_push)) self.defaultCallbacks(d) -class JMTakerClientProtocolFactory(protocol.ClientFactory): +class JMClientProtocolFactory(protocol.ClientFactory): protocol = JMTakerClientProtocol - def __init__(self, taker): - self.taker = taker + def __init__(self, client, proto_type="TAKER"): + self.client = client self.proto_client = None + self.proto_type = proto_type + if self.proto_type == "MAKER": + self.protocol = JMMakerClientProtocol + def setClient(self, client): self.proto_client = client def getClient(self): return self.proto_client def buildProtocol(self, addr): - return JMTakerClientProtocol(self, self.taker) - + return self.protocol(self, self.client) def start_reactor(host, port, factory, ish=True, daemon=False): #pragma: no cover #(Cannot start the reactor in tests) + #Not used in prod (twisted logging): + #startLogging(stdout) usessl = True if jm_single().config.get("DAEMON", "use_ssl") != 'false' else False if daemon: try: diff --git a/jmclient/jmclient/configure.py b/jmclient/jmclient/configure.py index b906fca..e0ba82a 100644 --- a/jmclient/jmclient/configure.py +++ b/jmclient/jmclient/configure.py @@ -280,37 +280,6 @@ def donation_address(reusable_donation_pubkey=None): #pragma: no cover log.debug('sending coins to ' + sender_address) return sender_address, sign_k - -def check_utxo_blacklist(commitment, persist=False): - """Compare a given commitment (H(P2) for PoDLE) - with the persisted blacklist log file; - if it has been used before, return False (disallowed), - else return True. - If flagged, persist the usage of this commitment to the blacklist file. - """ - #TODO format error checking? - fname = "blacklist" - if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == 'regtest': - fname += "_" + jm_single().nickname - with jm_single().blacklist_file_lock: - if os.path.isfile(fname): - with open(fname, "rb") as f: - blacklisted_commitments = [x.strip() for x in f.readlines()] - else: - blacklisted_commitments = [] - if commitment in blacklisted_commitments: - return False - elif persist: - blacklisted_commitments += [commitment] - with open(fname, "wb") as f: - f.write('\n'.join(blacklisted_commitments)) - f.flush() - #If the commitment is new and we are *not* persisting, nothing to do - #(we only add it to the list on sending io_auth, which represents actual - #usage). - return True - - def load_program_config(config_path=None, bs=None): global_singleton.config.readfp(io.BytesIO(defaultconfig)) if not config_path: diff --git a/jmclient/jmclient/support.py b/jmclient/jmclient/support.py index 2a85482..d8236fc 100644 --- a/jmclient/jmclient/support.py +++ b/jmclient/jmclient/support.py @@ -206,13 +206,15 @@ def cheapest_order_choose(orders, n): def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None, - pick=False): + pick=False, allowed_types=["swreloffer", "swabsoffer"]): if ignored_makers is None: ignored_makers = [] #Filter ignored makers and inappropriate amounts orders = [o for o in offers if o['counterparty'] not in ignored_makers] orders = [o for o in orders if o['minsize'] < cj_amount] orders = [o for o in orders if o['maxsize'] > cj_amount] + #Filter those not using wished-for offertypes + orders = [o for o in orders if o["ordertype"] in allowed_types] orders_fees = [( o, calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount) - o['txfee']) for o in orders] diff --git a/jmclient/jmclient/taker.py b/jmclient/jmclient/taker.py index 8b42372..a5a223c 100644 --- a/jmclient/jmclient/taker.py +++ b/jmclient/jmclient/taker.py @@ -301,7 +301,8 @@ class Taker(object): #Construct the Bitcoin address for the auth_pub field #Ensure that at least one address from utxos corresponds. input_addresses = [d['address'] for d in utxo_data] - auth_address = btc.pubkey_to_address(auth_pub, get_p2pk_vbyte()) + auth_address = btc.pubkey_to_p2sh_p2wpkh_address(auth_pub, + get_p2sh_vbyte()) if not auth_address in input_addresses: jlog.warn("ERROR maker's (" + nick + ")" " authorising pubkey is not included " diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index 8139cef..aedba99 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -385,6 +385,8 @@ class Wallet(AbstractWallet): class Bip39Wallet(Wallet): def entropy_to_seed(self, entropy): + if get_network() == "testnet": + return entropy self.entropy = entropy.decode('hex') m = Mnemonic("english") return m.to_seed(m.to_mnemonic(self.entropy)).encode('hex') diff --git a/jmdaemon/jmdaemon/daemon_protocol.py b/jmdaemon/jmdaemon/daemon_protocol.py index fc8c81b..9321ef5 100644 --- a/jmdaemon/jmdaemon/daemon_protocol.py +++ b/jmdaemon/jmdaemon/daemon_protocol.py @@ -6,10 +6,12 @@ from .orderbookwatch import OrderbookWatch from .enc_wrapper import (as_init_encryption, init_keypair, init_pubkey, NaclError) from .protocol import (COMMAND_PREFIX, ORDER_KEYS, NICK_HASH_LENGTH, - NICK_MAX_ENCODED, JM_VERSION, JOINMARKET_NICK_HEADER) + NICK_MAX_ENCODED, JM_VERSION, JOINMARKET_NICK_HEADER, + COMMITMENT_PREFIXES) from .irc import IRCMessageChannel from jmbase.commands import * +from jmbase import _byteify from twisted.protocols import amp from twisted.internet import reactor from twisted.internet.protocol import ServerFactory @@ -19,7 +21,35 @@ from twisted.python import failure, log import json import time import threading - +import os +import copy + +def check_utxo_blacklist(commitment, persist=False): + """Compare a given commitment with the persisted blacklist log file, + which is hardcoded to this directory and name 'commitmentlist' (no + security or privacy issue here). + If the commitment has been used before, return False (disallowed), + else return True. + If flagged, persist the usage of this commitment to the above file. + """ + #TODO format error checking? + fname = "commitmentlist" + if os.path.isfile(fname): + with open(fname, "rb") as f: + blacklisted_commitments = [x.strip() for x in f.readlines()] + else: + blacklisted_commitments = [] + if commitment in blacklisted_commitments: + return False + elif persist: + blacklisted_commitments += [commitment] + with open(fname, "wb") as f: + f.write('\n'.join(blacklisted_commitments)) + f.flush() + #If the commitment is new and we are *not* persisting, nothing to do + #(we only add it to the list on sending io_auth, which represents actual + #usage). + return True """Joinmarket application protocol control flow. For documentation on protocol (formats, message sequence) see @@ -45,8 +75,10 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): self.restart_mc_required = False self.irc_configs = None self.mcc = None + self.role = "TAKER" self.crypto_boxes = {} self.sig_lock = threading.Lock() + self.active_orders = {} def checkClientResponse(self, response): """A generic check of client acceptance; any failure @@ -95,6 +127,13 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): #register taker-specific msgchan callbacks here self.mcc.register_taker_callbacks(self.on_error, self.on_pubkey, self.on_ioauth, self.on_sig) + self.mcc.register_maker_callbacks(self.on_orderbook_requested, + self.on_order_fill, + self.on_seen_auth, + self.on_seen_tx, + self.on_push_tx, + self.on_commitment_seen, + self.on_commitment_transferred) self.mcc.set_daemon(self) d = self.callRemote(JMInitProto, nick_hash_length=NICK_HASH_LENGTH, @@ -137,12 +176,10 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): self.defaultCallbacks(d) @JMSetup.responder - def on_JM_SETUP(self, role, n_counterparties): + def on_JM_SETUP(self, role, initdata): assert self.jm_state == 0 - assert n_counterparties > 1 #TODO consider MAKER role implementation here - assert role == "TAKER" - self.requested_counterparties = n_counterparties + self.role = role self.crypto_boxes = {} self.kp = init_keypair() print("Received setup command") @@ -152,10 +189,26 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): #assumes messagechannels are in "up" state. Orders are read #in the callback on_order_seen in OrderbookWatch. #TODO: pubmsg should not (usually?) fire if already up from previous run. - self.mcc.pubmsg(COMMAND_PREFIX + "orderbook") + if self.role == "TAKER": + self.mcc.pubmsg(COMMAND_PREFIX + "orderbook") + elif self.role == "MAKER": + self.offerlist = json.loads(initdata) + self.mcc.announce_orders(self.offerlist) self.jm_state = 1 return {'accepted': True} + @JMMakerPubkey.responder + def on_JM_MAKER_PUBKEY(self, nick, pubkey): + self.mcc.prepare_privmsg(nick, "pubkey", pubkey) + + @JMTXSigs.responder + def on_JM_TX_SIGS(self, nick, sigs): + sigs = _byteify(json.loads(sigs)) + print('sending sigs: ' + str(sigs)) + for sig in sigs: + self.mcc.prepare_privmsg(nick, "sig", sig) + return {"accepted": True} + @JMRequestOffers.responder def on_JM_REQUEST_OFFERS(self): """Reports the current state of the orderbook. @@ -187,6 +240,161 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): self.jm_state = 2 return {'accepted': True} + @JMAnnounceOffers.responder + def on_JM_ANNOUNCE_OFFERS(self, offerlist): + if self.role != "MAKER": + return + self.offerlist = json.loads(offerlist) + return {"accepted": True} + + def on_orderbook_requested(self, nick, mc=None): + """Dealt with by daemon, assuming offerlist is up to date + """ + self.mcc.announce_orders(self.offerlist, nick, mc) + + def on_order_fill(self, nick, oid, amount, taker_pk, commit): + """Handled locally in daemon. + """ + if self.role != "MAKER": return + if nick in self.active_orders: + log.msg("Restarting transaction for nick: " + nick) + if not commit[0] in COMMITMENT_PREFIXES: + self.mcc.send_error(nick, + "Unsupported commitment type: " + str(commit[0])) + return + scommit = commit[1:] + if not check_utxo_blacklist(scommit): + log.msg("Taker utxo commitment is blacklisted, rejecting.") + self.mcc.send_error(nick, "Commitment is blacklisted: " + str(scommit)) + #Note that broadcast is happening here to reflect an already + #consumed commitment; it can also be broadcast separately (earlier) on + #valid usage + #Keep the type byte for communication so not scommit: + self.transfer_commitment(commit) + return + offer_s = [o for o in self.offerlist if o['oid'] == oid] + if len(offer_s) == 0: + self.mcc.send_error(nick, 'oid not found') + offer = offer_s[0] + if amount < offer['minsize'] or amount > offer['maxsize']: + self.mcc.send_error(nick, 'amount out of range') + #prepare a pubkey for this valid transaction + kp = init_keypair() + try: + crypto_box = as_init_encryption(kp, init_pubkey(taker_pk)) + self.active_orders[nick] = {"crypto_box": crypto_box, + "kp": kp, + "offer": offer, + "amount": amount, + "commit": scommit} + except NaclError as e: + log.msg("Unable to set up cryptobox with counterparty: " + repr(e)) + self.mcc.send_error(nick, "Invalid nacl pubkey: " + taker_pk) + self.mcc.prepare_privmsg(nick, "pubkey", kp.hex_pk()) + + def on_seen_auth(self, nick, commitment_revelation): + if not self.role == "MAKER": + return + if not nick in self.active_orders: + return + ao =self.active_orders[nick] + #ask the client to validate the commitment and prepare the utxo data + d = self.callRemote(JMAuthReceived, + nick=nick, + offer=json.dumps(ao["offer"]), + commitment=ao["commit"], + revelation=json.dumps(commitment_revelation), + amount=ao["amount"], + kphex=ao["kp"].hex_pk()) + self.defaultCallbacks(d) + + @JMIOAuth.responder + def on_JM_IOAUTH(self, nick, utxolist, pubkey, cjaddr, changeaddr, pubkeysig): + nick, utxolist, pubkey, cjaddr, changeaddr, pubkeysig = [_byteify( + x) for x in nick, utxolist, pubkey, cjaddr, changeaddr, pubkeysig] + if not self.role == "MAKER": + return + if not nick in self.active_orders: + return + utxos= json.loads(utxolist) + #completed population of order/offer object + self.active_orders[nick]["cjaddr"] = cjaddr + self.active_orders[nick]["changeaddr"] = changeaddr + self.active_orders[nick]["utxos"] = utxos + msg = str(",".join(utxos.keys())) + " " + " ".join( + [pubkey, cjaddr, changeaddr, pubkeysig]) + self.mcc.prepare_privmsg(nick, "ioauth", msg) + #In case of *blacklisted (ie already used) commitments, we already + #broadcasted them on receipt; in case of valid, and now used commitments, + #we broadcast them here, and not early - to avoid accidentally + #blacklisting commitments that are broadcast between makers in real time + #for the same transaction. + self.transfer_commitment(self.active_orders[nick]["commit"]) + #now persist the fact that the commitment is actually used. + check_utxo_blacklist(self.active_orders[nick]["commit"], persist=True) + return {"accepted": True} + + def transfer_commitment(self, commit): + """Send this commitment via privmsg to one (random) + other maker. + """ + crow = self.db.execute( + 'SELECT DISTINCT counterparty FROM orderbook ORDER BY ' + + 'RANDOM() LIMIT 1;' + ).fetchone() + if crow is None: + return + counterparty = crow['counterparty'] + #TODO de-hardcode hp2 + log.msg("Sending commitment to: " + str(counterparty)) + self.mcc.prepare_privmsg(counterparty, 'hp2', commit) + + def on_commitment_seen(self, nick, commitment): + """Triggered when we see a commitment for blacklisting + appear in the public pit channel. If the policy is set, + we blacklist this commitment. + """ + if jm_single().config.has_option("POLICY", "accept_commitment_broadcasts"): + blacklist_add = jm_single().config.getint("POLICY", + "accept_commitment_broadcasts") + else: + blacklist_add = 0 + if blacklist_add > 0: + #just add if necessary, ignore return value. + check_utxo_blacklist(commitment, persist=True) + log.msg("Received commitment broadcast by other maker: " + str( + commitment) + ", now blacklisted.") + else: + log.msg("Received commitment broadcast by other maker: " + str( + commitment) + ", ignored.") + + def on_commitment_transferred(self, nick, commitment): + """Triggered when a privmsg is received from another maker + with a commitment to announce in public (obfuscation of source). + We simply post it in public (not affected by whether we ourselves + are *accepting* commitment broadcasts. + """ + self.mcc.pubmsg("!hp2 " + commitment) + + def on_push_tx(self, nick, txhex): + log.msg('received pushtx message, ignoring, TODO') + + def on_seen_tx(self, nick, txhex): + if self.role != "MAKER": + return + if nick not in self.active_orders: + return + #we send a copy of the entire "active_orders" entry except the cryptobox, + #so make a temporary copy + ao = copy.deepcopy(self.active_orders[nick]) + del ao["crypto_box"] + del ao["kp"] + d = self.callRemote(JMTXReceived, + nick=nick, + txhex=txhex, + offer=json.dumps(ao)) + self.defaultCallbacks(d) + def on_pubkey(self, nick, maker_pk): """This is handled locally in the daemon; set up e2e encrypted messaging with this counterparty @@ -327,15 +535,20 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): return {'accepted': True} def get_crypto_box_from_nick(self, nick): + """Retrieve the libsodium box object for the counterparty; + stored differently for Taker and Maker + """ if nick in self.crypto_boxes and self.crypto_boxes[nick] != None: - return self.crypto_boxes[nick][1] # libsodium encryption object + return self.crypto_boxes[nick][1] + elif nick in self.active_orders and self.active_orders[nick] != None: + return self.active_orders[nick]["crypto_box"] else: log.msg('something wrong, no crypto object, nick=' + nick + ', message will be dropped') return None - def on_error(self): - log.msg("Unimplemented on_error") + def on_error(self, msg): + log.msg("Received error: " + str(msg)) def mc_shutdown(self): log.msg("Message channels being shutdown by daemon") diff --git a/jmdaemon/jmdaemon/message_channel.py b/jmdaemon/jmdaemon/message_channel.py index 245f783..c08088d 100644 --- a/jmdaemon/jmdaemon/message_channel.py +++ b/jmdaemon/jmdaemon/message_channel.py @@ -213,6 +213,7 @@ class MessageChannelCollection(object): else: return self.daemon.get_crypto_box_from_nick(nick), True + @check_privmsg def prepare_privmsg(self, nick, cmd, message, mc=None): # should we encrypt? box, encrypt = self.get_encryption_box(cmd, nick) @@ -294,24 +295,11 @@ class MessageChannelCollection(object): msg = ' '.join(orderlines[0].split(' ')[1:]) msg += ''.join(orderlines[1:]) if new_mc: - self.privmsg(nick, cmd, msg, new_mc) + self.prepare_privmsg(nick, cmd, msg, mc=new_mc) else: for mc in self.available_channels(): if nick in self.nicks_seen[mc]: - self.privmsg(nick, cmd, msg, mc) - - @check_privmsg - def send_pubkey(self, nick, pubkey): - self.active_channels[nick].privmsg(nick, 'pubkey', pubkey) - - @check_privmsg - def send_ioauth(self, nick, utxo_list, auth_pub, cj_addr, change_addr, sig): - self.active_channels[nick].send_ioauth(nick, utxo_list, auth_pub, - cj_addr, change_addr, sig) - - @check_privmsg - def send_sigs(self, nick, sig_list): - self.active_channels[nick].send_sigs(nick, sig_list) + self.prepare_privmsg(nick, cmd, msg, mc=mc) # Taker callbacks def fill_orders(self, nick_order_dict, cj_amount, taker_pubkey, commitment): @@ -337,7 +325,8 @@ class MessageChannelCollection(object): @check_privmsg def send_error(self, nick, errormsg): #TODO this might need to support non-active nicks TODO - self.active_channels[nick].send_error(nick, errormsg) + if nick in self.active_channels: + self.active_channels[nick].send_error(nick, errormsg) @check_privmsg def push_tx(self, nick, txhex): @@ -792,19 +781,6 @@ class MessageChannel(object): clines = [COMMAND_PREFIX + 'cancel ' + str(oid) for oid in oid_list] self.pubmsg(''.join(clines)) - def send_pubkey(self, nick, pubkey): - self.privmsg(nick, 'pubkey', pubkey) - - def send_ioauth(self, nick, utxo_list, auth_pub, cj_addr, change_addr, sig): - authmsg = str(','.join(utxo_list)) + ' ' + ' '.join([auth_pub, cj_addr, - change_addr, sig]) - self.privmsg(nick, 'ioauth', authmsg) - - def send_sigs(self, nick, sig_list): - # TODO make it send the sigs on one line if there's space - for s in sig_list: - self.privmsg(nick, 'sig', s) - # OrderbookWatch callback def request_orderbook(self): self.pubmsg(COMMAND_PREFIX + 'orderbook') @@ -937,6 +913,10 @@ class MessageChannel(object): if self.check_for_orders(nick, _chunks): pass # taker commands + elif _chunks[0] == 'error': + error = " ".join(_chunks[1:]) + if self.on_error: + self.on_error(error) elif _chunks[0] == 'pubkey': maker_pk = _chunks[1] if self.on_pubkey: diff --git a/jmdaemon/jmdaemon/protocol.py b/jmdaemon/jmdaemon/protocol.py index f7f71b8..914471d 100644 --- a/jmdaemon/jmdaemon/protocol.py +++ b/jmdaemon/jmdaemon/protocol.py @@ -1,5 +1,5 @@ #Protocol version -JM_VERSION = 5 +JM_VERSION = 6 #Username on all messagechannels; will be set in MessageChannelCollection nickname = None @@ -24,6 +24,8 @@ JOINMARKET_NICK_HEADER = 'J' NICK_HASH_LENGTH = 10 NICK_MAX_ENCODED = 14 #comes from base58 expansion; recalculate if above changes +#commitments; note multiple options may be used in future +COMMITMENT_PREFIXES = ["P"] #Lists of valid commands encrypted_commands = ["auth", "ioauth", "tx", "sig"] plaintext_commands = ["fill", "error", "pubkey", "orderbook", "push"] diff --git a/scripts/sendpayment.py b/scripts/sendpayment.py index 6a216e4..1da84ed 100644 --- a/scripts/sendpayment.py +++ b/scripts/sendpayment.py @@ -18,7 +18,7 @@ import os import pprint from jmclient import (Taker, load_program_config, get_schedule, - JMTakerClientProtocolFactory, start_reactor, + JMClientProtocolFactory, start_reactor, validate_address, jm_single, WalletError, choose_orders, choose_sweep_orders, cheapest_order_choose, weighted_order_choose, @@ -197,7 +197,7 @@ def main(): schedule, order_chooser=chooseOrdersFunc, callbacks=(filter_orders_callback, None, taker_finished)) - clientfactory = JMTakerClientProtocolFactory(taker) + clientfactory = JMClientProtocolFactory(taker) nodaemon = jm_single().config.getint("DAEMON", "no_daemon") daemon = True if nodaemon == 1 else False start_reactor(jm_single().config.get("DAEMON", "daemon_host"),