Added yieldgen functionality, both sides using segwit wallets.

jmbase/jmbase/__init__.py

@@ -1,6 +1,6 @@
 from __future__ import print_function
 
 from .support import (get_log, chunks, debug_silence, debug_dump_object,
-                      joinmarket_alert, core_alert, get_password)
+                      joinmarket_alert, core_alert, get_password, _byteify)
 from commands import *
 

jmbase/jmbase/commands.py

@@ -37,7 +37,10 @@ class JMStartMC(JMCommand):
 
 class JMSetup(JMCommand):
     arguments = [('role', String()),
-                 ('n_counterparties', Integer())]
+                 ('initdata', String())]
+
+"""Taker specific commands
+"""
 
 class JMRequestOffers(JMCommand):
     arguments = []
@@ -67,6 +70,28 @@ class JMMsgSignatureVerify(JMCommand):
                  ('nick', String()),
                  ('fullmsg', String()),
                  ('hostid', String())]
+
+"""Maker specific commands
+"""
+
+class JMAnnounceOffers(JMCommand):
+    arguments = [('offerlist', String())]
+
+class JMMakerPubkey(JMCommand):
+    arguments = [('nick', String()),
+                 ('pubkey', String())]
+
+class JMIOAuth(JMCommand):
+    arguments = [('nick', String()),
+                 ('utxolist', String()),
+                 ('pubkey', String()),
+                 ('cjaddr', String()),
+                 ('changeaddr', String()),
+                 ('pubkeysig', String())]
+
+class JMTXSigs(JMCommand):
+    arguments = [('nick', String()),
+                 ('sigs', String())]
     
 #commands from daemon to client
 
@@ -93,6 +118,26 @@ class JMSigReceived(JMCommand):
     arguments = [('nick', String()),
                  ('sig', String())]
 
+"""Maker specific daemon-client messages
+"""
+
+class JMCommitmentOpen(JMCommand):
+    arguments = [('nick', String()),
+                 ('revelation', String())]
+
+class JMAuthReceived(JMCommand):
+    arguments = [('nick', String()),
+                 ('offer', String()),
+                 ('commitment', String()),
+                 ('revelation', String()),
+                 ('amount', Integer()),
+                 ('kphex', String())]
+
+class JMTXReceived(JMCommand):
+    arguments = [('nick', String()),
+                 ('txhex', String()),
+                 ('offer', String())]
+
 class JMRequestMsgSig(JMCommand):
     arguments = [('nick', String()),
                  ('cmd', String()),

jmbase/jmbase/support.py

@@ -68,3 +68,20 @@ def debug_dump_object(obj, skip_fields=None):
             log.debug(pprint.pformat(v))
         else:
             log.debug(str(v))
+
+def _byteify(data, ignore_dicts = False):
+    # if this is a unicode string, return its string representation
+    if isinstance(data, unicode):
+        return data.encode('utf-8')
+    # if this is a list of values, return list of byteified values
+    if isinstance(data, list):
+        return [ _byteify(item, ignore_dicts=True) for item in data ]
+    # if this is a dictionary, return dictionary of byteified keys and values
+    # but only if we haven't already byteified it
+    if isinstance(data, dict) and not ignore_dicts:
+        return {
+            _byteify(key, ignore_dicts=True): _byteify(value, ignore_dicts=True)
+            for key, value in data.iteritems()
+        }
+    # if it's anything else, return it in its original form
+    return data

jmclient/jmclient/__init__.py

@@ -21,12 +21,11 @@ from .wallet import (AbstractWallet, BitcoinCoreInterface, Wallet,
                      create_wallet_file, SegwitWallet, Bip39Wallet)
 from .configure import (load_program_config, jm_single, get_p2pk_vbyte,
     get_network, jm_single, get_network, validate_address, get_irc_mchannels,
-    check_utxo_blacklist, get_blockchain_interface_instance, get_p2sh_vbyte,
-    set_config)
+    get_blockchain_interface_instance, get_p2sh_vbyte, set_config)
 from .blockchaininterface import (BlockrInterface, BlockchainInterface, sync_wallet,
                                   RegtestBitcoinCoreInterface, BitcoinCoreInterface)
 from .electruminterface import ElectrumInterface
-from .client_protocol import JMTakerClientProtocolFactory, start_reactor
+from .client_protocol import (JMClientProtocolFactory, start_reactor)
 from .podle import (set_commitment_file, get_commitment_file,
                     generate_podle_error_string, add_external_commitments,
                     PoDLE, generate_podle, get_podle_commitments,
@@ -39,6 +38,8 @@ from .taker_utils import (tumbler_taker_finished_update, restart_waiter,
                              restart_wait, get_tumble_log, direct_send,
                              tumbler_filter_orders_callback)
 from .wallet_utils import wallet_tool_main
+from .maker import Maker
+from .yieldgenerator import YieldGenerator, ygmain
 # Set default logging handler to avoid "No handler found" warnings.
 
 try:

jmclient/jmclient/client_protocol.py

@@ -34,22 +34,16 @@ jlog = get_log()
 class JMProtocolError(Exception):
     pass
 
-class JMTakerClientProtocol(amp.AMP):
-
-    def __init__(self, factory, taker, nick_priv=None):
-        self.taker = taker
-        self.factory = factory
-        self.orderbook = None
-        self.supported_messages = ["JM_UP", "JM_SETUP_DONE", "JM_FILL_RESPONSE",
-                                   "JM_OFFERS", "JM_SIG_RECEIVED",
-                                   "JM_REQUEST_MSGSIG",
-                                   "JM_REQUEST_MSGSIG_VERIFY", "JM_INIT_PROTO"]
-        if not nick_priv:
-            self.nick_priv = hashlib.sha256(os.urandom(16)).hexdigest() + '01'
-        else:
-            self.nick_priv = nick_priv
+class JMClientProtocol(amp.AMP):
+    def __init__(self, factory, client, nick_priv=None):
+            self.client = client
+            self.factory = factory
+            if not nick_priv:
+                self.nick_priv = hashlib.sha256(os.urandom(16)).hexdigest() + '01'
+            else:
+                self.nick_priv = nick_priv
 
-        self.shutdown_requested = False
+            self.shutdown_requested = False
 
     def checkClientResponse(self, response):
         """A generic check of client acceptance; any failure
@@ -72,12 +66,173 @@ class JMTakerClientProtocol(amp.AMP):
         self.factory.setClient(self)
         self.clientStart()
 
+    def set_nick(self):
+        self.nick_pubkey = btc.privtopub(self.nick_priv)
+        self.nick_pkh_raw = hashlib.sha256(self.nick_pubkey).digest()[
+                    :self.nick_hashlen]
+        self.nick_pkh = btc.changebase(self.nick_pkh_raw, 256, 58)
+        #right pad to maximum possible; b58 is not fixed length.
+        #Use 'O' as one of the 4 not included chars in base58.
+        self.nick_pkh += 'O' * (self.nick_maxencoded - len(self.nick_pkh))
+        #The constructed length will be 1 + 1 + NICK_MAX_ENCODED
+        self.nick = self.nick_header + str(self.jm_version) + self.nick_pkh
+        jm_single().nickname = self.nick
+
+    @commands.JMInitProto.responder
+    def on_JM_INIT_PROTO(self, nick_hash_length, nick_max_encoded,
+                         joinmarket_nick_header, joinmarket_version):
+        """Daemon indicates init-ed status and passes back protocol constants.
+        Use protocol settings to set actual nick from nick private key,
+        then call setup to instantiate message channel connections in the daemon.
+        """
+        self.nick_hashlen = nick_hash_length
+        self.nick_maxencoded = nick_max_encoded
+        self.nick_header = joinmarket_nick_header
+        self.jm_version = joinmarket_version
+        self.set_nick()
+        d = self.callRemote(commands.JMStartMC,
+                            nick=self.nick)
+        self.defaultCallbacks(d)
+        return {'accepted': True}
+
+    @commands.JMRequestMsgSig.responder
+    def on_JM_REQUEST_MSGSIG(self, nick, cmd, msg, msg_to_be_signed, hostid):
+        sig = btc.ecdsa_sign(str(msg_to_be_signed), self.nick_priv)
+        msg_to_return = str(msg) + " " + self.nick_pubkey + " " + sig
+        d = self.callRemote(commands.JMMsgSignature,
+                            nick=nick,
+                            cmd=cmd,
+                            msg_to_return=msg_to_return,
+                            hostid=hostid)
+        self.defaultCallbacks(d)
+        return {'accepted': True}
+
+    @commands.JMRequestMsgSigVerify.responder
+    def on_JM_REQUEST_MSGSIG_VERIFY(self, msg, fullmsg, sig, pubkey, nick,
+                                    hashlen, max_encoded, hostid):
+        verif_result = True
+        if not btc.ecdsa_verify(str(msg), sig, pubkey):
+            jlog.debug("nick signature verification failed, ignoring.")
+            verif_result = False
+        #check that nick matches hash of pubkey
+        nick_pkh_raw = hashlib.sha256(pubkey).digest()[:hashlen]
+        nick_stripped = nick[2:2 + max_encoded]
+        #strip right padding
+        nick_unpadded = ''.join([x for x in nick_stripped if x != 'O'])
+        if not nick_unpadded == btc.changebase(nick_pkh_raw, 256, 58):
+            jlog.debug("Nick hash check failed, expected: " + str(nick_unpadded)
+                       + ", got: " + str(btc.changebase(nick_pkh_raw, 256, 58)))
+            verif_result = False
+        d = self.callRemote(commands.JMMsgSignatureVerify,
+                            verif_result=verif_result,
+                            nick=nick,
+                            fullmsg=fullmsg,
+                            hostid=hostid)
+        self.defaultCallbacks(d)
+        return {'accepted': True}
+
+class JMMakerClientProtocol(JMClientProtocol):
+    def __init__(self, factory, maker, nick_priv=None):
+        self.factory = factory
+        JMClientProtocol.__init__(self, factory, maker, nick_priv)
+
+    @commands.JMUp.responder
+    def on_JM_UP(self):
+        d = self.callRemote(commands.JMSetup,
+                            role="MAKER",
+                            initdata=json.dumps(self.client.offerlist))
+        #for as long as the maker is up, it can asynchronously pass through
+        #its updated offer list
+        offer_loop = LoopingCall(self.get_offers)
+        offer_loop.start(3.0)
+        self.defaultCallbacks(d)
+        return {'accepted': True}
+
+    def get_offers(self):
+        """Feeds through current offers from Maker obect, to the daemon
+        """
+        offerlist = self.client.offerlist
+        d = self.callRemote(commands.JMAnnounceOffers,
+                            offerlist=json.dumps(offerlist))
+        self.defaultCallbacks(d)
+
+    @commands.JMSetupDone.responder
+    def on_JM_SETUP_DONE(self):
+        jlog.info("JM daemon setup complete")
+        return {'accepted': True}
+
     def clientStart(self):
         """Upon confirmation of network connection
         to daemon, request message channel initialization
         with relevant config data for our message channels
         """
-        if self.taker.aborted:
+        if self.client.aborted:
+            return
+        #needed only for naming convention in IRC currently
+        blockchain_source = jm_single().config.get("BLOCKCHAIN",
+                                                   "blockchain_source")
+        #needed only for channel naming convention
+        network = jm_single().config.get("BLOCKCHAIN", "network")
+        irc_configs = get_irc_mchannels()
+        #only here because Init message uses this field; not used by makers TODO
+        minmakers = jm_single().config.getint("POLICY", "minimum_makers")
+        maker_timeout_sec = jm_single().maker_timeout_sec
+
+        d = self.callRemote(commands.JMInit,
+                            bcsource=blockchain_source,
+                            network=network,
+                            irc_configs=json.dumps(irc_configs),
+                            minmakers=minmakers,
+                            maker_timeout_sec=maker_timeout_sec)
+        self.defaultCallbacks(d)
+
+    @commands.JMAuthReceived.responder
+    def on_JM_AUTH_RECEIVED(self, nick, offer, commitment, revelation, amount,
+                            kphex):
+        offer = json.loads(offer)
+        revelation = json.loads(revelation)
+        retval = self.client.on_auth_received(nick, offer,
+                                            commitment, revelation, amount, kphex)
+        if not retval[0]:
+            jlog.info("Maker refuses to continue on receiving auth.")
+        else:
+            utxos, auth_pub, cj_addr, change_addr, btc_sig = retval[1:]
+            d = self.callRemote(commands.JMIOAuth,
+                                nick=nick,
+                                utxolist=json.dumps(utxos),
+                                pubkey=auth_pub,
+                                cjaddr=cj_addr,
+                                changeaddr=change_addr,
+                                pubkeysig=btc_sig)
+            self.defaultCallbacks(d)
+        return {"accepted": True}
+
+    @commands.JMTXReceived.responder
+    def on_JM_TX_RECEIVED(self, nick, txhex, offer):
+        offer = json.loads(offer)
+        retval = self.client.on_tx_received(nick, txhex, offer)
+        if not retval[0]:
+            jlog.info("Maker refuses to continue on receipt of tx")
+        else:
+            sigs = retval[1]
+            d = self.callRemote(commands.JMTXSigs,
+                                nick=nick,
+                                sigs=json.dumps(sigs))
+            self.defaultCallbacks(d)
+        return {"accepted": True}
+
+class JMTakerClientProtocol(JMClientProtocol):
+
+    def __init__(self, factory, client, nick_priv=None):
+        self.orderbook = None
+        JMClientProtocol.__init__(self, factory, client, nick_priv)
+
+    def clientStart(self):
+        """Upon confirmation of network connection
+        to daemon, request message channel initialization
+        with relevant config data for our message channels
+        """
+        if self.client.aborted:
             return
         #needed only for naming convention in IRC currently
         blockchain_source = jm_single().config.get("BLOCKCHAIN",
@@ -90,9 +245,9 @@ class JMTakerClientProtocol(amp.AMP):
 
         #To avoid creating yet another config variable, we set the timeout
         #to 20 * maker_timeout_sec.
-        if not hasattr(self.taker, 'testflag'): #pragma: no cover
+        if not hasattr(self.client, 'testflag'): #pragma: no cover
             reactor.callLater(20*maker_timeout_sec, self.stallMonitor,
-                          self.taker.schedule_index+1)
+                          self.client.schedule_index+1)
 
         d = self.callRemote(commands.JMInit,
                             bcsource=blockchain_source,
@@ -112,60 +267,31 @@ class JMTakerClientProtocol(amp.AMP):
         on to the next item before we were woken up.
         """
         jlog.info("STALL MONITOR:")
-        if self.taker.aborted:
+        if self.client.aborted:
             jlog.info("Transaction was aborted.")
             return
-        if not self.taker.schedule_index == schedule_index:
+        if not self.client.schedule_index == schedule_index:
             #TODO pre-initialize() ?
             jlog.info("No stall detected, continuing")
             return
-        if self.taker.waiting_for_conf:
+        if self.client.waiting_for_conf:
             #Don't restart if the tx is already on the network!
             jlog.info("No stall detected, continuing")
             return
-        if not self.taker.txid:
+        if not self.client.txid:
             #txid is set on pushing; if it's not there, we have failed.
             jlog.info("Stall detected. Regenerating transactions and retrying.")
-            self.taker.on_finished_callback(False, True, 0.0)
+            self.client.on_finished_callback(False, True, 0.0)
         else:
             #This shouldn't really happen; if the tx confirmed,
             #the finished callback should already be called.
             jlog.info("Tx was already pushed; ignoring")
 
-    def set_nick(self):
-        self.nick_pubkey = btc.privtopub(self.nick_priv)
-        self.nick_pkh_raw = hashlib.sha256(self.nick_pubkey).digest()[
-                    :self.nick_hashlen]
-        self.nick_pkh = btc.changebase(self.nick_pkh_raw, 256, 58)
-        #right pad to maximum possible; b58 is not fixed length.
-        #Use 'O' as one of the 4 not included chars in base58.
-        self.nick_pkh += 'O' * (self.nick_maxencoded - len(self.nick_pkh))
-        #The constructed length will be 1 + 1 + NICK_MAX_ENCODED
-        self.nick = self.nick_header + str(self.jm_version) + self.nick_pkh
-        jm_single().nickname = self.nick
-
-    @commands.JMInitProto.responder
-    def on_JM_INIT_PROTO(self, nick_hash_length, nick_max_encoded,
-                         joinmarket_nick_header, joinmarket_version):
-        """Daemon indicates init-ed status and passes back protocol constants.
-        Use protocol settings to set actual nick from nick private key,
-        then call setup to instantiate message channel connections in the daemon.
-        """
-        self.nick_hashlen = nick_hash_length
-        self.nick_maxencoded = nick_max_encoded
-        self.nick_header = joinmarket_nick_header
-        self.jm_version = joinmarket_version
-        self.set_nick()
-        d = self.callRemote(commands.JMStartMC,
-                            nick=self.nick)
-        self.defaultCallbacks(d)
-        return {'accepted': True}
-
     @commands.JMUp.responder
     def on_JM_UP(self):
         d = self.callRemote(commands.JMSetup,
                             role="TAKER",
-                            n_counterparties=4) #TODO this number should be set
+                            initdata="none")
         self.defaultCallbacks(d)
         return {'accepted': True}
 
@@ -189,11 +315,11 @@ class JMTakerClientProtocol(amp.AMP):
         if not success:
             nonresponders = ioauth_data
             jlog.info("Makers didnt respond: " + str(nonresponders))
-            self.taker.add_ignored_makers(nonresponders)
+            self.client.add_ignored_makers(nonresponders)
             return {'accepted': True}
         else:
             jlog.info("Makers responded with: " + json.dumps(ioauth_data))
-            retval = self.taker.receive_utxos(ioauth_data)
+            retval = self.client.receive_utxos(ioauth_data)
             if not retval[0]:
                 jlog.info("Taker is not continuing, phase 2 abandoned.")
                 jlog.info("Reason: " + str(retval[1]))
@@ -208,7 +334,7 @@ class JMTakerClientProtocol(amp.AMP):
         self.orderbook = json.loads(orderbook)
         #Removed for now, as judged too large, even for DEBUG:
         #jlog.debug("Got the orderbook: " + str(self.orderbook))
-        retval = self.taker.initialize(self.orderbook)
+        retval = self.client.initialize(self.orderbook)
         #format of retval is:
         #True, self.cjamount, commitment, revelation, self.filtered_orderbook)
         if not retval[0]:
@@ -225,48 +351,12 @@ class JMTakerClientProtocol(amp.AMP):
 
     @commands.JMSigReceived.responder
     def on_JM_SIG_RECEIVED(self, nick, sig):
-        retval = self.taker.on_sig(nick, sig)
+        retval = self.client.on_sig(nick, sig)
         if retval:
             nick_to_use, txhex = retval
             self.push_tx(nick_to_use, txhex)
         return {'accepted': True}
 
-    @commands.JMRequestMsgSig.responder
-    def on_JM_REQUEST_MSGSIG(self, nick, cmd, msg, msg_to_be_signed, hostid):
-        sig = btc.ecdsa_sign(str(msg_to_be_signed), self.nick_priv)
-        msg_to_return = str(msg) + " " + self.nick_pubkey + " " + sig
-        d = self.callRemote(commands.JMMsgSignature,
-                            nick=nick,
-                            cmd=cmd,
-                            msg_to_return=msg_to_return,
-                            hostid=hostid)
-        self.defaultCallbacks(d)
-        return {'accepted': True}
-
-    @commands.JMRequestMsgSigVerify.responder
-    def on_JM_REQUEST_MSGSIG_VERIFY(self, msg, fullmsg, sig, pubkey, nick,
-                                    hashlen, max_encoded, hostid):
-        verif_result = True
-        if not btc.ecdsa_verify(str(msg), sig, pubkey):
-            jlog.debug("nick signature verification failed, ignoring.")
-            verif_result = False
-        #check that nick matches hash of pubkey
-        nick_pkh_raw = hashlib.sha256(pubkey).digest()[:hashlen]
-        nick_stripped = nick[2:2 + max_encoded]
-        #strip right padding
-        nick_unpadded = ''.join([x for x in nick_stripped if x != 'O'])
-        if not nick_unpadded == btc.changebase(nick_pkh_raw, 256, 58):
-            jlog.debug("Nick hash check failed, expected: " + str(nick_unpadded)
-                       + ", got: " + str(btc.changebase(nick_pkh_raw, 256, 58)))
-            verif_result = False
-        d = self.callRemote(commands.JMMsgSignatureVerify,
-                            verif_result=verif_result,
-                            nick=nick,
-                            fullmsg=fullmsg,
-                            hostid=hostid)
-        self.defaultCallbacks(d)
-        return {'accepted': True}
-
     def get_offers(self):
         d = self.callRemote(commands.JMRequestOffers)
         self.defaultCallbacks(d)
@@ -282,23 +372,28 @@ class JMTakerClientProtocol(amp.AMP):
                             txhex=str(txhex_to_push))
         self.defaultCallbacks(d)
 
-class JMTakerClientProtocolFactory(protocol.ClientFactory):
+class JMClientProtocolFactory(protocol.ClientFactory):
     protocol = JMTakerClientProtocol
 
-    def __init__(self, taker):
-        self.taker = taker
+    def __init__(self, client, proto_type="TAKER"):
+        self.client = client
         self.proto_client = None
+        self.proto_type = proto_type
+        if self.proto_type == "MAKER":
+            self.protocol = JMMakerClientProtocol
+
     def setClient(self, client):
         self.proto_client = client
     def getClient(self):
         return self.proto_client
 
     def buildProtocol(self, addr):
-        return JMTakerClientProtocol(self, self.taker)
-
+        return self.protocol(self, self.client)
 
 def start_reactor(host, port, factory, ish=True, daemon=False): #pragma: no cover
     #(Cannot start the reactor in tests)
+    #Not used in prod (twisted logging):
+    #startLogging(stdout)
     usessl = True if jm_single().config.get("DAEMON", "use_ssl") != 'false' else False
     if daemon:
         try:

jmclient/jmclient/configure.py

@@ -280,37 +280,6 @@ def donation_address(reusable_donation_pubkey=None): #pragma: no cover
     log.debug('sending coins to ' + sender_address)
     return sender_address, sign_k
 
-
-def check_utxo_blacklist(commitment, persist=False):
-    """Compare a given commitment (H(P2) for PoDLE)
-    with the persisted blacklist log file;
-    if it has been used before, return False (disallowed),
-    else return True.
-    If flagged, persist the usage of this commitment to the blacklist file.
-    """
-    #TODO format error checking?
-    fname = "blacklist"
-    if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == 'regtest':
-        fname += "_" + jm_single().nickname
-    with jm_single().blacklist_file_lock:
-        if os.path.isfile(fname):
-            with open(fname, "rb") as f:
-                blacklisted_commitments = [x.strip() for x in f.readlines()]
-        else:
-            blacklisted_commitments = []
-        if commitment in blacklisted_commitments:
-            return False
-        elif persist:
-            blacklisted_commitments += [commitment]
-            with open(fname, "wb") as f:
-                f.write('\n'.join(blacklisted_commitments))
-                f.flush()
-        #If the commitment is new and we are *not* persisting, nothing to do
-        #(we only add it to the list on sending io_auth, which represents actual
-        #usage).
-    return True
-
-
 def load_program_config(config_path=None, bs=None):
     global_singleton.config.readfp(io.BytesIO(defaultconfig))
     if not config_path:

jmclient/jmclient/support.py

@@ -206,13 +206,15 @@ def cheapest_order_choose(orders, n):
 
 
 def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None,
-                  pick=False):
+                  pick=False, allowed_types=["swreloffer", "swabsoffer"]):
     if ignored_makers is None:
         ignored_makers = []
     #Filter ignored makers and inappropriate amounts
     orders = [o for o in offers if o['counterparty'] not in ignored_makers]
     orders = [o for o in orders if o['minsize'] < cj_amount]
     orders = [o for o in orders if o['maxsize'] > cj_amount]
+    #Filter those not using wished-for offertypes
+    orders = [o for o in orders if o["ordertype"] in allowed_types]
     orders_fees = [(
         o, calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount) - o['txfee'])
                    for o in orders]

jmclient/jmclient/taker.py

@@ -301,7 +301,8 @@ class Taker(object):
             #Construct the Bitcoin address for the auth_pub field
             #Ensure that at least one address from utxos corresponds.
             input_addresses = [d['address'] for d in utxo_data]
-            auth_address = btc.pubkey_to_address(auth_pub, get_p2pk_vbyte())
+            auth_address = btc.pubkey_to_p2sh_p2wpkh_address(auth_pub,
+                                                             get_p2sh_vbyte())
             if not auth_address in input_addresses:
                 jlog.warn("ERROR maker's (" + nick + ")"
                          " authorising pubkey is not included "

jmclient/jmclient/wallet.py

@@ -385,6 +385,8 @@ class Wallet(AbstractWallet):
 
 class Bip39Wallet(Wallet):
     def entropy_to_seed(self, entropy):
+        if get_network() == "testnet":
+            return entropy
         self.entropy = entropy.decode('hex')
         m = Mnemonic("english")
         return m.to_seed(m.to_mnemonic(self.entropy)).encode('hex')

jmdaemon/jmdaemon/daemon_protocol.py

@@ -6,10 +6,12 @@ from .orderbookwatch import OrderbookWatch
 from .enc_wrapper import (as_init_encryption, init_keypair, init_pubkey,
                           NaclError)
 from .protocol import (COMMAND_PREFIX, ORDER_KEYS, NICK_HASH_LENGTH,
-                       NICK_MAX_ENCODED, JM_VERSION, JOINMARKET_NICK_HEADER)
+                       NICK_MAX_ENCODED, JM_VERSION, JOINMARKET_NICK_HEADER,
+                       COMMITMENT_PREFIXES)
 from .irc import IRCMessageChannel
 
 from jmbase.commands import *
+from jmbase import _byteify
 from twisted.protocols import amp
 from twisted.internet import reactor
 from twisted.internet.protocol import ServerFactory
@@ -19,7 +21,35 @@ from twisted.python import failure, log
 import json
 import time
 import threading
-
+import os
+import copy
+
+def check_utxo_blacklist(commitment, persist=False):
+    """Compare a given commitment with the persisted blacklist log file,
+    which is hardcoded to this directory and name 'commitmentlist' (no
+    security or privacy issue here).
+    If the commitment has been used before, return False (disallowed),
+    else return True.
+    If flagged, persist the usage of this commitment to the above file.
+    """
+    #TODO format error checking?
+    fname = "commitmentlist"
+    if os.path.isfile(fname):
+        with open(fname, "rb") as f:
+            blacklisted_commitments = [x.strip() for x in f.readlines()]
+    else:
+        blacklisted_commitments = []
+    if commitment in blacklisted_commitments:
+        return False
+    elif persist:
+        blacklisted_commitments += [commitment]
+        with open(fname, "wb") as f:
+            f.write('\n'.join(blacklisted_commitments))
+            f.flush()
+    #If the commitment is new and we are *not* persisting, nothing to do
+    #(we only add it to the list on sending io_auth, which represents actual
+    #usage).
+    return True
 
 """Joinmarket application protocol control flow.
 For documentation on protocol (formats, message sequence) see
@@ -45,8 +75,10 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch):
         self.restart_mc_required = False
         self.irc_configs = None
         self.mcc = None
+        self.role = "TAKER"
         self.crypto_boxes = {}
         self.sig_lock = threading.Lock()
+        self.active_orders = {}
 
     def checkClientResponse(self, response):
         """A generic check of client acceptance; any failure
@@ -95,6 +127,13 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch):
             #register taker-specific msgchan callbacks here
             self.mcc.register_taker_callbacks(self.on_error, self.on_pubkey,
                                               self.on_ioauth, self.on_sig)
+            self.mcc.register_maker_callbacks(self.on_orderbook_requested,
+                                              self.on_order_fill,
+                                              self.on_seen_auth,
+                                              self.on_seen_tx,
+                                              self.on_push_tx,
+                                              self.on_commitment_seen,
+                                              self.on_commitment_transferred)
             self.mcc.set_daemon(self)
         d = self.callRemote(JMInitProto,
                             nick_hash_length=NICK_HASH_LENGTH,
@@ -137,12 +176,10 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch):
         self.defaultCallbacks(d)
 
     @JMSetup.responder
-    def on_JM_SETUP(self, role, n_counterparties):
+    def on_JM_SETUP(self, role, initdata):
         assert self.jm_state == 0
-        assert n_counterparties > 1
         #TODO consider MAKER role implementation here
-        assert role == "TAKER"
-        self.requested_counterparties = n_counterparties
+        self.role = role
         self.crypto_boxes = {}
         self.kp = init_keypair()
         print("Received setup command")
@@ -152,10 +189,26 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch):
         #assumes messagechannels are in "up" state. Orders are read
         #in the callback on_order_seen in OrderbookWatch.
         #TODO: pubmsg should not (usually?) fire if already up from previous run.
-        self.mcc.pubmsg(COMMAND_PREFIX + "orderbook")
+        if self.role == "TAKER":
+            self.mcc.pubmsg(COMMAND_PREFIX + "orderbook")
+        elif self.role == "MAKER":
+            self.offerlist = json.loads(initdata)
+            self.mcc.announce_orders(self.offerlist)
         self.jm_state = 1
         return {'accepted': True}
 
+    @JMMakerPubkey.responder
+    def on_JM_MAKER_PUBKEY(self, nick, pubkey):
+        self.mcc.prepare_privmsg(nick, "pubkey", pubkey)
+
+    @JMTXSigs.responder
+    def on_JM_TX_SIGS(self, nick, sigs):
+        sigs = _byteify(json.loads(sigs))
+        print('sending sigs: ' + str(sigs))
+        for sig in sigs:
+            self.mcc.prepare_privmsg(nick, "sig", sig)
+        return {"accepted": True}
+
     @JMRequestOffers.responder
     def on_JM_REQUEST_OFFERS(self):
         """Reports the current state of the orderbook.
@@ -187,6 +240,161 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch):
         self.jm_state = 2
         return {'accepted': True}
 
+    @JMAnnounceOffers.responder
+    def on_JM_ANNOUNCE_OFFERS(self, offerlist):
+        if self.role != "MAKER":
+            return
+        self.offerlist = json.loads(offerlist)
+        return {"accepted": True}
+
+    def on_orderbook_requested(self, nick, mc=None):
+        """Dealt with by daemon, assuming offerlist is up to date
+        """
+        self.mcc.announce_orders(self.offerlist, nick, mc)
+
+    def on_order_fill(self, nick, oid, amount, taker_pk, commit):
+        """Handled locally in daemon.
+        """
+        if self.role != "MAKER": return
+        if nick in self.active_orders:
+            log.msg("Restarting transaction for nick: " + nick)
+        if not commit[0] in COMMITMENT_PREFIXES:
+            self.mcc.send_error(nick,
+                                "Unsupported commitment type: " + str(commit[0]))
+            return
+        scommit = commit[1:]
+        if not check_utxo_blacklist(scommit):
+            log.msg("Taker utxo commitment is blacklisted, rejecting.")
+            self.mcc.send_error(nick, "Commitment is blacklisted: " + str(scommit))
+            #Note that broadcast is happening here to reflect an already
+            #consumed commitment; it can also be broadcast separately (earlier) on
+            #valid usage
+            #Keep the type byte for communication so not scommit:
+            self.transfer_commitment(commit)
+            return
+        offer_s = [o for o in self.offerlist if o['oid'] == oid]
+        if len(offer_s) == 0:
+            self.mcc.send_error(nick, 'oid not found')
+        offer = offer_s[0]
+        if amount < offer['minsize'] or amount > offer['maxsize']:
+            self.mcc.send_error(nick, 'amount out of range')
+        #prepare a pubkey for this valid transaction
+        kp = init_keypair()
+        try:
+            crypto_box = as_init_encryption(kp, init_pubkey(taker_pk))
+            self.active_orders[nick] = {"crypto_box": crypto_box,
+                                        "kp": kp,
+                                        "offer": offer,
+                                        "amount": amount,
+                                        "commit": scommit}
+        except NaclError as e:
+            log.msg("Unable to set up cryptobox with counterparty: " + repr(e))
+            self.mcc.send_error(nick, "Invalid nacl pubkey: " + taker_pk)
+        self.mcc.prepare_privmsg(nick, "pubkey", kp.hex_pk())
+
+    def on_seen_auth(self, nick, commitment_revelation):
+        if not self.role == "MAKER":
+            return
+        if not nick in self.active_orders:
+            return
+        ao =self.active_orders[nick]
+        #ask the client to validate the commitment and prepare the utxo data
+        d = self.callRemote(JMAuthReceived,
+                            nick=nick,
+                            offer=json.dumps(ao["offer"]),
+                            commitment=ao["commit"],
+                            revelation=json.dumps(commitment_revelation),
+                            amount=ao["amount"],
+                            kphex=ao["kp"].hex_pk())
+        self.defaultCallbacks(d)
+
+    @JMIOAuth.responder
+    def on_JM_IOAUTH(self, nick, utxolist, pubkey, cjaddr, changeaddr, pubkeysig):
+        nick, utxolist, pubkey, cjaddr, changeaddr, pubkeysig = [_byteify(
+            x) for x in nick, utxolist, pubkey, cjaddr, changeaddr, pubkeysig]
+        if not self.role == "MAKER":
+            return
+        if not nick in self.active_orders:
+            return
+        utxos= json.loads(utxolist)
+        #completed population of order/offer object
+        self.active_orders[nick]["cjaddr"] = cjaddr
+        self.active_orders[nick]["changeaddr"] = changeaddr
+        self.active_orders[nick]["utxos"] = utxos
+        msg = str(",".join(utxos.keys())) + " " + " ".join(
+            [pubkey, cjaddr, changeaddr, pubkeysig])
+        self.mcc.prepare_privmsg(nick, "ioauth", msg)
+        #In case of *blacklisted (ie already used) commitments, we already
+        #broadcasted them on receipt; in case of valid, and now used commitments,
+        #we broadcast them here, and not early - to avoid accidentally
+        #blacklisting commitments that are broadcast between makers in real time
+        #for the same transaction.
+        self.transfer_commitment(self.active_orders[nick]["commit"])
+        #now persist the fact that the commitment is actually used.
+        check_utxo_blacklist(self.active_orders[nick]["commit"], persist=True)
+        return {"accepted": True}
+
+    def transfer_commitment(self, commit):
+        """Send this commitment via privmsg to one (random)
+	other maker.
+	"""
+        crow = self.db.execute(
+                        'SELECT DISTINCT counterparty FROM orderbook ORDER BY ' +
+                        'RANDOM() LIMIT 1;'
+                    ).fetchone()
+        if crow is None:
+            return
+        counterparty = crow['counterparty']
+        #TODO de-hardcode hp2
+        log.msg("Sending commitment to: " + str(counterparty))
+        self.mcc.prepare_privmsg(counterparty, 'hp2', commit)
+
+    def on_commitment_seen(self, nick, commitment):
+        """Triggered when we see a commitment for blacklisting
+	appear in the public pit channel. If the policy is set,
+	we blacklist this commitment.
+	"""
+        if jm_single().config.has_option("POLICY", "accept_commitment_broadcasts"):
+            blacklist_add = jm_single().config.getint("POLICY",
+                                                    "accept_commitment_broadcasts")
+        else:
+            blacklist_add = 0
+        if blacklist_add > 0:
+            #just add if necessary, ignore return value.
+            check_utxo_blacklist(commitment, persist=True)
+            log.msg("Received commitment broadcast by other maker: " + str(
+                commitment) + ", now blacklisted.")
+        else:
+            log.msg("Received commitment broadcast by other maker: " + str(
+                commitment) + ", ignored.")
+
+    def on_commitment_transferred(self, nick, commitment):
+        """Triggered when a privmsg is received from another maker
+	with a commitment to announce in public (obfuscation of source).
+        We simply post it in public (not affected by whether we ourselves
+        are *accepting* commitment broadcasts.
+	"""
+        self.mcc.pubmsg("!hp2 " + commitment)
+
+    def on_push_tx(self, nick, txhex):
+        log.msg('received pushtx message, ignoring, TODO')
+
+    def on_seen_tx(self, nick, txhex):
+        if self.role != "MAKER":
+            return
+        if nick not in self.active_orders:
+            return
+        #we send a copy of the entire "active_orders" entry except the cryptobox,
+        #so make a temporary copy
+        ao = copy.deepcopy(self.active_orders[nick])
+        del ao["crypto_box"]
+        del ao["kp"]
+        d = self.callRemote(JMTXReceived,
+                            nick=nick,
+                            txhex=txhex,
+                            offer=json.dumps(ao))
+        self.defaultCallbacks(d)
+
     def on_pubkey(self, nick, maker_pk):
         """This is handled locally in the daemon; set up e2e
         encrypted messaging with this counterparty
@@ -327,15 +535,20 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch):
         return {'accepted': True}
 
     def get_crypto_box_from_nick(self, nick):
+        """Retrieve the libsodium box object for the counterparty;
+        stored differently for Taker and Maker
+        """
         if nick in self.crypto_boxes and self.crypto_boxes[nick] != None:
-            return self.crypto_boxes[nick][1]  # libsodium encryption object
+            return self.crypto_boxes[nick][1]
+        elif nick in self.active_orders and self.active_orders[nick] != None:
+            return self.active_orders[nick]["crypto_box"]
         else:
             log.msg('something wrong, no crypto object, nick=' + nick +
                       ', message will be dropped')
             return None
 
-    def on_error(self):
-        log.msg("Unimplemented on_error")
+    def on_error(self, msg):
+        log.msg("Received error: " + str(msg))
 
     def mc_shutdown(self):
         log.msg("Message channels being shutdown by daemon")

jmdaemon/jmdaemon/message_channel.py

@@ -213,6 +213,7 @@ class MessageChannelCollection(object):
         else:
             return self.daemon.get_crypto_box_from_nick(nick), True
 
+    @check_privmsg
     def prepare_privmsg(self, nick, cmd, message, mc=None):
         # should we encrypt?
         box, encrypt = self.get_encryption_box(cmd, nick)
@@ -294,24 +295,11 @@ class MessageChannelCollection(object):
             msg = ' '.join(orderlines[0].split(' ')[1:])
             msg += ''.join(orderlines[1:])
             if new_mc:
-                self.privmsg(nick, cmd, msg, new_mc)
+                self.prepare_privmsg(nick, cmd, msg, mc=new_mc)
             else:
                 for mc in self.available_channels():
                     if nick in self.nicks_seen[mc]:
-                        self.privmsg(nick, cmd, msg, mc)
-
-    @check_privmsg
-    def send_pubkey(self, nick, pubkey):
-        self.active_channels[nick].privmsg(nick, 'pubkey', pubkey)
-
-    @check_privmsg
-    def send_ioauth(self, nick, utxo_list, auth_pub, cj_addr, change_addr, sig):
-        self.active_channels[nick].send_ioauth(nick, utxo_list, auth_pub,
-                                               cj_addr, change_addr, sig)
-
-    @check_privmsg
-    def send_sigs(self, nick, sig_list):
-        self.active_channels[nick].send_sigs(nick, sig_list)
+                        self.prepare_privmsg(nick, cmd, msg, mc=mc)
 
     # Taker callbacks
     def fill_orders(self, nick_order_dict, cj_amount, taker_pubkey, commitment):
@@ -337,7 +325,8 @@ class MessageChannelCollection(object):
     @check_privmsg
     def send_error(self, nick, errormsg):
         #TODO this might need to support non-active nicks TODO
-        self.active_channels[nick].send_error(nick, errormsg)
+        if nick in self.active_channels:
+            self.active_channels[nick].send_error(nick, errormsg)
 
     @check_privmsg
     def push_tx(self, nick, txhex):
@@ -792,19 +781,6 @@ class MessageChannel(object):
         clines = [COMMAND_PREFIX + 'cancel ' + str(oid) for oid in oid_list]
         self.pubmsg(''.join(clines))
 
-    def send_pubkey(self, nick, pubkey):
-        self.privmsg(nick, 'pubkey', pubkey)
-
-    def send_ioauth(self, nick, utxo_list, auth_pub, cj_addr, change_addr, sig):
-        authmsg = str(','.join(utxo_list)) + ' ' + ' '.join([auth_pub, cj_addr,
-                                                             change_addr, sig])
-        self.privmsg(nick, 'ioauth', authmsg)
-
-    def send_sigs(self, nick, sig_list):
-        # TODO make it send the sigs on one line if there's space
-        for s in sig_list:
-            self.privmsg(nick, 'sig', s)
-
     # OrderbookWatch callback
     def request_orderbook(self):
         self.pubmsg(COMMAND_PREFIX + 'orderbook')
@@ -937,6 +913,10 @@ class MessageChannel(object):
                 if self.check_for_orders(nick, _chunks):
                     pass
                 # taker commands
+                elif _chunks[0] == 'error':
+                    error = " ".join(_chunks[1:])
+                    if self.on_error:
+                        self.on_error(error)
                 elif _chunks[0] == 'pubkey':
                     maker_pk = _chunks[1]
                     if self.on_pubkey:

jmdaemon/jmdaemon/protocol.py

@@ -1,5 +1,5 @@
 #Protocol version
-JM_VERSION = 5
+JM_VERSION = 6
 
 #Username on all messagechannels; will be set in MessageChannelCollection
 nickname = None
@@ -24,6 +24,8 @@ JOINMARKET_NICK_HEADER = 'J'
 NICK_HASH_LENGTH = 10
 NICK_MAX_ENCODED = 14  #comes from base58 expansion; recalculate if above changes
 
+#commitments; note multiple options may be used in future
+COMMITMENT_PREFIXES = ["P"]
 #Lists of valid commands
 encrypted_commands = ["auth", "ioauth", "tx", "sig"]
 plaintext_commands = ["fill", "error", "pubkey", "orderbook", "push"]

scripts/sendpayment.py

@@ -18,7 +18,7 @@ import os
 import pprint
 
 from jmclient import (Taker, load_program_config, get_schedule,
-                              JMTakerClientProtocolFactory, start_reactor,
+                              JMClientProtocolFactory, start_reactor,
                               validate_address, jm_single, WalletError,
                               choose_orders, choose_sweep_orders,
                               cheapest_order_choose, weighted_order_choose,
@@ -197,7 +197,7 @@ def main():
                   schedule,
                   order_chooser=chooseOrdersFunc,
                   callbacks=(filter_orders_callback, None, taker_finished))
-    clientfactory = JMTakerClientProtocolFactory(taker)
+    clientfactory = JMClientProtocolFactory(taker)
     nodaemon = jm_single().config.getint("DAEMON", "no_daemon")
     daemon = True if nodaemon == 1 else False
     start_reactor(jm_single().config.get("DAEMON", "daemon_host"),