Include ygrunner for multiple yieldgens in test

Update blockchain interface notifier thread to account
for multiple concurrent notification functions from
multiple yieldgens in one process instance.

jmclient/jmclient/blockchaininterface.py

@@ -859,12 +859,13 @@ class NotifyRequestHeader(BaseHTTPServer.BaseHTTPRequestHandler):
             txnotify_tuple = None
             unconfirmfun, confirmfun, timeoutfun, uc_called = (None, None, None,
                                                                None)
+            to_be_updated = []
+            to_be_removed = []
             for tnf in self.btcinterface.txnotify_fun:
                 tx_out = tnf[0]
                 if tx_out == tx_output_set:
                     txnotify_tuple = tnf
                     tx_out, unconfirmfun, confirmfun, timeoutfun, uc_called = tnf
-                    break
                     if unconfirmfun is None:
                         log.debug('txid=' + txid + ' not being listened for')
                     else:
@@ -878,13 +879,7 @@ class NotifyRequestHeader(BaseHTTPServer.BaseHTTPRequestHandler):
                         assert txdata is not None
                         if txdata['confirmations'] == 0:
                             reactor.callFromThread(unconfirmfun, txd, txid)
-                    # TODO pass the total transfered amount value here somehow
+                            to_be_updated.append(txnotify_tuple)
-                    # wallet_name = self.get_wallet_name()
-                    # amount =
-                    # bitcoin-cli move wallet_name "" amount
-                    self.btcinterface.txnotify_fun.remove(txnotify_tuple)
-                    self.btcinterface.txnotify_fun.append(txnotify_tuple[:-1] +
-                                                          (True,))
                             log.debug('ran unconfirmfun')
                             if timeoutfun:
                                 threading.Timer(jm_single().config.getfloat(
@@ -899,8 +894,15 @@ class NotifyRequestHeader(BaseHTTPServer.BaseHTTPRequestHandler):
                                 log.debug('saw confirmed tx before unconfirmed, ' +
                                           'running unconfirmfun first')
                             reactor.callFromThread(confirmfun, txd, txid, txdata['confirmations'])
-                    self.btcinterface.txnotify_fun.remove(txnotify_tuple)
+                            to_be_removed.append(txnotify_tuple)
                             log.debug('ran confirmfun')
+            #If any notifyfun tuples need to update from unconfirm to confirm state:
+            for tbu in to_be_updated:
+                self.btcinterface.txnotify_fun.remove(tbu)
+                self.btcinterface.txnotify_fun.append(tbu[:-1] + (True,))
+            #If any notifyfun tuples need to be removed as confirmed
+            for tbr in to_be_removed:
+                self.btcinterface.txnotify_fun.remove(tbr)
 
         elif self.path.startswith('/alertnotify?'):
             jm_single().core_alert[0] = urllib.unquote(self.path[len(pages[
@@ -916,7 +918,7 @@ class NotifyRequestHeader(BaseHTTPServer.BaseHTTPRequestHandler):
         request.get_method = lambda: 'HEAD'
         try:
             urllib2.urlopen(request)
-        except urllib2.URLError:
+        except:
             pass
         self.send_response(200)
         # self.send_header('Connection', 'close')

jmclient/jmclient/client_protocol.py

@@ -435,7 +435,7 @@ class JMClientProtocolFactory(protocol.ClientFactory):
     def buildProtocol(self, addr):
         return self.protocol(self, self.client)
 
-def start_reactor(host, port, factory, ish=True, daemon=False): #pragma: no cover
+def start_reactor(host, port, factory, ish=True, daemon=False, rs=True): #pragma: no cover
     #(Cannot start the reactor in tests)
     #Not used in prod (twisted logging):
     #startLogging(stdout)
@@ -473,4 +473,5 @@ def start_reactor(host, port, factory, ish=True, daemon=False): #pragma: no cove
         reactor.connectSSL(host, port, factory, ctx)
     else:
         reactor.connectTCP(host, port, factory)
+    if rs:
         reactor.run(installSignalHandlers=ish)

jmclient/jmclient/wallet.py

@@ -345,7 +345,7 @@ class Wallet(AbstractWallet):
             removed_utxos[utxo] = self.unspent[utxo]
             del self.unspent[utxo]
         log.debug('removed utxos, wallet now is \n' + pprint.pformat(
-                self.get_utxos_by_mixdepth()))
+                self.get_utxos_by_mixdepth(verbose=False)))
         self.spent_utxos += removed_utxos.keys()
         return removed_utxos
 

jmclient/jmclient/yieldgenerator.py

@@ -78,7 +78,7 @@ class YieldGeneratorBasic(YieldGenerator):
         super(YieldGeneratorBasic,self).__init__(wallet)
 
     def create_my_orders(self):
-        mix_balance = self.wallet.get_balance_by_mixdepth()
+        mix_balance = self.wallet.get_balance_by_mixdepth(verbose=False)
         if len([b for m, b in mix_balance.iteritems() if b > 0]) == 0:
             jlog.error('do not have any coins left')
             return []
@@ -254,6 +254,7 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffe
                              options.ordertype, options.minsize])
     jlog.info('starting yield generator')
     clientfactory = JMClientProtocolFactory(maker, proto_type="MAKER")
+
     nodaemon = jm_single().config.getint("DAEMON", "no_daemon")
     daemon = True if nodaemon == 1 else False
     start_reactor(jm_single().config.get("DAEMON", "daemon_host"),

test/commontest.py

@@ -0,0 +1,109 @@
+#! /usr/bin/env python
+from __future__ import absolute_import
+'''Some helper functions for testing'''
+
+import sys
+import os
+import time
+import binascii
+import random
+from decimal import Decimal
+
+data_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+sys.path.insert(0, os.path.join(data_dir))
+
+from jmclient import SegwitWallet, Wallet, get_log, estimate_tx_fee, jm_single
+import jmbitcoin as btc
+from jmbase import chunks
+
+log = get_log()
+
+def make_sign_and_push(ins_full,
+                       wallet,
+                       amount,
+                       output_addr=None,
+                       change_addr=None,
+                       hashcode=btc.SIGHASH_ALL,
+                       estimate_fee = False):
+    """Utility function for easily building transactions
+    from wallets
+    """
+    total = sum(x['value'] for x in ins_full.values())
+    ins = ins_full.keys()
+    #random output address and change addr
+    output_addr = wallet.get_new_addr(1, 1) if not output_addr else output_addr
+    change_addr = wallet.get_new_addr(1, 0) if not change_addr else change_addr
+    fee_est = estimate_tx_fee(len(ins), 2) if estimate_fee else 10000
+    outs = [{'value': amount,
+             'address': output_addr}, {'value': total - amount - fee_est,
+                                       'address': change_addr}]
+
+    tx = btc.mktx(ins, outs)
+    de_tx = btc.deserialize(tx)
+    for index, ins in enumerate(de_tx['ins']):
+        utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
+        addr = ins_full[utxo]['address']
+        priv = wallet.get_key_from_addr(addr)
+        if index % 2:
+            priv = binascii.unhexlify(priv)
+        tx = btc.sign(tx, index, priv, hashcode=hashcode)
+    #pushtx returns False on any error
+    print btc.deserialize(tx)
+    push_succeed = jm_single().bc_interface.pushtx(tx)
+    if push_succeed:
+        return btc.txhash(tx)
+    else:
+        return False
+
+def make_wallets(n,
+                 wallet_structures=None,
+                 mean_amt=1,
+                 sdev_amt=0,
+                 start_index=0,
+                 fixed_seeds=None,
+                 test_wallet=False,
+                 passwords=None):
+    '''n: number of wallets to be created
+       wallet_structure: array of n arrays , each subarray
+       specifying the number of addresses to be populated with coins
+       at each depth (for now, this will only populate coins into 'receive' addresses)
+       mean_amt: the number of coins (in btc units) in each address as above
+       sdev_amt: if randomness in amouts is desired, specify here.
+       Returns: a dict of dicts of form {0:{'seed':seed,'wallet':Wallet object},1:..,}
+       Default Wallet constructor is joinmarket.Wallet, else use TestWallet,
+       which takes a password parameter as in the list passwords.
+       '''
+    if len(wallet_structures) != n:
+        raise Exception("Number of wallets doesn't match wallet structures")
+    if not fixed_seeds:
+        seeds = chunks(binascii.hexlify(os.urandom(15 * n)), 15 * 2)
+    else:
+        seeds = fixed_seeds
+    wallets = {}
+    for i in range(n):
+        if test_wallet:
+            w = TestWallet(seeds[i], max_mix_depth=5, pwd=passwords[i])
+        else:
+            w = SegwitWallet(seeds[i], pwd=None, max_mix_depth=5)
+        wallets[i + start_index] = {'seed': seeds[i],
+                                    'wallet': w}
+        for j in range(5):
+            for k in range(wallet_structures[i][j]):
+                deviation = sdev_amt * random.random()
+                amt = mean_amt - sdev_amt / 2.0 + deviation
+                if amt < 0: amt = 0.001
+                amt = float(Decimal(amt).quantize(Decimal(10)**-8))
+                jm_single().bc_interface.grab_coins(
+                    wallets[i + start_index]['wallet'].get_external_addr(j),
+                    amt)
+            #reset the index so the coins can be seen if running in same script
+            wallets[i + start_index]['wallet'].index[j][0] -= wallet_structures[i][j]
+    return wallets
+
+
+def interact(process, inputs, expected):
+    if len(inputs) != len(expected):
+        raise Exception("Invalid inputs to interact()")
+    for i, inp in enumerate(inputs):
+        process.expect(expected[i])
+        process.sendline(inp)

test/ygrunner.py

@@ -0,0 +1,66 @@
+#! /usr/bin/env python
+from __future__ import absolute_import
+'''Creates wallets and yield generators in regtest. 
+   Provides seed for joinmarket-qt test.
+   This should be run via pytest, even though
+   it's NOT part of the test-suite, because that
+   makes it much easier to handle start up and
+   shut down of the environment.
+   Run it like:
+   PYTHONPATH=.:$PYTHONPATH py.test \
+   --btcroot=/path/to/bitcoin/bin/ \
+   --btcpwd=123456abcdef --btcconf=/blah/bitcoin.conf \
+   --nirc=2 -s test/ygrunner.py
+   '''
+from commontest import make_wallets
+import os
+import pytest
+import sys
+import time
+from jmclient import (YieldGeneratorBasic, ygmain, load_program_config,
+                      jm_single, sync_wallet, JMClientProtocolFactory,
+                      start_reactor)
+
+@pytest.mark.parametrize(
+    "num_ygs, wallet_structures, mean_amt",
+    [
+        # 1sp 3yg, 2 mixdepths, sweep from depth1
+        (2, [[1, 3, 0, 0, 0]] * 3, 2),
+    ])
+def test_start_ygs(setup_ygrunner, num_ygs, wallet_structures, mean_amt):
+    """Set up some wallets, for the ygs and 1 sp.
+    Then start the ygs in background and publish
+    the seed of the sp wallet for easy import into -qt
+    """
+    wallets = make_wallets(num_ygs + 1,
+                           wallet_structures=wallet_structures,
+                           mean_amt=mean_amt)
+    #the sendpayment bot uses the last wallet in the list
+    wallet = wallets[num_ygs]['wallet']
+    print "Seed : " + wallets[num_ygs]['seed']
+    #useful to see the utxos on screen sometimes
+    sync_wallet(wallet)
+    print wallet.unspent
+    txfee = 1000
+    cjfee_a = 4200
+    cjfee_r = '0.001'
+    ordertype = 'swreloffer'
+    minsize = 100000
+    for i in range(num_ygs):
+        
+        cfg = [txfee, cjfee_a, cjfee_r, ordertype, minsize]
+        sync_wallet(wallets[i]["wallet"])
+        yg = YieldGeneratorBasic(wallets[i]["wallet"], cfg)
+        clientfactory = JMClientProtocolFactory(yg, proto_type="MAKER")
+        nodaemon = jm_single().config.getint("DAEMON", "no_daemon")
+        daemon = True if nodaemon == 1 else False
+        rs = True if i == num_ygs - 1 else False
+        start_reactor(jm_single().config.get("DAEMON", "daemon_host"),
+                      jm_single().config.getint("DAEMON", "daemon_port"),
+                      clientfactory, daemon=daemon, rs=rs)
+        time.sleep(2)  #give it a chance
+
+@pytest.fixture(scope="module")
+def setup_ygrunner():
+    load_program_config()
+