diff --git a/jmclient/jmclient/blockchaininterface.py b/jmclient/jmclient/blockchaininterface.py index f6faed1..8b246ce 100644 --- a/jmclient/jmclient/blockchaininterface.py +++ b/jmclient/jmclient/blockchaininterface.py @@ -859,48 +859,50 @@ class NotifyRequestHeader(BaseHTTPServer.BaseHTTPRequestHandler): txnotify_tuple = None unconfirmfun, confirmfun, timeoutfun, uc_called = (None, None, None, None) + to_be_updated = [] + to_be_removed = [] for tnf in self.btcinterface.txnotify_fun: tx_out = tnf[0] if tx_out == tx_output_set: txnotify_tuple = tnf tx_out, unconfirmfun, confirmfun, timeoutfun, uc_called = tnf - break - if unconfirmfun is None: - log.debug('txid=' + txid + ' not being listened for') - else: - # on rare occasions people spend their output without waiting - # for a confirm - txdata = None - for n in range(len(txd['outs'])): - txdata = self.btcinterface.rpc('gettxout', [txid, n, True]) - if txdata is not None: - break - assert txdata is not None - if txdata['confirmations'] == 0: - reactor.callFromThread(unconfirmfun, txd, txid) - # TODO pass the total transfered amount value here somehow - # wallet_name = self.get_wallet_name() - # amount = - # bitcoin-cli move wallet_name "" amount - self.btcinterface.txnotify_fun.remove(txnotify_tuple) - self.btcinterface.txnotify_fun.append(txnotify_tuple[:-1] + - (True,)) - log.debug('ran unconfirmfun') - if timeoutfun: - threading.Timer(jm_single().config.getfloat( - 'TIMEOUT', 'confirm_timeout_hours') * 60 * 60, - bitcoincore_timeout_callback, - args=(True, tx_output_set, - self.btcinterface.txnotify_fun, - timeoutfun)).start() - else: - if not uc_called: - reactor.callFromThread(unconfirmfun, txd, txid) - log.debug('saw confirmed tx before unconfirmed, ' + - 'running unconfirmfun first') - reactor.callFromThread(confirmfun, txd, txid, txdata['confirmations']) - self.btcinterface.txnotify_fun.remove(txnotify_tuple) - log.debug('ran confirmfun') + if unconfirmfun is None: + log.debug('txid=' + txid + ' not being listened for') + else: + # on rare occasions people spend their output without waiting + # for a confirm + txdata = None + for n in range(len(txd['outs'])): + txdata = self.btcinterface.rpc('gettxout', [txid, n, True]) + if txdata is not None: + break + assert txdata is not None + if txdata['confirmations'] == 0: + reactor.callFromThread(unconfirmfun, txd, txid) + to_be_updated.append(txnotify_tuple) + log.debug('ran unconfirmfun') + if timeoutfun: + threading.Timer(jm_single().config.getfloat( + 'TIMEOUT', 'confirm_timeout_hours') * 60 * 60, + bitcoincore_timeout_callback, + args=(True, tx_output_set, + self.btcinterface.txnotify_fun, + timeoutfun)).start() + else: + if not uc_called: + reactor.callFromThread(unconfirmfun, txd, txid) + log.debug('saw confirmed tx before unconfirmed, ' + + 'running unconfirmfun first') + reactor.callFromThread(confirmfun, txd, txid, txdata['confirmations']) + to_be_removed.append(txnotify_tuple) + log.debug('ran confirmfun') + #If any notifyfun tuples need to update from unconfirm to confirm state: + for tbu in to_be_updated: + self.btcinterface.txnotify_fun.remove(tbu) + self.btcinterface.txnotify_fun.append(tbu[:-1] + (True,)) + #If any notifyfun tuples need to be removed as confirmed + for tbr in to_be_removed: + self.btcinterface.txnotify_fun.remove(tbr) elif self.path.startswith('/alertnotify?'): jm_single().core_alert[0] = urllib.unquote(self.path[len(pages[ @@ -916,7 +918,7 @@ class NotifyRequestHeader(BaseHTTPServer.BaseHTTPRequestHandler): request.get_method = lambda: 'HEAD' try: urllib2.urlopen(request) - except urllib2.URLError: + except: pass self.send_response(200) # self.send_header('Connection', 'close') diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index d813e84..9000688 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -435,7 +435,7 @@ class JMClientProtocolFactory(protocol.ClientFactory): def buildProtocol(self, addr): return self.protocol(self, self.client) -def start_reactor(host, port, factory, ish=True, daemon=False): #pragma: no cover +def start_reactor(host, port, factory, ish=True, daemon=False, rs=True): #pragma: no cover #(Cannot start the reactor in tests) #Not used in prod (twisted logging): #startLogging(stdout) @@ -473,4 +473,5 @@ def start_reactor(host, port, factory, ish=True, daemon=False): #pragma: no cove reactor.connectSSL(host, port, factory, ctx) else: reactor.connectTCP(host, port, factory) - reactor.run(installSignalHandlers=ish) + if rs: + reactor.run(installSignalHandlers=ish) diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index aedba99..b64bf47 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -345,7 +345,7 @@ class Wallet(AbstractWallet): removed_utxos[utxo] = self.unspent[utxo] del self.unspent[utxo] log.debug('removed utxos, wallet now is \n' + pprint.pformat( - self.get_utxos_by_mixdepth())) + self.get_utxos_by_mixdepth(verbose=False))) self.spent_utxos += removed_utxos.keys() return removed_utxos diff --git a/jmclient/jmclient/yieldgenerator.py b/jmclient/jmclient/yieldgenerator.py index cbefc33..43554ea 100644 --- a/jmclient/jmclient/yieldgenerator.py +++ b/jmclient/jmclient/yieldgenerator.py @@ -78,7 +78,7 @@ class YieldGeneratorBasic(YieldGenerator): super(YieldGeneratorBasic,self).__init__(wallet) def create_my_orders(self): - mix_balance = self.wallet.get_balance_by_mixdepth() + mix_balance = self.wallet.get_balance_by_mixdepth(verbose=False) if len([b for m, b in mix_balance.iteritems() if b > 0]) == 0: jlog.error('do not have any coins left') return [] @@ -254,6 +254,7 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffe options.ordertype, options.minsize]) jlog.info('starting yield generator') clientfactory = JMClientProtocolFactory(maker, proto_type="MAKER") + nodaemon = jm_single().config.getint("DAEMON", "no_daemon") daemon = True if nodaemon == 1 else False start_reactor(jm_single().config.get("DAEMON", "daemon_host"), diff --git a/test/commontest.py b/test/commontest.py new file mode 100644 index 0000000..8ff993f --- /dev/null +++ b/test/commontest.py @@ -0,0 +1,109 @@ +#! /usr/bin/env python +from __future__ import absolute_import +'''Some helper functions for testing''' + +import sys +import os +import time +import binascii +import random +from decimal import Decimal + +data_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +sys.path.insert(0, os.path.join(data_dir)) + +from jmclient import SegwitWallet, Wallet, get_log, estimate_tx_fee, jm_single +import jmbitcoin as btc +from jmbase import chunks + +log = get_log() + +def make_sign_and_push(ins_full, + wallet, + amount, + output_addr=None, + change_addr=None, + hashcode=btc.SIGHASH_ALL, + estimate_fee = False): + """Utility function for easily building transactions + from wallets + """ + total = sum(x['value'] for x in ins_full.values()) + ins = ins_full.keys() + #random output address and change addr + output_addr = wallet.get_new_addr(1, 1) if not output_addr else output_addr + change_addr = wallet.get_new_addr(1, 0) if not change_addr else change_addr + fee_est = estimate_tx_fee(len(ins), 2) if estimate_fee else 10000 + outs = [{'value': amount, + 'address': output_addr}, {'value': total - amount - fee_est, + 'address': change_addr}] + + tx = btc.mktx(ins, outs) + de_tx = btc.deserialize(tx) + for index, ins in enumerate(de_tx['ins']): + utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + addr = ins_full[utxo]['address'] + priv = wallet.get_key_from_addr(addr) + if index % 2: + priv = binascii.unhexlify(priv) + tx = btc.sign(tx, index, priv, hashcode=hashcode) + #pushtx returns False on any error + print btc.deserialize(tx) + push_succeed = jm_single().bc_interface.pushtx(tx) + if push_succeed: + return btc.txhash(tx) + else: + return False + +def make_wallets(n, + wallet_structures=None, + mean_amt=1, + sdev_amt=0, + start_index=0, + fixed_seeds=None, + test_wallet=False, + passwords=None): + '''n: number of wallets to be created + wallet_structure: array of n arrays , each subarray + specifying the number of addresses to be populated with coins + at each depth (for now, this will only populate coins into 'receive' addresses) + mean_amt: the number of coins (in btc units) in each address as above + sdev_amt: if randomness in amouts is desired, specify here. + Returns: a dict of dicts of form {0:{'seed':seed,'wallet':Wallet object},1:..,} + Default Wallet constructor is joinmarket.Wallet, else use TestWallet, + which takes a password parameter as in the list passwords. + ''' + if len(wallet_structures) != n: + raise Exception("Number of wallets doesn't match wallet structures") + if not fixed_seeds: + seeds = chunks(binascii.hexlify(os.urandom(15 * n)), 15 * 2) + else: + seeds = fixed_seeds + wallets = {} + for i in range(n): + if test_wallet: + w = TestWallet(seeds[i], max_mix_depth=5, pwd=passwords[i]) + else: + w = SegwitWallet(seeds[i], pwd=None, max_mix_depth=5) + wallets[i + start_index] = {'seed': seeds[i], + 'wallet': w} + for j in range(5): + for k in range(wallet_structures[i][j]): + deviation = sdev_amt * random.random() + amt = mean_amt - sdev_amt / 2.0 + deviation + if amt < 0: amt = 0.001 + amt = float(Decimal(amt).quantize(Decimal(10)**-8)) + jm_single().bc_interface.grab_coins( + wallets[i + start_index]['wallet'].get_external_addr(j), + amt) + #reset the index so the coins can be seen if running in same script + wallets[i + start_index]['wallet'].index[j][0] -= wallet_structures[i][j] + return wallets + + +def interact(process, inputs, expected): + if len(inputs) != len(expected): + raise Exception("Invalid inputs to interact()") + for i, inp in enumerate(inputs): + process.expect(expected[i]) + process.sendline(inp) diff --git a/test/ygrunner.py b/test/ygrunner.py new file mode 100644 index 0000000..aead85c --- /dev/null +++ b/test/ygrunner.py @@ -0,0 +1,66 @@ +#! /usr/bin/env python +from __future__ import absolute_import +'''Creates wallets and yield generators in regtest. + Provides seed for joinmarket-qt test. + This should be run via pytest, even though + it's NOT part of the test-suite, because that + makes it much easier to handle start up and + shut down of the environment. + Run it like: + PYTHONPATH=.:$PYTHONPATH py.test \ + --btcroot=/path/to/bitcoin/bin/ \ + --btcpwd=123456abcdef --btcconf=/blah/bitcoin.conf \ + --nirc=2 -s test/ygrunner.py + ''' +from commontest import make_wallets +import os +import pytest +import sys +import time +from jmclient import (YieldGeneratorBasic, ygmain, load_program_config, + jm_single, sync_wallet, JMClientProtocolFactory, + start_reactor) + +@pytest.mark.parametrize( + "num_ygs, wallet_structures, mean_amt", + [ + # 1sp 3yg, 2 mixdepths, sweep from depth1 + (2, [[1, 3, 0, 0, 0]] * 3, 2), + ]) +def test_start_ygs(setup_ygrunner, num_ygs, wallet_structures, mean_amt): + """Set up some wallets, for the ygs and 1 sp. + Then start the ygs in background and publish + the seed of the sp wallet for easy import into -qt + """ + wallets = make_wallets(num_ygs + 1, + wallet_structures=wallet_structures, + mean_amt=mean_amt) + #the sendpayment bot uses the last wallet in the list + wallet = wallets[num_ygs]['wallet'] + print "Seed : " + wallets[num_ygs]['seed'] + #useful to see the utxos on screen sometimes + sync_wallet(wallet) + print wallet.unspent + txfee = 1000 + cjfee_a = 4200 + cjfee_r = '0.001' + ordertype = 'swreloffer' + minsize = 100000 + for i in range(num_ygs): + + cfg = [txfee, cjfee_a, cjfee_r, ordertype, minsize] + sync_wallet(wallets[i]["wallet"]) + yg = YieldGeneratorBasic(wallets[i]["wallet"], cfg) + clientfactory = JMClientProtocolFactory(yg, proto_type="MAKER") + nodaemon = jm_single().config.getint("DAEMON", "no_daemon") + daemon = True if nodaemon == 1 else False + rs = True if i == num_ygs - 1 else False + start_reactor(jm_single().config.get("DAEMON", "daemon_host"), + jm_single().config.getint("DAEMON", "daemon_port"), + clientfactory, daemon=daemon, rs=rs) + time.sleep(2) #give it a chance + +@pytest.fixture(scope="module") +def setup_ygrunner(): + load_program_config() + \ No newline at end of file