diff --git a/.travis.yml b/.travis.yml index 65cb168..57985a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,7 +40,7 @@ script: - chmod 600 /home/travis/.bitcoin/bitcoin.conf - mkdir logs - mkdir wallets - - python -m py.test --cov=jmclient --cov=jmbitcoin --cov=jmbase --cov=jmdaemon --cov-report html --btcpwd=123456abcdef --btcconf=/home/travis/.bitcoin/bitcoin.conf --btcuser=bitcoinrpc --nirc=2 + - python -m py.test --cov=jmclient --cov=jmbitcoin --cov=jmbase --cov=jmdaemon --cov-report html --btcpwd=123456abcdef --btcconf=/home/travis/.bitcoin/bitcoin.conf --btcuser=bitcoinrpc --nirc=2 --ignore jmclient/test/test_wallets.py --ignore test/test_segwit.py after_success: - coveralls branches: diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index 1a65217..5f8c5a6 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -25,7 +25,8 @@ from .configure import (load_program_config, jm_single, get_p2pk_vbyte, from .blockchaininterface import (BlockchainInterface, sync_wallet, RegtestBitcoinCoreInterface, BitcoinCoreInterface) from .electruminterface import ElectrumInterface -from .client_protocol import (JMClientProtocolFactory, start_reactor) +from .client_protocol import (JMTakerClientProtocol, JMClientProtocolFactory, + start_reactor) from .podle import (set_commitment_file, get_commitment_file, generate_podle_error_string, add_external_commitments, PoDLE, generate_podle, get_podle_commitments, diff --git a/jmclient/jmclient/blockchaininterface.py b/jmclient/jmclient/blockchaininterface.py index b0f678a..c358a90 100644 --- a/jmclient/jmclient/blockchaininterface.py +++ b/jmclient/jmclient/blockchaininterface.py @@ -9,7 +9,6 @@ import pprint import random import re import sys -import threading import time import traceback from decimal import Decimal @@ -200,7 +199,6 @@ class BitcoinCoreInterface(BlockchainInterface): if netmap[actualNet] != network: raise Exception('wrong network configured') - self.notifythread = None self.txnotify_fun = [] self.wallet_synced = False #task.LoopingCall objects that track transactions, keyed by txids. @@ -522,7 +520,8 @@ class BitcoinCoreInterface(BlockchainInterface): log.debug('bitcoind sync_unspent took ' + str((et - st)) + 'sec') def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr, - timeoutfun=None, spentfun=None, txid_flag=True, n=0, c=1, vb=None): + wallet_name=None, timeoutfun=None, spentfun=None, txid_flag=True, + n=0, c=1, vb=None): """Given a deserialized transaction txd, callback functions for broadcast and confirmation of the transaction, an address to import, and a callback function for timeout, set up @@ -551,8 +550,9 @@ class BitcoinCoreInterface(BlockchainInterface): txid = btc.txhash(btc.serialize(txd)) if not txid_flag: tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) - loop = task.LoopingCall(self.outputs_watcher, notifyaddr, tx_output_set, - unconfirmfun, confirmfun, timeoutfun) + loop = task.LoopingCall(self.outputs_watcher, wallet_name, notifyaddr, + tx_output_set, unconfirmfun, confirmfun, + timeoutfun) log.debug("Created watcher loop for address: " + notifyaddr) loopkey = notifyaddr else: @@ -562,7 +562,7 @@ class BitcoinCoreInterface(BlockchainInterface): loopkey = txid self.tx_watcher_loops[loopkey] = [loop, False, False, False] #Hardcoded polling interval, but in any case it can be very short. - loop.start(2.0) + loop.start(5.0) #TODO Hardcoded very long timeout interval reactor.callLater(7200, self.tx_timeout, txd, loopkey, timeoutfun) @@ -589,16 +589,17 @@ class BitcoinCoreInterface(BlockchainInterface): hexval = str(rpcretval["hex"]) return btc.deserialize(hexval) - def outputs_watcher(self, notifyaddr, tx_output_set, unconfirmfun, confirmfun, - timeoutfun): - """Given a key for the watcher loop (txid), a set of outputs, and - unconfirm, confirm and timeout callbacks, check to see if a transaction - matching that output set has appeared in the wallet. Call the callbacks - and update the watcher loop state. End the loop when the confirmation - has been seen (no spent monitoring here). + def outputs_watcher(self, wallet_name, notifyaddr, tx_output_set, + unconfirmfun, confirmfun, timeoutfun): + """Given a key for the watcher loop (notifyaddr), a wallet name (account), + a set of outputs, and unconfirm, confirm and timeout callbacks, + check to see if a transaction matching that output set has appeared in + the wallet. Call the callbacks and update the watcher loop state. + End the loop when the confirmation has been seen (no spent monitoring here). """ wl = self.tx_watcher_loops[notifyaddr] - txlist = self.rpc("listtransactions", ["*", 1000, 0, True]) + account_name = wallet_name if wallet_name else "*" + txlist = self.rpc("listtransactions", [wallet_name, 100, 0, True]) for tx in txlist[::-1]: #changed syntax in 0.14.0; allow both syntaxes try: @@ -606,10 +607,12 @@ class BitcoinCoreInterface(BlockchainInterface): except: try: res = self.rpc("gettransaction", [tx["txid"], 1]) - except: + except JsonRpcError as e: #This should never happen (gettransaction is a wallet rpc). log.info("Failed any gettransaction call") res = None + except Exception as e: + log.info(str(e)) if not res: continue if "confirmations" not in res: @@ -774,26 +777,6 @@ class BitcoinCoreInterface(BlockchainInterface): else: return estimate - -class TickChainThread(threading.Thread): - - def __init__(self, bcinterface, forever=False): - threading.Thread.__init__(self, name='TickChainThread') - self.bcinterface = bcinterface - self.forever = forever - def run(self): - if self.bcinterface.tick_forward_chain_interval < 0: - log.debug('not ticking forward chain') - return - if self.forever: - while True: - if self.bcinterface.shutdown_signal: - return - time.sleep(self.bcinterface.tick_forward_chain_interval) - self.bcinterface.tick_forward_chain(1) - time.sleep(self.bcinterface.tick_forward_chain_interval) - self.bcinterface.tick_forward_chain(1) - # class for regtest chain access # running on local daemon. Only # to be instantiated after network is up @@ -816,8 +799,19 @@ class RegtestBitcoinCoreInterface(BitcoinCoreInterface): #pragma: no cover return jm_single().config.getint("POLICY", "absurd_fee_per_kb") + 100 + def tickchain(self): + if self.tick_forward_chain_interval < 0: + log.debug('not ticking forward chain') + self.tickchainloop.stop() + return + if self.shutdown_signal: + self.tickchainloop.stop() + return + self.tick_forward_chain(1) + def simulate_blocks(self): - TickChainThread(self, forever=True).start() + self.tickchainloop = task.LoopingCall(self.tickchain) + self.tickchainloop.start(self.tick_forward_chain_interval) self.simulating = True def pushtx(self, txhex): @@ -828,8 +822,10 @@ class RegtestBitcoinCoreInterface(BitcoinCoreInterface): #pragma: no cover return True ret = super(RegtestBitcoinCoreInterface, self).pushtx(txhex) - if not self.simulating: - TickChainThread(self).start() + if not self.simulating and self.tick_forward_chain_interval > 0: + print('will call tfc after ' + str(self.tick_forward_chain_interval) + ' seconds.') + reactor.callLater(self.tick_forward_chain_interval, + self.tick_forward_chain, 1) return ret def tick_forward_chain(self, n): diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index fd1d262..0266476 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -212,10 +212,11 @@ class JMMakerClientProtocol(JMClientProtocol): tx = btc.deserialize(txhex) self.finalized_offers[nick]["txd"] = tx jm_single().bc_interface.add_tx_notify(tx, self.unconfirm_callback, - self.confirm_callback, - offer["cjaddr"], - txid_flag=False, - vb=get_p2sh_vbyte()) + self.confirm_callback, offer["cjaddr"], + wallet_name=jm_single().bc_interface.get_wallet_name( + self.client.wallet), + txid_flag=False, + vb=get_p2sh_vbyte()) d = self.callRemote(commands.JMTXSigs, nick=nick, sigs=json.dumps(sigs)) diff --git a/jmclient/jmclient/jsonrpc.py b/jmclient/jmclient/jsonrpc.py index d12243f..39024d0 100644 --- a/jmclient/jmclient/jsonrpc.py +++ b/jmclient/jmclient/jsonrpc.py @@ -54,6 +54,7 @@ class JsonRpc(object): def __init__(self, host, port, user, password): self.host = host self.port = port + self.conn = httplib.HTTPConnection(self.host, self.port) self.authstr = "%s:%s" % (user, password) self.queryId = 1 @@ -74,27 +75,27 @@ class JsonRpc(object): body = json.dumps(obj) try: - conn = httplib.HTTPConnection(self.host, self.port) - conn.request("POST", "", body, headers) - response = conn.getresponse() + self.conn.request("POST", "", body, headers) + response = self.conn.getresponse() if response.status == 401: - conn.close() + self.conn.close() raise JsonRpcConnectionError( "authentication for JSON-RPC failed") # All of the codes below are 'fine' from a JSON-RPC point of view. if response.status not in [200, 404, 500]: - conn.close() + self.conn.close() raise JsonRpcConnectionError("unknown error in JSON-RPC") data = response.read() - conn.close() return json.loads(data) except JsonRpcConnectionError as exc: raise exc + except httplib.BadStatusLine: + return "CONNFAILURE" except Exception as exc: raise JsonRpcConnectionError("JSON-RPC connection failed. Err:" + repr(exc)) @@ -108,12 +109,23 @@ class JsonRpc(object): self.queryId += 1 request = {"method": method, "params": params, "id": currentId} - response = self.queryHTTP(request) - + #query can fail from keepalive timeout; keep retrying if it does, up + #to a reasonable limit, then raise (failure to access blockchain + #is a critical failure). Note that a real failure to connect (e.g. + #wrong port) is raised in queryHTTP directly. + response_received = False + for i in range(100): + response = self.queryHTTP(request) + if response != "CONNFAILURE": + response_received = True + break + #Failure means keepalive timed out, just make a new one + self.conn = httplib.HTTPConnection(self.host, self.port) + if not response_received: + raise JsonRpcConnectionError("Unable to connect over RPC") if response["id"] != currentId: raise JsonRpcConnectionError("invalid id returned by query") if response["error"] is not None: raise JsonRpcError(response["error"]) - return response["result"] diff --git a/jmclient/jmclient/support.py b/jmclient/jmclient/support.py index d8236fc..42d888d 100644 --- a/jmclient/jmclient/support.py +++ b/jmclient/jmclient/support.py @@ -263,7 +263,8 @@ def choose_sweep_orders(offers, total_txfee, n, chooseOrdersBy, - ignored_makers=None): + ignored_makers=None, + allowed_types=['swreloffer', 'swabsoffer']): """ choose an order given that we want to be left with no change i.e. sweep an entire group of utxos @@ -300,6 +301,7 @@ def choose_sweep_orders(offers, log.debug('choosing sweep orders for total_input_value = ' + str( total_input_value) + ' n=' + str(n)) + offers = [o for o in offers if o["ordertype"] in allowed_types] #Filter ignored makers and inappropriate amounts offers = [o for o in offers if o['counterparty'] not in ignored_makers] offers = [o for o in offers if o['minsize'] < total_input_value] diff --git a/jmclient/test/test_support.py b/jmclient/test/test_support.py index bf81bfc..a9b40df 100644 --- a/jmclient/test/test_support.py +++ b/jmclient/test/test_support.py @@ -98,7 +98,7 @@ def test_choose_orders(): assert total_fee == 0 #here we doctor the orderbook; (a) include an absfee - #(b) add an unrecognized ordertype + #(b) add an unrecognized ordertype (does not raise, ignores) #(c) put an order with wrong minsize orderbook.append({u'counterparty': u'fake', u'ordertype': u'swabsoffer', u'oid': 0, @@ -114,8 +114,7 @@ def test_choose_orders(): u'ordertype': u'dummyoffer', u'oid': 0, u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700, u'cjfee': 9000}) - with pytest.raises(RuntimeError) as e_info: - result, cjamount, total_fee = choose_sweep_orders(orderbook, + result, cjamount, total_fee = choose_sweep_orders(orderbook, 50000000, 30000, 8, diff --git a/jmdaemon/jmdaemon/__init__.py b/jmdaemon/jmdaemon/__init__.py index 9eba1ca..0b5ce8c 100644 --- a/jmdaemon/jmdaemon/__init__.py +++ b/jmdaemon/jmdaemon/__init__.py @@ -9,7 +9,7 @@ from jmbase.support import get_log from .message_channel import MessageChannel, MessageChannelCollection from .orderbookwatch import OrderbookWatch from jmbase import commands -from .daemon_protocol import JMDaemonServerProtocolFactory +from .daemon_protocol import JMDaemonServerProtocolFactory, JMDaemonServerProtocol from .protocol import (COMMAND_PREFIX, ORDER_KEYS, NICK_HASH_LENGTH, NICK_MAX_ENCODED, JM_VERSION, JOINMARKET_NICK_HEADER) from .message_channel import MessageChannelCollection