From c654de05ff57abb4d3b895e307b239dab914f84f Mon Sep 17 00:00:00 2001 From: AdamISZ Date: Thu, 20 Jun 2019 18:11:56 +0200 Subject: [PATCH] Wallet and blockchain refactoring Introduces WalletService object which is in control of blockchain and wallet access. The service manages a single transaction monitoring loop, instead of multiple, and allows updates to the wallet from external sources to be handled in real time, so that both Qt and other apps (yg) can respond to deposits or withdrawals automatically. The refactoring also controls access to both wallet and blockchain so that client apps (Taker, Maker) will not need to be changed for future new versions e.g. client-side filtering. Also updates and improves Wallet Tab behaviour in Qt (memory of expansion state). Additionally, blockchain sync is now --fast by default, with the former default of detailed sync being renamed --recoversync. --- jmbase/jmbase/__init__.py | 3 +- jmbase/jmbase/support.py | 3 + jmbitcoin/jmbitcoin/secp256k1_transaction.py | 14 + jmclient/jmclient/__init__.py | 5 +- jmclient/jmclient/blockchaininterface.py | 606 +---------------- jmclient/jmclient/client_protocol.py | 90 +-- jmclient/jmclient/maker.py | 68 +- jmclient/jmclient/output.py | 16 +- jmclient/jmclient/storage.py | 6 + jmclient/jmclient/taker.py | 112 ++-- jmclient/jmclient/taker_utils.py | 26 +- jmclient/jmclient/wallet.py | 187 ++++-- jmclient/jmclient/wallet_service.py | 661 +++++++++++++++++++ jmclient/jmclient/wallet_utils.py | 99 +-- jmclient/jmclient/yieldgenerator.py | 53 +- jmclient/test/commontest.py | 46 +- jmclient/test/test_blockchaininterface.py | 113 ++-- jmclient/test/test_client_protocol.py | 14 +- jmclient/test/test_coinjoin.py | 95 +-- jmclient/test/test_maker.py | 6 +- jmclient/test/test_payjoin.py | 41 +- jmclient/test/test_podle.py | 6 +- jmclient/test/test_storage.py | 1 - jmclient/test/test_taker.py | 46 +- jmclient/test/test_tx_creation.py | 78 +-- jmclient/test/test_utxomanager.py | 16 +- jmclient/test/test_wallet.py | 12 +- jmclient/test/test_wallets.py | 24 +- jmclient/test/test_yieldgenerator.py | 23 +- scripts/add-utxo.py | 20 +- scripts/cli_options.py | 20 +- scripts/joinmarket-qt.py | 162 +++-- scripts/receive-payjoin.py | 10 +- scripts/sendpayment.py | 25 +- scripts/tumbler.py | 22 +- scripts/yg-privacyenhanced.py | 4 +- test/common.py | 18 +- test/test_full_coinjoin.py | 2 +- test/test_segwit.py | 29 +- test/ygrunner.py | 17 +- 40 files changed, 1568 insertions(+), 1231 deletions(-) create mode 100644 jmclient/jmclient/wallet_service.py diff --git a/jmbase/jmbase/__init__.py b/jmbase/jmbase/__init__.py index 67417c8..83d4ddc 100644 --- a/jmbase/jmbase/__init__.py +++ b/jmbase/jmbase/__init__.py @@ -4,6 +4,7 @@ from builtins import * from .support import (get_log, chunks, debug_silence, jmprint, joinmarket_alert, core_alert, get_password, - set_logging_level, set_logging_color) + set_logging_level, set_logging_color, + JM_WALLET_NAME_PREFIX) from .commands import * diff --git a/jmbase/jmbase/support.py b/jmbase/jmbase/support.py index 4330b29..f85c043 100644 --- a/jmbase/jmbase/support.py +++ b/jmbase/jmbase/support.py @@ -5,6 +5,9 @@ from builtins import * # noqa: F401 import logging from getpass import getpass +# global Joinmarket constants +JM_WALLET_NAME_PREFIX = "joinmarket-wallet-" + from chromalog.log import ( ColorizingStreamHandler, ColorizingFormatter, diff --git a/jmbitcoin/jmbitcoin/secp256k1_transaction.py b/jmbitcoin/jmbitcoin/secp256k1_transaction.py index 031bcc4..eb5c5f3 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_transaction.py +++ b/jmbitcoin/jmbitcoin/secp256k1_transaction.py @@ -9,6 +9,8 @@ import copy import re import os import struct +# note, only used for non-cryptographic randomness: +import random from jmbitcoin.secp256k1_main import * from jmbitcoin.bech32 import * @@ -871,3 +873,15 @@ def mktx(ins, outs, version=1, locktime=0): txobj["outs"].append(outobj) return serialize(txobj) +def make_shuffled_tx(ins, outs, deser=True, version=1, locktime=0): + """ Simple utility to ensure transaction + inputs and outputs are randomly ordered. + Can possibly be replaced by BIP69 in future + """ + random.shuffle(ins) + random.shuffle(outs) + tx = mktx(ins, outs, version=version, locktime=locktime) + if deser: + return deserialize(tx) + else: + return tx \ No newline at end of file diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index 4f2b158..ac53155 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -16,7 +16,7 @@ from .taker import Taker, P2EPTaker from .wallet import (Mnemonic, estimate_tx_fee, WalletError, BaseWallet, ImportWalletMixin, BIP39WalletMixin, BIP32Wallet, BIP49Wallet, LegacyWallet, SegwitWallet, SegwitLegacyWallet, UTXOManager, - WALLET_IMPLEMENTATIONS, make_shuffled_tx) + WALLET_IMPLEMENTATIONS) from .storage import (Argon2Hash, Storage, StorageError, StoragePasswordError, VolatileStorage) from .cryptoengine import BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, EngineError @@ -24,7 +24,7 @@ from .configure import ( load_program_config, get_p2pk_vbyte, jm_single, get_network, validate_address, get_irc_mchannels, get_blockchain_interface_instance, get_p2sh_vbyte, set_config, is_segwit_mode, is_native_segwit_mode) -from .blockchaininterface import (BlockchainInterface, sync_wallet, +from .blockchaininterface import (BlockchainInterface, RegtestBitcoinCoreInterface, BitcoinCoreInterface) from .electruminterface import ElectrumInterface from .client_protocol import (JMTakerClientProtocol, JMClientProtocolFactory, @@ -46,6 +46,7 @@ from .wallet_utils import ( wallet_tool_main, wallet_generate_recover_bip39, open_wallet, open_test_wallet_maybe, create_wallet, get_wallet_cls, get_wallet_path, wallet_display, get_utxos_enabled_disabled) +from .wallet_service import WalletService from .maker import Maker, P2EPMaker from .yieldgenerator import YieldGenerator, YieldGeneratorBasic, ygmain # Set default logging handler to avoid "No handler found" warnings. diff --git a/jmclient/jmclient/blockchaininterface.py b/jmclient/jmclient/blockchaininterface.py index b219f9f..3e6e8a0 100644 --- a/jmclient/jmclient/blockchaininterface.py +++ b/jmclient/jmclient/blockchaininterface.py @@ -3,34 +3,21 @@ from __future__ import (absolute_import, division, from builtins import * # noqa: F401 import abc -import ast import random import sys -import time -import binascii -from copy import deepcopy from decimal import Decimal from twisted.internet import reactor, task import jmbitcoin as btc from jmclient.jsonrpc import JsonRpcConnectionError, JsonRpcError -from jmclient.configure import get_p2pk_vbyte, jm_single +from jmclient.configure import jm_single from jmbase.support import get_log, jmprint -log = get_log() - +# an inaccessible blockheight; consider rewriting in 1900 years +INF_HEIGHT = 10**8 -def sync_wallet(wallet, fast=False): - """Wrapper function to choose fast syncing where it's - both possible and requested. - """ - if fast and ( - isinstance(jm_single().bc_interface, BitcoinCoreInterface) or isinstance( - jm_single().bc_interface, RegtestBitcoinCoreInterface)): - jm_single().bc_interface.sync_wallet(wallet, fast=True) - else: - jm_single().bc_interface.sync_wallet(wallet) +log = get_log() class BlockchainInterface(object): __metaclass__ = abc.ABCMeta @@ -38,150 +25,12 @@ class BlockchainInterface(object): def __init__(self): pass - def sync_wallet(self, wallet, restart_cb=None): - """Default behaviour is for Core and similar interfaces. - If address sync fails, flagged with wallet_synced value; - do not attempt to sync_unspent in that case. - """ - self.sync_addresses(wallet, restart_cb) - if self.wallet_synced: - self.sync_unspent(wallet) - - @staticmethod - def get_wallet_name(wallet): - return 'joinmarket-wallet-' + wallet.get_wallet_id() - - @abc.abstractmethod - def sync_addresses(self, wallet): - """Finds which addresses have been used""" - - @abc.abstractmethod - def sync_unspent(self, wallet): - """Finds the unspent transaction outputs belonging to this wallet""" - def is_address_imported(self, addr): try: return self.rpc('getaccount', [addr]) != '' except JsonRpcError: return len(self.rpc('getaddressinfo', [addr])['labels']) > 0 - def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr, - wallet_name=None, timeoutfun=None, spentfun=None, txid_flag=True, - n=0, c=1, vb=None): - """Given a deserialized transaction txd, - callback functions for broadcast and confirmation of the transaction, - an address to import, and a callback function for timeout, set up - a polling loop to check for events on the transaction. Also optionally set - to trigger "confirmed" callback on number of confirmations c. Also checks - for spending (if spentfun is not None) of the outpoint n. - If txid_flag is True, we create a watcher loop on the txid (hence only - really usable in a segwit context, and only on fully formed transactions), - else we create a watcher loop on the output set of the transaction (taken - from the outs field of the txd). - """ - if not vb: - vb = get_p2pk_vbyte() - if isinstance(self, BitcoinCoreInterface) or isinstance(self, - RegtestBitcoinCoreInterface): - #This code ensures that a walletnotify is triggered, by - #ensuring that at least one of the output addresses is - #imported into the wallet (note the sweep special case, where - #none of the output addresses belong to me). - one_addr_imported = False - for outs in txd['outs']: - addr = btc.script_to_address(outs['script'], vb) - try: - if self.is_address_imported(addr): - one_addr_imported = True - break - except JsonRpcError as e: - log.debug("Failed to getaccount for address: " + addr) - log.debug("This is normal for bech32 addresses.") - continue - if not one_addr_imported: - try: - self.rpc('importaddress', [notifyaddr, 'joinmarket-notify', False]) - except JsonRpcError as e: - #In edge case of address already controlled - #by another account, warn but do not quit in middle of tx. - #Can occur if destination is owned in Core wallet. - if e.code == -4 and e.message == "The wallet already " + \ - "contains the private key for this address or script": - log.warn("WARNING: Failed to import address: " + notifyaddr) - #No other error should be possible - else: - raise - - #Warning! In case of txid_flag false, this is *not* a valid txid, - #but only a hash of an incomplete transaction serialization. - txid = btc.txhash(btc.serialize(txd)) - if not txid_flag: - tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) - loop = task.LoopingCall(self.outputs_watcher, wallet_name, notifyaddr, - tx_output_set, unconfirmfun, confirmfun, - timeoutfun) - log.debug("Created watcher loop for address: " + notifyaddr) - loopkey = notifyaddr - else: - loop = task.LoopingCall(self.tx_watcher, txd, unconfirmfun, confirmfun, - spentfun, c, n) - log.debug("Created watcher loop for txid: " + txid) - loopkey = txid - self.tx_watcher_loops[loopkey] = [loop, False, False, False] - #Hardcoded polling interval, but in any case it can be very short. - loop.start(5.0) - #Give up on un-broadcast transactions and broadcast but not confirmed - #transactions as per settings in the config. - reactor.callLater(float(jm_single().config.get("TIMEOUT", - "unconfirm_timeout_sec")), self.tx_network_timeout, loopkey) - confirm_timeout_sec = int(jm_single().config.get( - "TIMEOUT", "confirm_timeout_hours")) * 3600 - reactor.callLater(confirm_timeout_sec, self.tx_timeout, txd, loopkey, timeoutfun) - - def tx_network_timeout(self, loopkey): - """If unconfirm has not been called by the time this - is triggered, we abandon monitoring, assuming the tx has - not been broadcast. - """ - if not self.tx_watcher_loops[loopkey][1]: - log.info("Abandoning monitoring of un-broadcast tx for: " + str(loopkey)) - if self.tx_watcher_loops[loopkey][0].running: - self.tx_watcher_loops[loopkey][0].stop() - - def tx_timeout(self, txd, loopkey, timeoutfun): - """Assuming we are watching for an already-broadcast - transaction, give up once this triggers if confirmation has not occurred. - """ - if not loopkey in self.tx_watcher_loops: - #Occurs if the tx has already confirmed before this - return - if not self.tx_watcher_loops[loopkey][2]: - #Not confirmed after prescribed timeout in hours; give up - log.info("Timed out waiting for confirmation of: " + str(loopkey)) - if self.tx_watcher_loops[loopkey][0].running: - self.tx_watcher_loops[loopkey][0].stop() - if timeoutfun: - timeoutfun(txd, loopkey) - - @abc.abstractmethod - def outputs_watcher(self, wallet_name, notifyaddr, tx_output_set, - unconfirmfun, confirmfun, timeoutfun): - """Given a key for the watcher loop (notifyaddr), a wallet name (account), - a set of outputs, and unconfirm, confirm and timeout callbacks, - check to see if a transaction matching that output set has appeared in - the wallet. Call the callbacks and update the watcher loop state. - End the loop when the confirmation has been seen (no spent monitoring here). - """ - - @abc.abstractmethod - def tx_watcher(self, txd, unconfirmfun, confirmfun, spentfun, c, n): - """Called at a polling interval, checks if the given deserialized - transaction (which must be fully signed) is (a) broadcast, (b) confirmed - and (c) spent from at index n, and notifies confirmation if number - of confs = c. - TODO: Deal with conflicts correctly. Here just abandons monitoring. - """ - @abc.abstractmethod def pushtx(self, txhex): """pushes tx to the network, returns False if failed""" @@ -233,16 +82,6 @@ class ElectrumWalletInterface(BlockchainInterface): #pragma: no cover def sync_unspent(self, wallet): log.debug("Dummy electrum interface, no sync unspent") - def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr): - log.debug("Dummy electrum interface, no add tx notify") - - def outputs_watcher(self, wallet_name, notifyaddr, - tx_output_set, uf, cf, tf): - log.debug("Dummy electrum interface, no outputs watcher") - - def tx_watcher(self, txd, ucf, cf, sf, c, n): - log.debug("Dummy electrum interface, no tx watcher") - def pushtx(self, txhex, timeout=10): #synchronous send from electrum.transaction import Transaction @@ -319,7 +158,6 @@ class BitcoinCoreInterface(BlockchainInterface): def __init__(self, jsonRpc, network): super(BitcoinCoreInterface, self).__init__() self.jsonRpc = jsonRpc - self.fast_sync_called = False blockchainInfo = self.jsonRpc.call("getblockchaininfo", []) actualNet = blockchainInfo['chain'] @@ -327,13 +165,6 @@ class BitcoinCoreInterface(BlockchainInterface): if netmap[actualNet] != network: raise Exception('wrong network configured') - self.txnotify_fun = [] - self.wallet_synced = False - #task.LoopingCall objects that track transactions, keyed by txids. - #Format: {"txid": (loop, unconfirmed true/false, confirmed true/false, - #spent true/false), ..} - self.tx_watcher_loops = {} - def get_block(self, blockheight): """Returns full serialized block at a given height. """ @@ -346,7 +177,7 @@ class BitcoinCoreInterface(BlockchainInterface): def rpc(self, method, args): if method not in ['importaddress', 'walletpassphrase', 'getaccount', 'gettransaction', 'getrawtransaction', 'gettxout', - 'importmulti']: + 'importmulti', 'listtransactions', 'getblockcount']: log.debug('rpc: ' + method + " " + str(args)) res = self.jsonRpc.call(method, args) return res @@ -357,8 +188,6 @@ class BitcoinCoreInterface(BlockchainInterface): of another account/label (see console output), and quits. Do NOT use for in-run imports, use rpc('importaddress',..) instead. """ - log.debug('importing ' + str(len(addr_list)) + - ' addresses with label ' + wallet_name) requests = [] for addr in addr_list: requests.append({ @@ -395,7 +224,7 @@ class BitcoinCoreInterface(BlockchainInterface): but in some cases a rescan is not required (if the address is known to be new/unused). For that case use import_addresses instead. """ - self.import_addresses(addr_list, wallet_name, restart_cb) + self.import_addresses(addr_list, wallet_name) if jm_single().config.get("BLOCKCHAIN", "blockchain_source") != 'regtest': #pragma: no cover #Exit conditions cannot be included in tests @@ -409,218 +238,6 @@ class BitcoinCoreInterface(BlockchainInterface): jmprint(restart_msg, "important") sys.exit(0) - def sync_wallet(self, wallet, fast=False, restart_cb=None): - #trigger fast sync if the index_cache is available - #(and not specifically disabled). - if fast: - self.sync_wallet_fast(wallet) - self.fast_sync_called = True - return - super(BitcoinCoreInterface, self).sync_wallet(wallet, restart_cb=restart_cb) - self.fast_sync_called = False - - def sync_wallet_fast(self, wallet): - """Exploits the fact that given an index_cache, - all addresses necessary should be imported, so we - can just list all used addresses to find the right - index values. - """ - self.get_address_usages(wallet) - self.sync_unspent(wallet) - - def get_address_usages(self, wallet): - """Use rpc `listaddressgroupings` to locate all used - addresses in the account (whether spent or unspent outputs). - This will not result in a full sync if working with a new - Bitcoin Core instance, in which case "fast" should have been - specifically disabled by the user. - """ - wallet_name = self.get_wallet_name(wallet) - agd = self.rpc('listaddressgroupings', []) - #flatten all groups into a single list; then, remove duplicates - fagd = (tuple(item) for sublist in agd for item in sublist) - #"deduplicated flattened address grouping data" = dfagd - dfagd = set(fagd) - used_addresses = set() - for addr_info in dfagd: - if len(addr_info) < 3 or addr_info[2] != wallet_name: - continue - used_addresses.add(addr_info[0]) - - #for a first run, import first chunk - if not used_addresses: - log.info("Detected new wallet, performing initial import") - # delegate inital address import to sync_addresses - # this should be fast because "getaddressesbyaccount" should return - # an empty list in this case - self.sync_addresses(wallet) - self.wallet_synced = True - return - - #Wallet has been used; scan forwards. - log.debug("Fast sync in progress. Got this many used addresses: " + str( - len(used_addresses))) - #Need to have wallet.index point to the last used address - #Algo: - # 1. Scan batch 1 of each branch, record matched wallet addresses. - # 2. Check if all addresses in 'used addresses' have been matched, if - # so, break. - # 3. Repeat the above for batch 2, 3.. up to max 20 batches. - # 4. If after all 20 batches not all used addresses were matched, - # quit with error. - # 5. Calculate used indices. - # 6. If all used addresses were matched, set wallet index to highest - # found index in each branch and mark wallet sync complete. - #Rationale for this algo: - # Retrieving addresses is a non-zero computational load, so batching - # and then caching allows a small sync to complete *reasonably* - # quickly while a larger one is not really negatively affected. - # The downside is another free variable, batch size, but this need - # not be exposed to the user; it is not the same as gap limit, in fact, - # the concept of gap limit does not apply to this kind of sync, which - # *assumes* that the most recent usage of addresses is indeed recorded. - remaining_used_addresses = used_addresses.copy() - addresses, saved_indices = self._collect_addresses_init(wallet) - for addr in addresses: - remaining_used_addresses.discard(addr) - - BATCH_SIZE = 100 - MAX_ITERATIONS = 20 - current_indices = deepcopy(saved_indices) - for j in range(MAX_ITERATIONS): - if not remaining_used_addresses: - break - for addr in \ - self._collect_addresses_gap(wallet, gap_limit=BATCH_SIZE): - remaining_used_addresses.discard(addr) - - # increase wallet indices for next iteration - for md in current_indices: - current_indices[md][0] += BATCH_SIZE - current_indices[md][1] += BATCH_SIZE - self._rewind_wallet_indices(wallet, current_indices, - current_indices) - else: - self._rewind_wallet_indices(wallet, saved_indices, saved_indices) - raise Exception("Failed to sync in fast mode after 20 batches; " - "please re-try wallet sync without --fast flag.") - - # creating used_indices on-the-fly would be more efficient, but the - # overall performance gain is probably negligible - used_indices = self._get_used_indices(wallet, used_addresses) - self._rewind_wallet_indices(wallet, used_indices, saved_indices) - self.wallet_synced = True - - def sync_addresses(self, wallet, restart_cb=None): - log.debug("requesting detailed wallet history") - wallet_name = self.get_wallet_name(wallet) - - addresses, saved_indices = self._collect_addresses_init(wallet) - try: - imported_addresses = set(self.rpc('getaddressesbyaccount', - [wallet_name])) - except JsonRpcError: - if wallet_name in self.rpc('listlabels', []): - imported_addresses = set(self.rpc('getaddressesbylabel', - [wallet_name]).keys()) - else: - imported_addresses = set() - - if not addresses.issubset(imported_addresses): - self.add_watchonly_addresses(addresses - imported_addresses, - wallet_name, restart_cb) - return - - used_addresses_gen = (tx['address'] - for tx in self._yield_transactions(wallet_name) - if tx['category'] == 'receive') - - used_indices = self._get_used_indices(wallet, used_addresses_gen) - log.debug("got used indices: {}".format(used_indices)) - gap_limit_used = not self._check_gap_indices(wallet, used_indices) - self._rewind_wallet_indices(wallet, used_indices, saved_indices) - - new_addresses = self._collect_addresses_gap(wallet) - if not new_addresses.issubset(imported_addresses): - log.debug("Syncing iteration finished, additional step required") - self.add_watchonly_addresses(new_addresses - imported_addresses, - wallet_name, restart_cb) - self.wallet_synced = False - elif gap_limit_used: - log.debug("Syncing iteration finished, additional step required") - self.wallet_synced = False - else: - log.debug("Wallet successfully synced") - self._rewind_wallet_indices(wallet, used_indices, saved_indices) - self.wallet_synced = True - - @staticmethod - def _rewind_wallet_indices(wallet, used_indices, saved_indices): - for md in used_indices: - for int_type in (0, 1): - index = max(used_indices[md][int_type], - saved_indices[md][int_type]) - wallet.set_next_index(md, int_type, index, force=True) - - @staticmethod - def _get_used_indices(wallet, addr_gen): - indices = {x: [0, 0] for x in range(wallet.max_mixdepth + 1)} - - for addr in addr_gen: - if not wallet.is_known_addr(addr): - continue - md, internal, index = wallet.get_details( - wallet.addr_to_path(addr)) - if internal not in (0, 1): - assert internal == 'imported' - continue - indices[md][internal] = max(indices[md][internal], index + 1) - - return indices - - @staticmethod - def _check_gap_indices(wallet, used_indices): - for md in used_indices: - for internal in (0, 1): - if used_indices[md][internal] >\ - max(wallet.get_next_unused_index(md, internal), 0): - return False - return True - - @staticmethod - def _collect_addresses_init(wallet): - addresses = set() - saved_indices = dict() - - for md in range(wallet.max_mixdepth + 1): - saved_indices[md] = [0, 0] - for internal in (0, 1): - next_unused = wallet.get_next_unused_index(md, internal) - for index in range(next_unused): - addresses.add(wallet.get_addr(md, internal, index)) - for index in range(wallet.gap_limit): - addresses.add(wallet.get_new_addr(md, internal)) - wallet.set_next_index(md, internal, next_unused) - saved_indices[md][internal] = next_unused - for path in wallet.yield_imported_paths(md): - addresses.add(wallet.get_addr_path(path)) - - return addresses, saved_indices - - @staticmethod - def _collect_addresses_gap(wallet, gap_limit=None): - gap_limit = gap_limit or wallet.gap_limit - addresses = set() - - for md in range(wallet.max_mixdepth + 1): - for internal in (True, False): - old_next = wallet.get_next_unused_index(md, internal) - for index in range(gap_limit): - addresses.add(wallet.get_new_addr(md, internal)) - wallet.set_next_index(md, internal, old_next) - - return addresses - def _yield_transactions(self, wallet_name): batch_size = 1000 iteration = 0 @@ -634,47 +251,6 @@ class BitcoinCoreInterface(BlockchainInterface): return iteration += 1 - def start_unspent_monitoring(self, wallet): - self.unspent_monitoring_loop = task.LoopingCall(self.sync_unspent, wallet) - self.unspent_monitoring_loop.start(1.0) - - def stop_unspent_monitoring(self): - self.unspent_monitoring_loop.stop() - - def sync_unspent(self, wallet): - st = time.time() - wallet_name = self.get_wallet_name(wallet) - wallet.reset_utxos() - - listunspent_args = [] - if 'listunspent_args' in jm_single().config.options('POLICY'): - listunspent_args = ast.literal_eval(jm_single().config.get( - 'POLICY', 'listunspent_args')) - - unspent_list = self.rpc('listunspent', listunspent_args) - for u in unspent_list: - if not wallet.is_known_addr(u['address']): - continue - self._add_unspent_utxo(wallet, u) - et = time.time() - log.debug('bitcoind sync_unspent took ' + str((et - st)) + 'sec') - self.wallet_synced = True - - @staticmethod - def _add_unspent_utxo(wallet, utxo): - """ - Add a UTXO as returned by rpc's listunspent call to the wallet. - - params: - wallet: wallet - utxo: single utxo dict as returned by listunspent - """ - txid = binascii.unhexlify(utxo['txid']) - script = binascii.unhexlify(utxo['scriptPubKey']) - value = int(Decimal(str(utxo['amount'])) * Decimal('1e8')) - - wallet.add_utxo(txid, int(utxo['vout']), script, value) - def get_deser_from_gettransaction(self, rpcretval): """Get full transaction deserialization from a call to `gettransaction` @@ -686,156 +262,36 @@ class BitcoinCoreInterface(BlockchainInterface): hexval = str(rpcretval["hex"]) return btc.deserialize(hexval) - def outputs_watcher(self, wallet_name, notifyaddr, tx_output_set, - unconfirmfun, confirmfun, timeoutfun): - """Given a key for the watcher loop (notifyaddr), a wallet name (label), - a set of outputs, and unconfirm, confirm and timeout callbacks, - check to see if a transaction matching that output set has appeared in - the wallet. Call the callbacks and update the watcher loop state. - End the loop when the confirmation has been seen (no spent monitoring here). + def list_transactions(self, num): + """ Return a list of the last `num` transactions seen + in the wallet (under any label/account). """ - wl = self.tx_watcher_loops[notifyaddr] - txlist = self.rpc("listtransactions", ["*", 100, 0, True]) - for tx in txlist[::-1]: - #changed syntax in 0.14.0; allow both syntaxes - try: - res = self.rpc("gettransaction", [tx["txid"], True]) - except: - try: - res = self.rpc("gettransaction", [tx["txid"], 1]) - except JsonRpcError as e: - #This should never happen (gettransaction is a wallet rpc). - log.warn("Failed gettransaction call; JsonRpcError") - res = None - except Exception as e: - log.warn("Failed gettransaction call; unexpected error:") - log.warn(str(e)) - res = None - if not res: - continue - if "confirmations" not in res: - log.debug("Malformed gettx result: " + str(res)) - return - txd = self.get_deser_from_gettransaction(res) - if txd is None: - continue - txos = set([(sv['script'], sv['value']) for sv in txd['outs']]) - if not txos == tx_output_set: - continue - #Here we have found a matching transaction in the wallet. - real_txid = btc.txhash(btc.serialize(txd)) - if not wl[1] and res["confirmations"] == 0: - log.debug("Tx: " + str(real_txid) + " seen on network.") - unconfirmfun(txd, real_txid) - wl[1] = True - return - if not wl[2] and res["confirmations"] > 0: - log.debug("Tx: " + str(real_txid) + " has " + str( - res["confirmations"]) + " confirmations.") - confirmfun(txd, real_txid, res["confirmations"]) - wl[2] = True - wl[0].stop() - return - if res["confirmations"] < 0: - log.debug("Tx: " + str(real_txid) + " has a conflict. Abandoning.") - wl[0].stop() - return + return self.rpc("listtransactions", ["*", num, 0, True]) - def tx_watcher(self, txd, unconfirmfun, confirmfun, spentfun, c, n): - """Called at a polling interval, checks if the given deserialized - transaction (which must be fully signed) is (a) broadcast, (b) confirmed - and (c) spent from at index n, and notifies confirmation if number - of confs = c. - TODO: Deal with conflicts correctly. Here just abandons monitoring. + def get_transaction(self, txid): + """ Returns a serialized transaction for txid txid, + in hex as returned by Bitcoin Core rpc, or None + if no transaction can be retrieved. Works also for + watch-only wallets. """ - txid = btc.txhash(btc.serialize(txd)) - wl = self.tx_watcher_loops[txid] + #changed syntax in 0.14.0; allow both syntaxes try: - res = self.rpc('gettransaction', [txid, True]) - except JsonRpcError as e: - return - if not res: - return + res = self.rpc("gettransaction", [txid, True]) + except: + try: + res = self.rpc("gettransaction", [txid, 1]) + except JsonRpcError as e: + #This should never happen (gettransaction is a wallet rpc). + log.warn("Failed gettransaction call; JsonRpcError") + return None + except Exception as e: + log.warn("Failed gettransaction call; unexpected error:") + log.warn(str(e)) + return None if "confirmations" not in res: - log.debug("Malformed gettx result: " + str(res)) - return - if not wl[1] and res["confirmations"] == 0: - log.debug("Tx: " + str(txid) + " seen on network.") - unconfirmfun(txd, txid) - wl[1] = True - return - if not wl[2] and res["confirmations"] > 0: - log.debug("Tx: " + str(txid) + " has " + str( - res["confirmations"]) + " confirmations.") - confirmfun(txd, txid, res["confirmations"]) - if c <= res["confirmations"]: - wl[2] = True - #Note we do not stop the monitoring loop when - #confirmations occur, since we are also monitoring for spending. - return - if res["confirmations"] < 0: - log.debug("Tx: " + str(txid) + " has a conflict. Abandoning.") - wl[0].stop() - return - if not spentfun or wl[3]: - return - #To trigger the spent callback, we check if this utxo outpoint appears in - #listunspent output with 0 or more confirmations. Note that this requires - #we have added the destination address to the watch-only wallet, otherwise - #that outpoint will not be returned by listunspent. - res2 = self.rpc('listunspent', [0, 999999]) - if not res2: - return - txunspent = False - for r in res2: - if "txid" not in r: - continue - if txid == r["txid"] and n == r["vout"]: - txunspent = True - break - if not txunspent: - #We need to find the transaction which spent this one; - #assuming the address was added to the wallet, then this - #transaction must be in the recent list retrieved via listunspent. - #For each one, use gettransaction to check its inputs. - #This is a bit expensive, but should only occur once. - txlist = self.rpc("listtransactions", ["*", 1000, 0, True]) - for tx in txlist[::-1]: - #changed syntax in 0.14.0; allow both syntaxes - try: - res = self.rpc("gettransaction", [tx["txid"], True]) - except: - try: - res = self.rpc("gettransaction", [tx["txid"], 1]) - except: - #This should never happen (gettransaction is a wallet rpc). - log.info("Failed any gettransaction call") - res = None - if not res: - continue - deser = self.get_deser_from_gettransaction(res) - if deser is None: - continue - for vin in deser["ins"]: - if not "outpoint" in vin: - #coinbases - continue - if vin["outpoint"]["hash"] == txid and vin["outpoint"]["index"] == n: - #recover the deserialized form of the spending transaction. - log.info("We found a spending transaction: " + \ - btc.txhash(binascii.unhexlify(res["hex"]))) - res2 = self.rpc("gettransaction", [tx["txid"], True]) - spending_deser = self.get_deser_from_gettransaction(res2) - if not spending_deser: - log.info("ERROR: could not deserialize spending tx.") - #Should never happen, it's a parsing bug. - #No point continuing to monitor, we just hope we - #can extract the secret by scanning blocks. - wl[3] = True - return - spentfun(spending_deser, vin["outpoint"]["hash"]) - wl[3] = True - return + log.warning("Malformed gettx result: " + str(res)) + return None + return res def pushtx(self, txhex): try: diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index 60f7c2a..4b99382 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -18,9 +18,8 @@ import hashlib import os import sys from jmbase import get_log -from jmclient import (jm_single, get_irc_mchannels, get_p2sh_vbyte, +from jmclient import (jm_single, get_irc_mchannels, RegtestBitcoinCoreInterface) -from .output import fmt_tx_data import jmbitcoin as btc jlog = get_log() @@ -263,72 +262,79 @@ class JMMakerClientProtocol(JMClientProtocol): self.finalized_offers[nick] = offer tx = btc.deserialize(txhex) self.finalized_offers[nick]["txd"] = tx - jm_single().bc_interface.add_tx_notify(tx, self.unconfirm_callback, - self.confirm_callback, offer["cjaddr"], - wallet_name=jm_single().bc_interface.get_wallet_name( - self.client.wallet), - txid_flag=False, - vb=get_p2sh_vbyte()) + txid = btc.txhash(btc.serialize(tx)) + # we index the callback by the out-set of the transaction, + # because the txid is not known until all scriptSigs collected + # (hence this is required for Makers, but not Takers). + # For more info see WalletService.transaction_monitor(): + txinfo = tuple((x["script"], x["value"]) for x in tx["outs"]) + self.client.wallet_service.register_callbacks([self.unconfirm_callback], + txinfo, "unconfirmed") + self.client.wallet_service.register_callbacks([self.confirm_callback], + txinfo, "confirmed") + + task.deferLater(reactor, float(jm_single().config.getint("TIMEOUT", + "unconfirm_timeout_sec")), + self.client.wallet_service.check_callback_called, + txinfo, self.unconfirm_callback, "unconfirmed", + "transaction with outputs: " + str(txinfo) + " not broadcast.") + d = self.callRemote(commands.JMTXSigs, nick=nick, sigs=json.dumps(sigs)) self.defaultCallbacks(d) return {"accepted": True} - def unconfirm_callback(self, txd, txid): - #find the offer for this tx - offerinfo = None + def tx_match(self, txd): for k,v in iteritems(self.finalized_offers): #Tx considered defined by its output set if v["txd"]["outs"] == txd["outs"]: offerinfo = v break + else: + return False + return offerinfo + + def unconfirm_callback(self, txd, txid): + #find the offer for this tx + offerinfo = self.tx_match(txd) if not offerinfo: - jlog.info("Failed to find notified unconfirmed transaction: " + txid) - return - removed_utxos = self.client.wallet.remove_old_utxos(txd) - jlog.info('saw tx on network, removed_utxos=\n{}'.format('\n'.join( - '{} - {}'.format(u, fmt_tx_data(tx_data, self.client.wallet)) - for u, tx_data in removed_utxos.items()))) + return False to_cancel, to_announce = self.client.on_tx_unconfirmed(offerinfo, - txid, removed_utxos) + txid) self.client.modify_orders(to_cancel, to_announce) + + txinfo = tuple((x["script"], x["value"]) for x in txd["outs"]) + confirm_timeout_sec = float(jm_single().config.get( + "TIMEOUT", "confirm_timeout_hours")) * 3600 + task.deferLater(reactor, confirm_timeout_sec, + self.client.wallet_service.check_callback_called, + txinfo, self.confirm_callback, "confirmed", + "transaction with outputs " + str(txinfo) + " not confirmed.") + d = self.callRemote(commands.JMAnnounceOffers, to_announce=json.dumps(to_announce), to_cancel=json.dumps(to_cancel), offerlist=json.dumps(self.client.offerlist)) self.defaultCallbacks(d) + return True - def confirm_callback(self, txd, txid, confirmations): + def confirm_callback(self, txd, txid, confirms): #find the offer for this tx - offerinfo = None - for k,v in iteritems(self.finalized_offers): - #Tx considered defined by its output set - if v["txd"]["outs"] == txd["outs"]: - offerinfo = v - break + offerinfo = self.tx_match(txd) if not offerinfo: - jlog.info("Failed to find notified unconfirmed transaction: " + txid) - return - jm_single().bc_interface.wallet_synced = False - jm_single().bc_interface.sync_unspent(self.client.wallet) - jlog.info('tx in a block: ' + txid) - self.wait_for_sync_loop = task.LoopingCall(self.modify_orders, offerinfo, - confirmations, txid) - self.wait_for_sync_loop.start(2.0) - - def modify_orders(self, offerinfo, confirmations, txid): - if not jm_single().bc_interface.wallet_synced: - return - self.wait_for_sync_loop.stop() + return False + jlog.info('tx in a block: ' + txid + ' with ' + str( + confirms) + ' confirmations.') to_cancel, to_announce = self.client.on_tx_confirmed(offerinfo, - confirmations, txid) + txid, confirms) self.client.modify_orders(to_cancel, to_announce) d = self.callRemote(commands.JMAnnounceOffers, - to_announce=json.dumps(to_announce), - to_cancel=json.dumps(to_cancel), - offerlist=json.dumps(self.client.offerlist)) + to_announce=json.dumps(to_announce), + to_cancel=json.dumps(to_cancel), + offerlist=json.dumps(self.client.offerlist)) self.defaultCallbacks(d) + return True class JMTakerClientProtocol(JMClientProtocol): diff --git a/jmclient/jmclient/maker.py b/jmclient/jmclient/maker.py index 6241e8b..501a9a6 100644 --- a/jmclient/jmclient/maker.py +++ b/jmclient/jmclient/maker.py @@ -13,7 +13,8 @@ from binascii import unhexlify from jmbitcoin import SerializationError, SerializationTruncationError import jmbitcoin as btc -from jmclient.wallet import estimate_tx_fee, make_shuffled_tx +from jmclient.wallet import estimate_tx_fee +from jmclient.wallet_service import WalletService from jmclient.configure import jm_single from jmbase.support import get_log from jmclient.support import calc_cj_fee, select_one_utxo @@ -24,9 +25,10 @@ from .cryptoengine import EngineError jlog = get_log() class Maker(object): - def __init__(self, wallet): + def __init__(self, wallet_service): self.active_orders = {} - self.wallet = wallet + assert isinstance(wallet_service, WalletService) + self.wallet_service = wallet_service self.nextoid = -1 self.offerlist = None self.sync_wait_loop = task.LoopingCall(self.try_to_create_my_orders) @@ -40,7 +42,7 @@ class Maker(object): is flagged as True. TODO: Use a deferred, probably. Note that create_my_orders() is defined by subclasses. """ - if not jm_single().bc_interface.wallet_synced: + if not self.wallet_service.synced: return self.offerlist = self.create_my_orders() self.sync_wait_loop.stop() @@ -89,7 +91,7 @@ class Maker(object): return reject(reason) try: - if not self.wallet.pubkey_has_script( + if not self.wallet_service.pubkey_has_script( unhexlify(cr_dict['P']), unhexlify(res[0]['script'])): raise EngineError() except EngineError: @@ -102,13 +104,14 @@ class Maker(object): if not utxos: #could not find funds return (False,) - self.wallet.update_cache_index() + # for index update persistence: + self.wallet_service.save_wallet() # Construct data for auth request back to taker. # Need to choose an input utxo pubkey to sign with # (no longer using the coinjoin pubkey from 0.2.0) # Just choose the first utxo in self.utxos and retrieve key from wallet. auth_address = utxos[list(utxos.keys())[0]]['address'] - auth_key = self.wallet.get_key_from_addr(auth_address) + auth_key = self.wallet_service.get_key_from_addr(auth_address) auth_pub = btc.privtopub(auth_key) btc_sig = btc.ecdsa_sign(kphex, auth_key) return (True, utxos, auth_pub, cj_addr, change_addr, btc_sig) @@ -137,11 +140,11 @@ class Maker(object): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) if utxo not in utxos: continue - script = self.wallet.addr_to_script(utxos[utxo]['address']) + script = self.wallet_service.addr_to_script(utxos[utxo]['address']) amount = utxos[utxo]['value'] our_inputs[index] = (script, amount) - txs = self.wallet.sign_tx(btc.deserialize(unhexlify(txhex)), our_inputs) + txs = self.wallet_service.sign_tx(btc.deserialize(unhexlify(txhex)), our_inputs) for index in our_inputs: sigmsg = unhexlify(txs['ins'][index]['script']) if 'txinwitness' in txs['ins'][index]: @@ -268,13 +271,13 @@ class Maker(object): """ @abc.abstractmethod - def on_tx_unconfirmed(self, cjorder, txid, removed_utxos): + def on_tx_unconfirmed(self, cjorder, txid): """Performs action on receipt of transaction into the mempool in the blockchain instance (e.g. announcing orders) """ @abc.abstractmethod - def on_tx_confirmed(self, cjorder, confirmations, txid): + def on_tx_confirmed(self, cjorder, txid, confirmations): """Performs actions on receipt of 1st confirmation of a transaction into a block (e.g. announce orders) """ @@ -292,15 +295,16 @@ class P2EPMaker(Maker): and partially signed transaction, thus information leak to snoopers is not possible. """ - def __init__(self, wallet, mixdepth, amount): + def __init__(self, wallet_service, mixdepth, amount): + super(P2EPMaker, self).__init__(wallet_service) self.receiving_amount = amount self.mixdepth = mixdepth # destination mixdepth must be different from that # which we source coins from; use the standard "next" - dest_mixdepth = (self.mixdepth + 1) % wallet.max_mixdepth + dest_mixdepth = (self.mixdepth + 1) % self.wallet_service.max_mixdepth # Select an unused destination in the external branch - self.destination_addr = wallet.get_new_addr(dest_mixdepth, 0) - super(P2EPMaker, self).__init__(wallet) + self.destination_addr = self.wallet_service.get_external_addr( + dest_mixdepth) # Callback to request user permission (for e.g. GUI) # args: (1) message, as string # returns: True or False @@ -371,15 +375,19 @@ class P2EPMaker(Maker): pass def on_tx_unconfirmed(self, txd, txid): + """ For P2EP Maker there is no "offer", so + the second argument is repurposed as the deserialized + transaction. + """ self.user_info("The transaction has been broadcast.") self.user_info("Txid is: " + txid) self.user_info("Transaction in detail: " + pprint.pformat(txd)) self.user_info("shutting down.") reactor.stop() - def on_tx_confirmed(self, offer, confirmations, txid): + def on_tx_confirmed(self, txd, txid, confirmations): # will not be reached except in testing - self.on_tx_unconfirmed(offer, confirmations) + self.on_tx_unconfirmed(txd, txid) def on_tx_received(self, nick, txhex): """ Called when the sender-counterparty has sent a transaction proposal. @@ -461,7 +469,7 @@ class P2EPMaker(Maker): self.user_info("Network transaction fee of fallback tx is: " + str( btc_fee) + " satoshis.") fee_est = estimate_tx_fee(len(tx['ins']), len(tx['outs']), - txtype=self.wallet.get_txtype()) + txtype=self.wallet_service.get_txtype()) fee_ok = False if btc_fee > 0.3 * fee_est and btc_fee < 3 * fee_est: fee_ok = True @@ -546,7 +554,7 @@ class P2EPMaker(Maker): # sweeping dust ... self.user_info("Choosing one coin at random") try: - my_utxos = self.wallet.select_utxos( + my_utxos = self.wallet_service.select_utxos( self.mixdepth, jm_single().DUST_THRESHOLD, select_fn=select_one_utxo) except: @@ -556,15 +564,15 @@ class P2EPMaker(Maker): # get an approximate required amount assuming 4 inputs, which is # fairly conservative (but guess by necessity). fee_for_select = estimate_tx_fee(len(tx['ins']) + 4, 2, - txtype=self.wallet.get_txtype()) + txtype=self.wallet_service.get_txtype()) approx_sum = max_sender_amt - self.receiving_amount + fee_for_select try: - my_utxos = self.wallet.select_utxos(self.mixdepth, approx_sum) + my_utxos = self.wallet_service.select_utxos(self.mixdepth, approx_sum) not_uih2 = True except Exception: # TODO probably not logical to always sweep here. self.user_info("Sweeping all coins in this mixdepth.") - my_utxos = self.wallet.get_utxos_by_mixdepth()[self.mixdepth] + my_utxos = self.wallet_service.get_utxos_by_mixdepth()[self.mixdepth] if my_utxos == {}: return self.no_coins_fallback() if not_uih2: @@ -582,7 +590,7 @@ class P2EPMaker(Maker): new_destination_amount = self.receiving_amount + my_total_in # estimate the required fee for the new version of the transaction total_ins = len(tx["ins"]) + len(my_utxos.keys()) - est_fee = estimate_tx_fee(total_ins, 2, txtype=self.wallet.get_txtype()) + est_fee = estimate_tx_fee(total_ins, 2, txtype=self.wallet_service.get_txtype()) self.user_info("We estimated a fee of: " + str(est_fee)) new_change_amount = total_sender_input + my_total_in - \ new_destination_amount - est_fee @@ -601,7 +609,7 @@ class P2EPMaker(Maker): # this call should never fail so no catch here. currentblock = jm_single().bc_interface.rpc( "getblockchaininfo", [])["blocks"] - new_tx = make_shuffled_tx(new_ins, new_outs, False, 2, currentblock) + new_tx = btc.make_shuffled_tx(new_ins, new_outs, False, 2, currentblock) new_tx_deser = btc.deserialize(new_tx) # sign our inputs before transfer @@ -610,16 +618,14 @@ class P2EPMaker(Maker): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) if utxo not in my_utxos: continue - script = self.wallet.addr_to_script(my_utxos[utxo]['address']) + script = self.wallet_service.addr_to_script(my_utxos[utxo]['address']) amount = my_utxos[utxo]['value'] our_inputs[index] = (script, amount) - txs = self.wallet.sign_tx(btc.deserialize(new_tx), our_inputs) - jm_single().bc_interface.add_tx_notify(txs, - self.on_tx_unconfirmed, self.on_tx_confirmed, - self.destination_addr, - wallet_name=jm_single().bc_interface.get_wallet_name(self.wallet), - txid_flag=False, vb=self.wallet._ENGINE.VBYTE) + txs = self.wallet_service.sign_tx(btc.deserialize(new_tx), our_inputs) + txinfo = tuple((x["script"], x["value"]) for x in txs["outs"]) + self.wallet_service.register_callbacks([self.on_tx_unconfirmed], txinfo, "unconfirmed") + self.wallet_service.register_callbacks([self.on_tx_confirmed], txinfo, "confirmed") # The blockchain interface just abandons monitoring if the transaction # is not broadcast before the configured timeout; we want to take # action in this case, so we add an additional callback to the reactor: diff --git a/jmclient/jmclient/output.py b/jmclient/jmclient/output.py index 7457123..94ea565 100644 --- a/jmclient/jmclient/output.py +++ b/jmclient/jmclient/output.py @@ -4,11 +4,11 @@ from builtins import * # noqa: F401 from binascii import hexlify -def fmt_utxos(utxos, wallet, prefix=''): +def fmt_utxos(utxos, wallet_service, prefix=''): output = [] for u in utxos: utxo_str = '{}{} - {}'.format( - prefix, fmt_utxo(u), fmt_tx_data(utxos[u], wallet)) + prefix, fmt_utxo(u), fmt_tx_data(utxos[u], wallet_service)) output.append(utxo_str) return '\n'.join(output) @@ -17,13 +17,13 @@ def fmt_utxo(utxo): return '{}:{}'.format(hexlify(utxo[0]).decode('ascii'), utxo[1]) -def fmt_tx_data(tx_data, wallet): +def fmt_tx_data(tx_data, wallet_service): return 'path: {}, address: {}, value: {}'.format( - wallet.get_path_repr(wallet.script_to_path(tx_data['script'])), - wallet.script_to_addr(tx_data['script']), tx_data['value']) + wallet_service.get_path_repr(wallet_service.script_to_path(tx_data['script'])), + wallet_service.script_to_addr(tx_data['script']), tx_data['value']) -def generate_podle_error_string(priv_utxo_pairs, to, ts, wallet, cjamount, +def generate_podle_error_string(priv_utxo_pairs, to, ts, wallet_service, cjamount, taker_utxo_age, taker_utxo_amtpercent): """Gives detailed error information on why commitment sourcing failed. """ @@ -64,9 +64,9 @@ def generate_podle_error_string(priv_utxo_pairs, to, ts, wallet, cjamount, "with 'python add-utxo.py --help'\n\n") errmsg += ("***\nFor reference, here are the utxos in your wallet:\n") - for md, utxos in wallet.get_utxos_by_mixdepth_().items(): + for md, utxos in wallet_service.get_utxos_by_mixdepth(hexfmt=False).items(): if not utxos: continue errmsg += ("\nmixdepth {}:\n{}".format( - md, fmt_utxos(utxos, wallet, prefix=' '))) + md, fmt_utxos(utxos, wallet_service, prefix=' '))) return (errmsgheader, errmsg) diff --git a/jmclient/jmclient/storage.py b/jmclient/jmclient/storage.py index 54d4b43..32fc387 100644 --- a/jmclient/jmclient/storage.py +++ b/jmclient/jmclient/storage.py @@ -217,6 +217,9 @@ class Storage(object): with open(self.path, 'rb') as fh: return fh.read() + def get_location(self): + return self.path + @staticmethod def _serialize(data): return bencoder.bencode(data) @@ -334,3 +337,6 @@ class VolatileStorage(Storage): def _read_file(self): return self.file_data + + def get_location(self): + return None diff --git a/jmclient/jmclient/taker.py b/jmclient/jmclient/taker.py index 956105d..3ebe421 100644 --- a/jmclient/jmclient/taker.py +++ b/jmclient/jmclient/taker.py @@ -7,17 +7,18 @@ from future.utils import iteritems import base64 import pprint import random -from twisted.internet import reactor +from twisted.internet import reactor, task from binascii import hexlify, unhexlify from jmbitcoin import SerializationError, SerializationTruncationError import jmbitcoin as btc -from jmclient.configure import get_p2sh_vbyte, jm_single, validate_address +from jmclient.configure import jm_single, validate_address from jmbase.support import get_log from jmclient.support import (calc_cj_fee, weighted_order_choose, choose_orders, choose_sweep_orders) -from jmclient.wallet import estimate_tx_fee, make_shuffled_tx +from jmclient.wallet import estimate_tx_fee from jmclient.podle import generate_podle, get_podle_commitments, PoDLE +from jmclient.wallet_service import WalletService from .output import generate_podle_error_string from .cryptoengine import EngineError @@ -31,7 +32,7 @@ class JMTakerError(Exception): class Taker(object): def __init__(self, - wallet, + wallet_service, schedule, order_chooser=weighted_order_choose, callbacks=None, @@ -80,7 +81,8 @@ class Taker(object): None """ self.aborted = False - self.wallet = wallet + assert isinstance(wallet_service, WalletService) + self.wallet_service = wallet_service self.schedule = schedule self.order_chooser = order_chooser self.max_cj_fee = max_cj_fee @@ -173,7 +175,7 @@ class Taker(object): #mixdepth in tumble schedules: if self.schedule_index == 0 or si[0] != self.schedule[ self.schedule_index - 1]: - self.mixdepthbal = self.wallet.get_balance_by_mixdepth( + self.mixdepthbal = self.wallet_service.get_balance_by_mixdepth( )[self.mixdepth] #reset to satoshis self.cjamount = int(self.cjamount * self.mixdepthbal) @@ -183,15 +185,20 @@ class Taker(object): self.cjamount = jm_single().mincjamount self.n_counterparties = si[2] self.my_cj_addr = si[3] + # for sweeps to external addresses we need an in-wallet import + # for the transaction monitor (this will be a no-op for txs to + # in-wallet addresses). + if self.cjamount == 0: + self.wallet_service.import_non_wallet_address(self.my_cj_addr) + #if destination is flagged "INTERNAL", choose a destination #from the next mixdepth modulo the maxmixdepth if self.my_cj_addr == "INTERNAL": next_mixdepth = (self.mixdepth + 1) % ( - self.wallet.mixdepth + 1) + self.wallet_service.mixdepth + 1) jlog.info("Choosing a destination from mixdepth: " + str( next_mixdepth)) - self.my_cj_addr = self.wallet.get_internal_addr(next_mixdepth, - bci=jm_single().bc_interface) + self.my_cj_addr = self.wallet_service.get_internal_addr(next_mixdepth) jlog.info("Chose destination address: " + self.my_cj_addr) self.outputs = [] self.cjfee_total = 0 @@ -277,8 +284,7 @@ class Taker(object): self.my_change_addr = None if self.cjamount != 0: try: - self.my_change_addr = self.wallet.get_internal_addr(self.mixdepth, - bci=jm_single().bc_interface) + self.my_change_addr = self.wallet_service.get_internal_addr(self.mixdepth) except: self.taker_info_callback("ABORT", "Failed to get a change address") return False @@ -291,15 +297,15 @@ class Taker(object): total_amount = self.cjamount + self.total_cj_fee + self.total_txfee jlog.info('total estimated amount spent = ' + str(total_amount)) try: - self.input_utxos = self.wallet.select_utxos(self.mixdepth, - total_amount) + self.input_utxos = self.wallet_service.select_utxos(self.mixdepth, total_amount, + minconfs=1) except Exception as e: self.taker_info_callback("ABORT", "Unable to select sufficient coins: " + repr(e)) return False else: #sweep - self.input_utxos = self.wallet.get_utxos_by_mixdepth()[self.mixdepth] + self.input_utxos = self.wallet_service.get_utxos_by_mixdepth()[self.mixdepth] #do our best to estimate the fee based on the number of #our own utxos; this estimate may be significantly higher #than the default set in option.txfee * makercount, where @@ -310,7 +316,7 @@ class Taker(object): est_outs = 2*self.n_counterparties + 1 jlog.debug("Estimated outs: "+str(est_outs)) estimated_fee = estimate_tx_fee(est_ins, est_outs, - txtype=self.wallet.get_txtype()) + txtype=self.wallet_service.get_txtype()) jlog.debug("We have a fee estimate: "+str(estimated_fee)) jlog.debug("And a requested fee of: "+str( self.txfee_default * self.n_counterparties)) @@ -387,7 +393,7 @@ class Taker(object): auth_pub_bin = unhexlify(auth_pub) for inp in utxo_data: try: - if self.wallet.pubkey_has_script( + if self.wallet_service.pubkey_has_script( auth_pub_bin, unhexlify(inp['script'])): break except EngineError: @@ -454,7 +460,7 @@ class Taker(object): #Estimate fee per choice of next/3/6 blocks targetting. estimated_fee = estimate_tx_fee( len(sum(self.utxos.values(), [])), len(self.outputs) + 2, - txtype=self.wallet.get_txtype()) + txtype=self.wallet_service.get_txtype()) jlog.info("Based on initial guess: " + str(self.total_txfee) + ", we estimated a miner fee of: " + str(estimated_fee)) #reset total @@ -495,7 +501,7 @@ class Taker(object): for u in sum(self.utxos.values(), [])] self.outputs.append({'address': self.coinjoin_address(), 'value': self.cjamount}) - tx = make_shuffled_tx(self.utxo_tx, self.outputs, False) + tx = btc.make_shuffled_tx(self.utxo_tx, self.outputs, False) jlog.info('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) self.latest_tx = btc.deserialize(tx) @@ -687,7 +693,7 @@ class Taker(object): new_utxos_dict = {k: v for k, v in utxos.items() if k in new_utxos} for k, v in iteritems(new_utxos_dict): addr = v['address'] - priv = self.wallet.get_key_from_addr(addr) + priv = self.wallet_service.get_key_from_addr(addr) if priv: #can be null from create-unsigned priv_utxo_pairs.append((priv, k)) return priv_utxo_pairs, too_old, too_small @@ -714,9 +720,9 @@ class Taker(object): #in the transaction, about to be consumed, rather than use #random utxos that will persist after. At this step we also #allow use of external utxos in the json file. - if any(self.wallet.get_utxos_by_mixdepth_().values()): + if any(self.wallet_service.get_utxos_by_mixdepth(hexfmt=False).values()): utxos = {} - for mdutxo in self.wallet.get_utxos_by_mixdepth().values(): + for mdutxo in self.wallet_service.get_utxos_by_mixdepth().values(): utxos.update(mdutxo) priv_utxo_pairs, to, ts = priv_utxo_pairs_from_utxos( utxos, age, amt) @@ -740,7 +746,7 @@ class Taker(object): "Commitment sourced OK") else: errmsgheader, errmsg = generate_podle_error_string(priv_utxo_pairs, - to, ts, self.wallet, self.cjamount, + to, ts, self.wallet_service, self.cjamount, jm_single().config.get("POLICY", "taker_utxo_age"), jm_single().config.get("POLICY", "taker_utxo_amtpercent")) @@ -766,10 +772,10 @@ class Taker(object): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) if utxo not in self.input_utxos.keys(): continue - script = self.wallet.addr_to_script(self.input_utxos[utxo]['address']) + script = self.wallet_service.addr_to_script(self.input_utxos[utxo]['address']) amount = self.input_utxos[utxo]['value'] our_inputs[index] = (script, amount) - self.latest_tx = self.wallet.sign_tx(self.latest_tx, our_inputs) + self.latest_tx = self.wallet_service.sign_tx(self.latest_tx, our_inputs) def push(self): tx = btc.serialize(self.latest_tx) @@ -783,13 +789,21 @@ class Taker(object): notify_addr = btc.address_to_script(self.my_cj_addr) else: notify_addr = self.my_cj_addr - #add the txnotify callbacks *before* pushing in case the - #walletnotify is triggered before the notify callbacks are set up; + #add the callbacks *before* pushing to ensure triggering; #this does leave a dangling notify callback if the push fails, but #that doesn't cause problems. - jm_single().bc_interface.add_tx_notify(self.latest_tx, - self.unconfirm_callback, self.confirm_callback, - notify_addr, vb=get_p2sh_vbyte()) + self.wallet_service.register_callbacks([self.unconfirm_callback], self.txid, + "unconfirmed") + self.wallet_service.register_callbacks([self.confirm_callback], self.txid, + "confirmed") + task.deferLater(reactor, + float(jm_single().config.getint( + "TIMEOUT", "unconfirm_timeout_sec")), + self.wallet_service.check_callback_called, + self.txid, self.unconfirm_callback, + "unconfirmed", + "transaction with txid: " + str(self.txid) + " not broadcast.") + tx_broadcast = jm_single().config.get('POLICY', 'tx_broadcast') nick_to_use = None if tx_broadcast == 'self': @@ -820,22 +834,44 @@ class Taker(object): self.self_sign() return self.push() + def tx_match(self, txd): + # Takers process only in series, so this should not occur: + assert self.latest_tx is not None + # check if the transaction matches our created tx: + if txd['outs'] != self.latest_tx['outs']: + return False + return True + def unconfirm_callback(self, txd, txid): + if not self.tx_match(txd): + return False jlog.info("Transaction seen on network, waiting for confirmation") #To allow client to mark transaction as "done" (e.g. by persisting state) self.on_finished_callback(True, fromtx="unconfirmed") self.waiting_for_conf = True + confirm_timeout_sec = float(jm_single().config.get( + "TIMEOUT", "confirm_timeout_hours")) * 3600 + task.deferLater(reactor, confirm_timeout_sec, + self.wallet_service.check_callback_called, + txid, self.confirm_callback, "confirmed", + "transaction with txid " + str(txid) + " not confirmed.") + return True def confirm_callback(self, txd, txid, confirmations): + if not self.tx_match(txd): + return False self.waiting_for_conf = False if self.aborted: - #do not trigger on_finished processing (abort whole schedule) - return + #do not trigger on_finished processing (abort whole schedule), + # but we still return True as we have finished our listening + # for this tx: + return True jlog.debug("Confirmed callback in taker, confs: " + str(confirmations)) fromtx=False if self.schedule_index + 1 == len(self.schedule) else True waittime = self.schedule[self.schedule_index][4] self.on_finished_callback(True, fromtx=fromtx, waittime=waittime, txdetails=(txd, txid)) + return True class P2EPTaker(Taker): """ The P2EP Taker will initialize its protocol directly @@ -845,8 +881,8 @@ class P2EPTaker(Taker): improves the privacy of the operation. """ - def __init__(self, counterparty, wallet, schedule, callbacks): - super(P2EPTaker, self).__init__(wallet, schedule, callbacks=callbacks) + def __init__(self, counterparty, wallet_service, schedule, callbacks): + super(P2EPTaker, self).__init__(wallet_service, schedule, callbacks=callbacks) self.p2ep_receiver_nick = counterparty # Callback to request user permission (for e.g. GUI) # args: (1) message, as string @@ -951,7 +987,7 @@ class P2EPTaker(Taker): # estimate the fee for the version of the transaction which is # not coinjoined: est_fee = estimate_tx_fee(len(self.input_utxos), 2, - txtype=self.wallet.get_txtype()) + txtype=self.wallet_service.get_txtype()) my_change_value = my_total_in - self.cjamount - est_fee if my_change_value <= 0: # as discussed in initialize(), this should be an extreme edge case. @@ -973,7 +1009,7 @@ class P2EPTaker(Taker): "getblockchaininfo", [])["blocks"] # As for JM coinjoins, the `None` key is used for our own inputs # to the transaction; this preparatory version contains only those. - tx = make_shuffled_tx(self.utxos[None], self.outputs, + tx = btc.make_shuffled_tx(self.utxos[None], self.outputs, False, 2, currentblock) jlog.info('Created proposed fallback tx\n' + pprint.pformat( btc.deserialize(tx))) @@ -984,10 +1020,10 @@ class P2EPTaker(Taker): dtx = btc.deserialize(tx) for index, ins in enumerate(dtx['ins']): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - script = self.wallet.addr_to_script(self.input_utxos[utxo]['address']) + script = self.wallet_service.addr_to_script(self.input_utxos[utxo]['address']) amount = self.input_utxos[utxo]['value'] our_inputs[index] = (script, amount) - self.signed_noncj_tx = btc.serialize(self.wallet.sign_tx(dtx, our_inputs)) + self.signed_noncj_tx = btc.serialize(self.wallet_service.sign_tx(dtx, our_inputs)) self.taker_info_callback("INFO", "Built tx proposal, sending to receiver.") return (True, [self.p2ep_receiver_nick], self.signed_noncj_tx) @@ -1153,7 +1189,7 @@ class P2EPTaker(Taker): # in joinmarket.cfg; allow either (a) automatic agreement for any value within # a range of 0.3 to 3x this figure, or (b) user to agree on prompt. fee_est = estimate_tx_fee(len(tx['ins']), len(tx['outs']), - txtype=self.wallet.get_txtype()) + txtype=self.wallet_service.get_txtype()) fee_ok = False if btc_fee > 0.3 * fee_est and btc_fee < 3 * fee_est: fee_ok = True diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index b75eb85..2592668 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -20,7 +20,7 @@ Utility functions for tumbler-style takers; Currently re-used by CLI script tumbler.py and joinmarket-qt """ -def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False, +def direct_send(wallet_service, amount, mixdepth, destaddr, answeryes=False, accept_callback=None, info_callback=None): """Send coins directly from one mixdepth to one destination address; does not need IRC. Sweep as for normal sendpayment (set amount=0). @@ -46,12 +46,12 @@ def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False, assert mixdepth >= 0 assert isinstance(amount, numbers.Integral) assert amount >=0 - assert isinstance(wallet, BaseWallet) + assert isinstance(wallet_service.wallet, BaseWallet) from pprint import pformat - txtype = wallet.get_txtype() + txtype = wallet_service.get_txtype() if amount == 0: - utxos = wallet.get_utxos_by_mixdepth()[mixdepth] + utxos = wallet_service.get_utxos_by_mixdepth()[mixdepth] if utxos == {}: log.error( "There are no utxos in mixdepth: " + str(mixdepth) + ", quitting.") @@ -62,7 +62,7 @@ def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False, else: #8 inputs to be conservative initial_fee_est = estimate_tx_fee(8,2, txtype=txtype) - utxos = wallet.select_utxos(mixdepth, amount + initial_fee_est) + utxos = wallet_service.select_utxos(mixdepth, amount + initial_fee_est) if len(utxos) < 8: fee_est = estimate_tx_fee(len(utxos), 2, txtype=txtype) else: @@ -70,15 +70,14 @@ def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False, total_inputs_val = sum([va['value'] for u, va in iteritems(utxos)]) changeval = total_inputs_val - fee_est - amount outs = [{"value": amount, "address": destaddr}] - change_addr = wallet.get_internal_addr(mixdepth, - jm_single().bc_interface) + change_addr = wallet_service.get_internal_addr(mixdepth) outs.append({"value": changeval, "address": change_addr}) #Now ready to construct transaction log.info("Using a fee of : " + str(fee_est) + " satoshis.") if amount != 0: log.info("Using a change value of: " + str(changeval) + " satoshis.") - txsigned = sign_tx(wallet, mktx(list(utxos.keys()), outs), utxos) + txsigned = sign_tx(wallet_service, mktx(list(utxos.keys()), outs), utxos) log.info("Got signed transaction:\n") log.info(pformat(txsigned)) tx = serialize(txsigned) @@ -104,15 +103,15 @@ def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False, return txid -def sign_tx(wallet, tx, utxos): +def sign_tx(wallet_service, tx, utxos): stx = deserialize(tx) our_inputs = {} for index, ins in enumerate(stx['ins']): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - script = wallet.addr_to_script(utxos[utxo]['address']) + script = wallet_service.addr_to_script(utxos[utxo]['address']) amount = utxos[utxo]['value'] our_inputs[index] = (script, amount) - return wallet.sign_tx(stx, our_inputs) + return wallet_service.sign_tx(stx, our_inputs) def get_tumble_log(logsdir): tumble_log = logging.getLogger('tumbler') @@ -178,7 +177,7 @@ def unconf_update(taker, schedulefile, tumble_log, addtolog=False): #because addresses are not public until broadcast (whereas for makers, #they are public *during* negotiation). So updating the cache here #is sufficient - taker.wallet.update_cache_index() + taker.wallet_service.save_wallet() #If honest-only was set, and we are going to continue (e.g. Tumbler), #we switch off the honest-only filter. We also wipe the honest maker @@ -255,9 +254,6 @@ def tumbler_taker_finished_update(taker, schedulefile, tumble_log, options, waiting_message = "Waiting for: " + str(waittime) + " minutes." tumble_log.info(waiting_message) log.info(waiting_message) - txd, txid = txdetails - taker.wallet.remove_old_utxos(txd) - taker.wallet.add_new_utxos(txd, txid) else: #a transaction failed, either because insufficient makers #(acording to minimum_makers) responded in Phase 1, or not all diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index d6dddbc..9dd201a 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -4,7 +4,6 @@ from builtins import * # noqa: F401 from configparser import NoOptionError import warnings -import random import functools import collections import numbers @@ -19,6 +18,7 @@ from numbers import Integral from .configure import jm_single +from .blockchaininterface import INF_HEIGHT from .support import select_gradual, select_greedy, select_greediest, \ select from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH,\ @@ -26,6 +26,7 @@ from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH,\ from .support import get_random_bytes from . import mn_encode, mn_decode import jmbitcoin as btc +from jmbase import JM_WALLET_NAME_PREFIX """ @@ -67,19 +68,6 @@ class Mnemonic(MnemonicParent): def detect_language(cls, code): return "english" -def make_shuffled_tx(ins, outs, deser=True, version=1, locktime=0): - """ Simple utility to ensure transaction - inputs and outputs are randomly ordered. - Can possibly be replaced by BIP69 in future - """ - random.shuffle(ins) - random.shuffle(outs) - tx = btc.mktx(ins, outs, version=version, locktime=locktime) - if deser: - return btc.deserialize(tx) - else: - return tx - def estimate_tx_fee(ins, outs, txtype='p2pkh'): '''Returns an estimate of the number of satoshis required for a transaction with the given number of inputs and outputs, @@ -123,7 +111,7 @@ class UTXOManager(object): def __init__(self, storage, merge_func): self.storage = storage self.selector = merge_func - # {mixdexpth: {(txid, index): (path, value)}} + # {mixdexpth: {(txid, index): (path, value, height)}} self._utxo = None # metadata kept as a separate key in the database # for backwards compat; value as dict for forward-compat. @@ -197,16 +185,20 @@ class UTXOManager(object): return self._utxo[mixdepth].pop((txid, index)) - def add_utxo(self, txid, index, path, value, mixdepth): + def add_utxo(self, txid, index, path, value, mixdepth, height=None): # Assumed: that we add a utxo only if we want it enabled, # so metadata is not currently added. + # The height (blockheight) field will be "infinity" for unconfirmed. assert isinstance(txid, bytes) assert len(txid) == self.TXID_LEN assert isinstance(index, numbers.Integral) assert isinstance(value, numbers.Integral) assert isinstance(mixdepth, numbers.Integral) + if height is None: + height = INF_HEIGHT + assert isinstance(height, numbers.Integral) - self._utxo[mixdepth][(txid, index)] = (path, value) + self._utxo[mixdepth][(txid, index)] = (path, value, height) def is_disabled(self, txid, index): if not self._utxo_meta: @@ -231,28 +223,37 @@ class UTXOManager(object): def enable_utxo(self, txid, index): self.disable_utxo(txid, index, disable=False) - def select_utxos(self, mixdepth, amount, utxo_filter=(), select_fn=None): + def select_utxos(self, mixdepth, amount, utxo_filter=(), select_fn=None, + maxheight=None): assert isinstance(mixdepth, numbers.Integral) utxos = self._utxo[mixdepth] # do not select anything in the filter available = [{'utxo': utxo, 'value': val} - for utxo, (addr, val) in utxos.items() if utxo not in utxo_filter] + for utxo, (addr, val, height) in utxos.items() if utxo not in utxo_filter] + # do not select anything with insufficient confirmations: + if maxheight is not None: + available = [{'utxo': utxo, 'value': val} + for utxo, (addr, val, height) in utxos.items( + ) if height <= maxheight] # do not select anything disabled available = [u for u in available if not self.is_disabled(*u['utxo'])] selector = select_fn or self.selector selected = selector(available, amount) + # note that we do not return height; for selection, we expect + # the caller will not want this (after applying the height filter) return {s['utxo']: {'path': utxos[s['utxo']][0], 'value': utxos[s['utxo']][1]} for s in selected} def get_balance_by_mixdepth(self, max_mixdepth=float('Inf'), - include_disabled=True): + include_disabled=True, maxheight=None): """ By default this returns a dict of aggregated bitcoin balance per mixdepth: {0: N sats, 1: M sats, ...} for all currently available mixdepths. If max_mixdepth is set it will return balances only up to that mixdepth. To get only enabled balance, set include_disabled=False. + To get balances only with a certain number of confs, use maxheight. """ balance_dict = collections.defaultdict(int) for mixdepth, utxomap in self._utxo.items(): @@ -261,6 +262,9 @@ class UTXOManager(object): if not include_disabled: utxomap = {k: v for k, v in utxomap.items( ) if not self.is_disabled(*k)} + if maxheight is not None: + utxomap = {k: v for k, v in utxomap.items( + ) if v[2] <= maxheight} value = sum(x[1] for x in utxomap.values()) balance_dict[mixdepth] = value return balance_dict @@ -345,6 +349,14 @@ class BaseWallet(object): self.network = self._storage.data[b'network'].decode('ascii') self._utxos = UTXOManager(self._storage, self.merge_algorithm) + def get_storage_location(self): + """ Return the location of the + persistent storage, if it exists, or None. + """ + if not self._storage: + return None + return self._storage.get_location() + def save(self): """ Write data to associated storage object and trigger persistent update. @@ -426,36 +438,25 @@ class BaseWallet(object): privkey = self._get_priv_from_path(path)[0] return hexlify(privkey).decode('ascii') - def _get_addr_int_ext(self, get_script_func, mixdepth, bci=None): - script = get_script_func(mixdepth) - addr = self.script_to_addr(script) - if bci is not None and hasattr(bci, 'import_addresses'): - assert hasattr(bci, 'get_wallet_name') - bci.import_addresses([addr], bci.get_wallet_name(self)) - return addr + def _get_addr_int_ext(self, internal, mixdepth): + script = self.get_internal_script(mixdepth) if internal else \ + self.get_external_script(mixdepth) + return self.script_to_addr(script) - def get_external_addr(self, mixdepth, bci=None): + def get_external_addr(self, mixdepth): """ Return an address suitable for external distribution, including funding the wallet from other sources, or receiving payments or donations. JoinMarket will never generate these addresses for internal use. - If the argument bci is non-null, we attempt to import the new - address into this blockchaininterface instance - (based on Bitcoin Core's model). """ - return self._get_addr_int_ext(self.get_external_script, mixdepth, - bci=bci) + return self._get_addr_int_ext(False, mixdepth) - def get_internal_addr(self, mixdepth, bci=None): + def get_internal_addr(self, mixdepth): """ Return an address for internal usage, as change addresses and when participating in transactions initiated by other parties. - If the argument bci is non-null, we attempt to import the new - address into this blockchaininterface instance - (based on Bitcoin Core's model). """ - return self._get_addr_int_ext(self.get_internal_script, mixdepth, - bci=bci) + return self._get_addr_int_ext(True, mixdepth) def get_external_script(self, mixdepth): return self.get_new_script(mixdepth, False) @@ -575,7 +576,7 @@ class BaseWallet(object): args: tx: transaction dict returns: - {(txid, index): {'script': bytes, 'value': int} for all removed utxos + {(txid, index): {'script': bytes, 'path': str, 'value': int} for all removed utxos """ removed_utxos = {} for inp in tx['ins']: @@ -583,7 +584,7 @@ class BaseWallet(object): md = self._utxos.have_utxo(txid, index) if md is False: continue - path, value = self._utxos.remove_utxo(txid, index, md) + path, value, height = self._utxos.remove_utxo(txid, index, md) script = self.get_script_path(path) removed_utxos[(txid, index)] = {'script': script, 'path': path, @@ -591,12 +592,12 @@ class BaseWallet(object): return removed_utxos @deprecated - def add_new_utxos(self, tx, txid): + def add_new_utxos(self, tx, txid, height=None): tx = deepcopy(tx) for out in tx['outs']: out['script'] = unhexlify(out['script']) - ret = self.add_new_utxos_(tx, unhexlify(txid)) + ret = self.add_new_utxos_(tx, unhexlify(txid), height=height) added_utxos = {} for (txid_bin, index), val in ret.items(): @@ -605,12 +606,14 @@ class BaseWallet(object): added_utxos[txid + ':' + str(index)] = val return added_utxos - def add_new_utxos_(self, tx, txid): + def add_new_utxos_(self, tx, txid, height=None): """ Add all outputs of tx for this wallet to internal utxo list. args: tx: transaction dict + height: blockheight in which tx was included, or None + if unconfirmed. returns: {(txid, index): {'script': bytes, 'path': tuple, 'value': int} for all added utxos @@ -619,7 +622,8 @@ class BaseWallet(object): added_utxos = {} for index, outs in enumerate(tx['outs']): try: - self.add_utxo(txid, index, outs['script'], outs['value']) + self.add_utxo(txid, index, outs['script'], outs['value'], + height=height) except WalletError: continue @@ -629,7 +633,7 @@ class BaseWallet(object): 'value': outs['value']} return added_utxos - def add_utxo(self, txid, index, script, value): + def add_utxo(self, txid, index, script, value, height=None): assert isinstance(txid, bytes) assert isinstance(index, Integral) assert isinstance(script, bytes) @@ -640,15 +644,30 @@ class BaseWallet(object): path = self.script_to_path(script) mixdepth = self._get_mixdepth_from_path(path) - self._utxos.add_utxo(txid, index, path, value, mixdepth) + self._utxos.add_utxo(txid, index, path, value, mixdepth, height=height) + + def process_new_tx(self, txd, txid, height=None): + """ Given a newly seen transaction, deserialized as txd and + with transaction id, process its inputs and outputs and update + the utxo contents of this wallet accordingly. + NOTE: this should correctly handle transactions that are not + actually related to the wallet; it will not add (or remove, + obviously) utxos that were not related since the underlying + functions check this condition. + """ + removed_utxos = self.remove_old_utxos(txd) + added_utxos = self.add_new_utxos(txd, txid, height=height) + return (removed_utxos, added_utxos) @deprecated - def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None): + def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None, + maxheight=None): utxo_filter_new = None if utxo_filter: utxo_filter_new = [(unhexlify(utxo[:64]), int(utxo[65:])) for utxo in utxo_filter] - ret = self.select_utxos_(mixdepth, amount, utxo_filter_new, select_fn) + ret = self.select_utxos_(mixdepth, amount, utxo_filter_new, select_fn, + maxheight=maxheight) ret_conv = {} for utxo, data in ret.items(): addr = self.get_addr_path(data['path']) @@ -657,7 +676,7 @@ class BaseWallet(object): return ret_conv def select_utxos_(self, mixdepth, amount, utxo_filter=None, - select_fn=None): + select_fn=None, maxheight=None): """ Select a subset of available UTXOS for a given mixdepth whose value is greater or equal to amount. @@ -667,6 +686,7 @@ class BaseWallet(object): equal to wallet.max_mixdepth amount: int, total minimum amount of all selected utxos utxo_filter: list of (txid, index), utxos not to select + maxheight: only select utxos with blockheight <= this. returns: {(txid, index): {'script': bytes, 'path': tuple, 'value': int}} @@ -681,7 +701,7 @@ class BaseWallet(object): assert isinstance(i[0], bytes) assert isinstance(i[1], numbers.Integral) ret = self._utxos.select_utxos( - mixdepth, amount, utxo_filter, select_fn) + mixdepth, amount, utxo_filter, select_fn, maxheight=maxheight) for data in ret.values(): data['script'] = self.get_script_path(data['path']) @@ -701,20 +721,24 @@ class BaseWallet(object): self._utxos.reset() def get_balance_by_mixdepth(self, verbose=True, - include_disabled=False): + include_disabled=False, + maxheight=None): """ Get available funds in each active mixdepth. By default ignores disabled utxos in calculation. + By default returns unconfirmed transactions, to filter + confirmations, set maxheight to max acceptable blockheight. returns: {mixdepth: value} """ # TODO: verbose return self._utxos.get_balance_by_mixdepth(max_mixdepth=self.mixdepth, - include_disabled=include_disabled) + include_disabled=include_disabled, + maxheight=maxheight) @deprecated - def get_utxos_by_mixdepth(self, verbose=True): + def get_utxos_by_mixdepth(self, verbose=True, includeheight=False): # TODO: verbose - ret = self.get_utxos_by_mixdepth_() + ret = self.get_utxos_by_mixdepth_(includeheight=includeheight) utxos_conv = collections.defaultdict(dict) for md, utxos in ret.items(): @@ -725,13 +749,14 @@ class BaseWallet(object): utxos_conv[md][utxo_str] = data return utxos_conv - def get_utxos_by_mixdepth_(self, include_disabled=False): + def get_utxos_by_mixdepth_(self, include_disabled=False, includeheight=False): """ Get all UTXOs for active mixdepths. returns: {mixdepth: {(txid, index): {'script': bytes, 'path': tuple, 'value': int}}} + (if `includeheight` is True, adds key 'height': int) """ mix_utxos = self._utxos.get_utxos_by_mixdepth() @@ -739,13 +764,15 @@ class BaseWallet(object): for md, data in mix_utxos.items(): if md > self.mixdepth: continue - for utxo, (path, value) in data.items(): + for utxo, (path, value, height) in data.items(): if not include_disabled and self._utxos.is_disabled(*utxo): continue script = self.get_script_path(path) script_utxos[md][utxo] = {'script': script, 'path': path, 'value': value} + if includeheight: + script_utxos[md][utxo]['height'] = height return script_utxos @classmethod @@ -841,6 +868,12 @@ class BaseWallet(object): priv, engine = self._get_priv_from_path(path) return engine.sign_message(priv, message) + def get_wallet_name(self): + """ Returns the name used as a label for this + specific Joinmarket wallet in Bitcoin Core. + """ + return JM_WALLET_NAME_PREFIX + self.get_wallet_id() + def get_wallet_id(self): """ Get a human-readable identifier for the wallet. @@ -926,6 +959,46 @@ class BaseWallet(object): """ raise NotImplementedError() + def rewind_wallet_indices(self, used_indices, saved_indices): + for md in used_indices: + for int_type in (0, 1): + index = max(used_indices[md][int_type], + saved_indices[md][int_type]) + self.set_next_index(md, int_type, index, force=True) + + def get_used_indices(self, addr_gen): + """ Returns a dict of max used indices for each branch in + the wallet, from the given addresses addr_gen, assuming + that they are known to the wallet. + """ + + indices = {x: [0, 0] for x in range(self.max_mixdepth + 1)} + + for addr in addr_gen: + if not self.is_known_addr(addr): + continue + md, internal, index = self.get_details( + self.addr_to_path(addr)) + if internal not in (0, 1): + assert internal == 'imported' + continue + indices[md][internal] = max(indices[md][internal], index + 1) + + return indices + + def check_gap_indices(self, used_indices): + """ Return False if any of the provided indices (which should be + those seen from listtransactions as having been used, for + this wallet/label) are higher than the ones recorded in the index + cache.""" + + for md in used_indices: + for internal in (0, 1): + if used_indices[md][internal] >\ + max(self.get_next_unused_index(md, internal), 0): + return False + return True + def close(self): self._storage.close() diff --git a/jmclient/jmclient/wallet_service.py b/jmclient/jmclient/wallet_service.py new file mode 100644 index 0000000..836616d --- /dev/null +++ b/jmclient/jmclient/wallet_service.py @@ -0,0 +1,661 @@ +#! /usr/bin/env python +from __future__ import (absolute_import, division, + print_function, unicode_literals) + +import collections +import time +import ast +import binascii +from decimal import Decimal +from copy import deepcopy +from twisted.internet import reactor +from twisted.internet import task +from twisted.application.service import Service +from numbers import Integral +from jmclient.configure import jm_single, get_log +from jmclient.output import fmt_tx_data +from jmclient.jsonrpc import JsonRpcError +from jmclient.blockchaininterface import INF_HEIGHT +"""Wallet service + +The purpose of this independent service is to allow +running applications to keep an up to date, asynchronous +view of the current state of its wallet, deferring any +polling mechanisms needed against the backend blockchain +interface here. +""" + +jlog = get_log() + +class WalletService(Service): + EXTERNAL_WALLET_LABEL = "joinmarket-notify" + + def __init__(self, wallet): + # The two principal member variables + # are the blockchaininterface instance, + # which is currently global in JM but + # could be more flexible in future, and + # the JM wallet object. + self.bci = jm_single().bc_interface + # keep track of the quasi-real-time blockheight + # (updated in main monitor loop) + self.update_blockheight() + self.wallet = wallet + self.synced = False + + # Dicts of registered callbacks, by type + # and then by txinfo, for events + # on transactions. + self.callbacks = {} + self.callbacks["all"] = [] + self.callbacks["unconfirmed"] = {} + self.callbacks["confirmed"] = {} + + self.restart_callback = None + + # transactions we are actively monitoring, + # i.e. they are not new but we want to track: + self.active_txids = [] + # to ensure transactions are only logged once: + self.logged_txids = [] + + def update_blockheight(self): + """ Can be called manually (on startup, or for tests) + but will be called as part of main monitoring + loop to ensure new transactions are added at + the right height. + """ + try: + self.current_blockheight = self.bci.rpc("getblockcount", []) + assert isinstance(self.current_blockheight, Integral) + except Exception as e: + jlog.error("Failure to get blockheight from Bitcoin Core:") + jlog.error(repr(e)) + return + + def startService(self): + """ Encapsulates start up actions. + Here wallet sync. + """ + super(WalletService, self).startService() + self.request_sync_wallet() + + def stopService(self): + """ Encapsulates shut down actions. + Here shut down main tx monitoring loop. + """ + self.monitor_loop.stop() + super(WalletService, self).stopService() + + def isRunning(self): + if self.running == 1: + return True + return False + + def add_restart_callback(self, callback): + """ Sets the function that will be + called in the event that the wallet + sync completes with a restart warning. + The only argument is a message string, + which the calling function displays to + the user before quitting gracefully. + """ + self.restart_callback = callback + + def request_sync_wallet(self): + """ Ensures wallet sync is complete + before the main event loop starts. + """ + d = task.deferLater(reactor, 0.0, self.sync_wallet) + d.addCallback(self.start_wallet_monitoring) + + def register_callbacks(self, callbacks, txinfo, cb_type="all"): + """ Register callbacks that will be called by the + transaction monitor loop, on transactions stored under + our wallet label (see WalletService.get_wallet_name()). + Callback arguments are currently (txd, txid) and return + is boolean, except "confirmed" callbacks which have + arguments (txd, txid, confirmations). + Note that callbacks MUST correctly return True if they + recognized the transaction and processed it, and False + if not. The True return value will be used to remove + the callback from the list. + Arguments: + `callbacks` - a list of functions with signature as above + and return type boolean. + `txinfo` - either a txid expected for the transaction, if + known, or a tuple of the ordered output set, of the form + (('script': script), ('value': value), ..). This can be + constructed from jmbitcoin.deserialize output, key "outs", + using tuple(). See WalletService.transaction_monitor(). + `cb_type` - must be one of "all", "unconfirmed", "confirmed"; + the first type will be called back once for every new + transaction, the second only once when the number of + confirmations is 0, and the third only once when the number + of confirmations is > 0. + """ + if cb_type == "all": + # note that in this case, txid is ignored. + self.callbacks["all"].extend(callbacks) + elif cb_type in ["unconfirmed", "confirmed"]: + if txinfo not in self.callbacks[cb_type]: + self.callbacks[cb_type][txinfo] = [] + self.callbacks[cb_type][txinfo].extend(callbacks) + else: + assert False, "Invalid argument: " + cb_type + + + def start_wallet_monitoring(self, syncresult): + """ Once the initialization of the service + (currently, means: wallet sync) is complete, + we start the main monitoring jobs of the + wallet service (currently, means: monitoring + all new transactions on the blockchain that + are recognised as belonging to the Bitcoin + Core wallet with the JM wallet's label). + """ + if not syncresult: + jlog.error("Failed to sync the bitcoin wallet. Shutting down.") + reactor.stop() + return + jlog.info("Starting transaction monitor in walletservice") + self.monitor_loop = task.LoopingCall( + self.transaction_monitor) + self.monitor_loop.start(5.0) + + def import_non_wallet_address(self, address): + """ Used for keeping track of transactions which + have no in-wallet destinations. External wallet + label is used to avoid breaking fast sync (which + assumes label => wallet) + """ + if not self.bci.is_address_imported(address): + self.bci.import_addresses([address], self.EXTERNAL_WALLET_LABEL, + restart_cb=self.restart_callback) + + def transaction_monitor(self): + """Keeps track of any changes in the wallet (new transactions). + Intended to be run as a twisted task.LoopingCall so that this + Service is constantly in near-realtime sync with the blockchain. + """ + + self.update_blockheight() + + txlist = self.bci.list_transactions(100) + new_txs = [] + for x in txlist: + # process either (a) a completely new tx or + # (b) a tx that reached unconf status but we are still + # waiting for conf (active_txids) + if x['txid'] in self.active_txids or x['txid'] not in self.old_txs: + new_txs.append(x) + # reset for next polling event: + self.old_txs = [x['txid'] for x in txlist] + + for tx in new_txs: + txid = tx["txid"] + res = self.bci.get_transaction(txid) + if not res: + continue + confs = res["confirmations"] + if not isinstance(confs, Integral): + jlog.warning("Malformed gettx result: " + str(res)) + continue + if confs < 0: + jlog.info( + "Transaction: " + txid + " has a conflict, abandoning.") + continue + if confs == 0: + height = None + else: + height = self.current_blockheight - confs + 1 + + txd = self.bci.get_deser_from_gettransaction(res) + if txd is None: + continue + removed_utxos, added_utxos = self.wallet.process_new_tx(txd, txid, height) + # TODO note that this log message will be missed if confirmation + # is absurdly fast, this is considered acceptable compared with + # additional complexity. + if txid not in self.logged_txids: + self.log_new_tx(removed_utxos, added_utxos, txid) + self.logged_txids.append(txid) + + # first fire 'all' type callbacks, irrespective of if the + # transaction pertains to anything known (but must + # have correct label per above); filter on this Joinmarket wallet label, + # or the external monitoring label: + if "label" in tx and tx["label"] in [ + self.EXTERNAL_WALLET_LABEL, self.get_wallet_name()]: + for f in self.callbacks["all"]: + # note we need no return value as we will never + # remove these from the list + f(txd, txid) + + # The tuple given as the second possible key for the dict + # is such because dict keys must be hashable types, so a simple + # replication of the entries in the list tx["outs"], where tx + # was generated via jmbitcoin.deserialize, is unacceptable to + # Python, since they are dicts. However their keyset is deterministic + # so it is sufficient to convert these dicts to tuples with fixed + # ordering, thus it can be used as a key into the self.callbacks + # dicts. (This is needed because txid is not always available + # at the time of callback registration). + possible_keys = [txid, tuple( + (x["script"], x["value"]) for x in txd["outs"])] + + # note that len(added_utxos) > 0 is not a sufficient condition for + # the tx being new, since wallet.add_new_utxos will happily re-add + # a utxo that already exists; but this does not cause re-firing + # of callbacks since we in these cases delete the callback after being + # called once. + # Note also that it's entirely possible that there are only removals, + # not additions, to the utxo set, specifically in sweeps to external + # addresses. In this case, since removal can by definition only + # happen once, we must allow entries in self.active_txids through the + # filter. + if len(added_utxos) > 0 or len(removed_utxos) > 0 \ + or txid in self.active_txids: + if confs == 0: + for k in possible_keys: + if k in self.callbacks["unconfirmed"]: + for f in self.callbacks["unconfirmed"][k]: + # True implies success, implies removal: + if f(txd, txid): + self.callbacks["unconfirmed"][k].remove(f) + # keep monitoring for conf > 0: + self.active_txids.append(txid) + elif confs > 0: + for k in possible_keys: + if k in self.callbacks["confirmed"]: + for f in self.callbacks["confirmed"][k]: + if f(txd, txid, confs): + self.callbacks["confirmed"][k].remove(f) + if txid in self.active_txids: + self.active_txids.remove(txid) + + def check_callback_called(self, txinfo, callback, cbtype, msg): + """ Intended to be a deferred Task to be scheduled some + set time after the callback was registered. "all" type + callbacks do not expire and are not included. + """ + assert cbtype in ["unconfirmed", "confirmed"] + if txinfo in self.callbacks[cbtype]: + if callback in self.callbacks[cbtype][txinfo]: + # the callback was not called, drop it and warn + self.callbacks[cbtype][txinfo].remove(callback) + # TODO - dangling txids in self.active_txids will + # be caused by this, but could also happen for + # other reasons; possibly add logic to ensure that + # this never occurs, although their presence should + # not cause a functional error. + jlog.info("Timed out: " + msg) + # if callback is not in the list, it was already + # processed and so do nothing. + + def log_new_tx(self, removed_utxos, added_utxos, txid): + """ Changes to the wallet are logged at INFO level by + the WalletService. + """ + def report_changed(x, utxos): + if len(utxos.keys()) > 0: + jlog.info(x + ' utxos=\n{}'.format('\n'.join( + '{} - {}'.format(u, fmt_tx_data(tx_data, self)) + for u, tx_data in utxos.items()))) + + report_changed("Removed", removed_utxos) + report_changed("Added", added_utxos) + + + """ Wallet syncing code + """ + + def sync_wallet(self, fast=True): + """ Syncs wallet; note that if slow sync + requires multiple rounds this must be called + until self.synced is True. + Before starting the event loop, we cache + the current most recent transactions as + reported by the blockchain interface, since + we are interested in deltas. + """ + # If this is called when syncing already complete: + if self.synced: + return True + if fast: + self.sync_wallet_fast() + else: + self.sync_addresses() + self.sync_unspent() + # Don't attempt updates on transactions that existed + # before startup + self.old_txs = [x['txid'] for x in self.bci.list_transactions(100)] + return self.synced + + def resync_wallet(self, fast=True): + """ The self.synced state is generally + updated to True, once, at the start of + a run of a particular program. Here we + can manually force re-sync. + """ + self.synced = False + self.sync_wallet(fast=fast) + + def sync_wallet_fast(self): + """Exploits the fact that given an index_cache, + all addresses necessary should be imported, so we + can just list all used addresses to find the right + index values. + """ + self.get_address_usages() + self.sync_unspent() + + def get_address_usages(self): + """Use rpc `listaddressgroupings` to locate all used + addresses in the account (whether spent or unspent outputs). + This will not result in a full sync if working with a new + Bitcoin Core instance, in which case "fast" should have been + specifically disabled by the user. + """ + wallet_name = self.get_wallet_name() + agd = self.bci.rpc('listaddressgroupings', []) + # flatten all groups into a single list; then, remove duplicates + fagd = (tuple(item) for sublist in agd for item in sublist) + # "deduplicated flattened address grouping data" = dfagd + dfagd = set(fagd) + used_addresses = set() + for addr_info in dfagd: + if len(addr_info) < 3 or addr_info[2] != wallet_name: + continue + used_addresses.add(addr_info[0]) + + # for a first run, import first chunk + if not used_addresses: + jlog.info("Detected new wallet, performing initial import") + # delegate inital address import to sync_addresses + # this should be fast because "getaddressesbyaccount" should return + # an empty list in this case + self.sync_addresses() + self.synced = True + return + + # Wallet has been used; scan forwards. + jlog.debug("Fast sync in progress. Got this many used addresses: " + str( + len(used_addresses))) + # Need to have wallet.index point to the last used address + # Algo: + # 1. Scan batch 1 of each branch, record matched wallet addresses. + # 2. Check if all addresses in 'used addresses' have been matched, if + # so, break. + # 3. Repeat the above for batch 2, 3.. up to max 20 batches. + # 4. If after all 20 batches not all used addresses were matched, + # quit with error. + # 5. Calculate used indices. + # 6. If all used addresses were matched, set wallet index to highest + # found index in each branch and mark wallet sync complete. + # Rationale for this algo: + # Retrieving addresses is a non-zero computational load, so batching + # and then caching allows a small sync to complete *reasonably* + # quickly while a larger one is not really negatively affected. + # The downside is another free variable, batch size, but this need + # not be exposed to the user; it is not the same as gap limit, in fact, + # the concept of gap limit does not apply to this kind of sync, which + # *assumes* that the most recent usage of addresses is indeed recorded. + remaining_used_addresses = used_addresses.copy() + addresses, saved_indices = self.collect_addresses_init() + for addr in addresses: + remaining_used_addresses.discard(addr) + + BATCH_SIZE = 100 + MAX_ITERATIONS = 20 + current_indices = deepcopy(saved_indices) + for j in range(MAX_ITERATIONS): + if not remaining_used_addresses: + break + for addr in \ + self.collect_addresses_gap(gap_limit=BATCH_SIZE): + remaining_used_addresses.discard(addr) + + # increase wallet indices for next iteration + for md in current_indices: + current_indices[md][0] += BATCH_SIZE + current_indices[md][1] += BATCH_SIZE + self.rewind_wallet_indices(current_indices, current_indices) + else: + self.rewind_wallet_indices(saved_indices, saved_indices) + raise Exception("Failed to sync in fast mode after 20 batches; " + "please re-try wallet sync with --recoversync flag.") + + # creating used_indices on-the-fly would be more efficient, but the + # overall performance gain is probably negligible + used_indices = self.get_used_indices(used_addresses) + self.rewind_wallet_indices(used_indices, saved_indices) + self.synced = True + + def sync_addresses(self): + """ Triggered by use of --recoversync option in scripts, + attempts a full scan of the blockchain without assuming + anything about past usages of addresses (does not use + wallet.index_cache as hint). + """ + jlog.debug("requesting detailed wallet history") + wallet_name = self.get_wallet_name() + addresses, saved_indices = self.collect_addresses_init() + try: + imported_addresses = set(self.bci.rpc('getaddressesbyaccount', + [wallet_name])) + except JsonRpcError: + if wallet_name in self.bci.rpc('listlabels', []): + imported_addresses = set(self.bci.rpc('getaddressesbylabel', + [wallet_name]).keys()) + else: + imported_addresses = set() + + if not addresses.issubset(imported_addresses): + self.bci.add_watchonly_addresses(addresses - imported_addresses, + wallet_name, self.restart_callback) + return + + used_addresses_gen = (tx['address'] + for tx in self.bci._yield_transactions(wallet_name) + if tx['category'] == 'receive') + used_indices = self.get_used_indices(used_addresses_gen) + jlog.debug("got used indices: {}".format(used_indices)) + gap_limit_used = not self.check_gap_indices(used_indices) + self.rewind_wallet_indices(used_indices, saved_indices) + + new_addresses = self.collect_addresses_gap() + if not new_addresses.issubset(imported_addresses): + jlog.debug("Syncing iteration finished, additional step required") + self.bci.add_watchonly_addresses(new_addresses - imported_addresses, + wallet_name, self.restart_callback) + self.synced = False + elif gap_limit_used: + jlog.debug("Syncing iteration finished, additional step required") + self.synced = False + else: + jlog.debug("Wallet successfully synced") + self.rewind_wallet_indices(used_indices, saved_indices) + self.synced = True + + def sync_unspent(self): + st = time.time() + # block height needs to be real time for addition to our utxos: + current_blockheight = self.bci.rpc("getblockcount", []) + wallet_name = self.get_wallet_name() + self.reset_utxos() + + listunspent_args = [] + if 'listunspent_args' in jm_single().config.options('POLICY'): + listunspent_args = ast.literal_eval(jm_single().config.get( + 'POLICY', 'listunspent_args')) + + unspent_list = self.bci.rpc('listunspent', listunspent_args) + unspent_list = [x for x in unspent_list if "label" in x] + # filter on label, but note (a) in certain circumstances (in- + # wallet transfer) it is possible for the utxo to be labeled + # with the external label, and (b) the wallet will know if it + # belongs or not anyway (is_known_addr): + our_unspent_list = [x for x in unspent_list if x["label"] in [ + wallet_name, self.EXTERNAL_WALLET_LABEL]] + for u in our_unspent_list: + if not self.is_known_addr(u['address']): + continue + self._add_unspent_utxo(u, current_blockheight) + et = time.time() + jlog.debug('bitcoind sync_unspent took ' + str((et - st)) + 'sec') + + def _add_unspent_utxo(self, utxo, current_blockheight): + """ + Add a UTXO as returned by rpc's listunspent call to the wallet. + + params: + utxo: single utxo dict as returned by listunspent + current_blockheight: blockheight as integer, used to + set the block in which a confirmed utxo is included. + """ + txid = binascii.unhexlify(utxo['txid']) + script = binascii.unhexlify(utxo['scriptPubKey']) + value = int(Decimal(str(utxo['amount'])) * Decimal('1e8')) + confs = int(utxo['confirmations']) + # wallet's utxo database needs to store an absolute rather + # than relative height measure: + height = None + if confs < 0: + jlog.warning("Utxo not added, has a conflict: " + str(utxo)) + return + if confs >=1 : + height = current_blockheight - confs + 1 + self.add_utxo(txid, int(utxo['vout']), script, value, height) + + + """ The following functions mostly are not pure + pass through to the underlying wallet, so declared + here; the remainder are covered by the __getattr__ + fallback. + """ + + def save_wallet(self): + self.wallet.save() + + def get_utxos_by_mixdepth(self, include_disabled=False, + verbose=False, hexfmt=True, includeconfs=False): + """ Returns utxos by mixdepth in a dict, optionally including + information about how many confirmations each utxo has. + TODO clean up underlying wallet.get_utxos_by_mixdepth (including verbosity + and formatting options) to make this less confusing. + """ + def height_to_confs(x): + # convert height entries to confirmations: + ubym_conv = collections.defaultdict(dict) + for m, i in x.items(): + for u, d in i.items(): + ubym_conv[m][u] = d + h = ubym_conv[m][u].pop("height") + if h == INF_HEIGHT: + confs = 0 + else: + confs = self.current_blockheight - h + 1 + ubym_conv[m][u]["confs"] = confs + return ubym_conv + + if hexfmt: + ubym = self.wallet.get_utxos_by_mixdepth(verbose=verbose, + includeheight=includeconfs) + if not includeconfs: + return ubym + else: + return height_to_confs(ubym) + else: + ubym = self.wallet.get_utxos_by_mixdepth_( + include_disabled=include_disabled, includeheight=includeconfs) + if not includeconfs: + return ubym + else: + return height_to_confs(ubym) + + def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None, + minconfs=None): + """ Request utxos from the wallet in a particular mixdepth to satisfy + a certain total amount, optionally set the selector function (or use + the currently configured function set by the wallet, and optionally + require a minimum of minconfs confirmations (default none means + unconfirmed are allowed). + """ + if minconfs is None: + maxheight = None + else: + maxheight = self.current_blockheight - minconfs + 1 + return self.wallet.select_utxos(mixdepth, amount, utxo_filter=utxo_filter, + select_fn=select_fn, maxheight=maxheight) + + def get_balance_by_mixdepth(self, verbose=True, + include_disabled=False, + minconfs=None): + if minconfs is None: + maxheight = None + else: + maxheight = self.current_blockheight - minconfs + 1 + return self.wallet.get_balance_by_mixdepth(verbose=verbose, + include_disabled=include_disabled, + maxheight=maxheight) + + def get_internal_addr(self, mixdepth): + if self.bci is not None and hasattr(self.bci, 'import_addresses'): + addr = self.wallet.get_internal_addr(mixdepth) + self.bci.import_addresses([addr], + self.wallet.get_wallet_name()) + return addr + + def collect_addresses_init(self): + """ Collects the "current" set of addresses, + as defined by the indices recorded in the wallet's + index cache (persisted in the wallet file usually). + Note that it collects up to the current indices plus + the gap limit. + """ + addresses = set() + saved_indices = dict() + + for md in range(self.max_mixdepth + 1): + saved_indices[md] = [0, 0] + for internal in (0, 1): + next_unused = self.get_next_unused_index(md, internal) + for index in range(next_unused): + addresses.add(self.get_addr(md, internal, index)) + for index in range(self.gap_limit): + addresses.add(self.get_new_addr(md, internal)) + # reset the indices to the value we had before the + # new address calls: + self.set_next_index(md, internal, next_unused) + saved_indices[md][internal] = next_unused + # include any imported addresses + for path in self.yield_imported_paths(md): + addresses.add(self.get_addr_path(path)) + + return addresses, saved_indices + + def collect_addresses_gap(self, gap_limit=None): + gap_limit = gap_limit or self.gap_limit + addresses = set() + + for md in range(self.max_mixdepth + 1): + for internal in (True, False): + old_next = self.get_next_unused_index(md, internal) + for index in range(gap_limit): + addresses.add(self.get_new_addr(md, internal)) + self.set_next_index(md, internal, old_next) + + return addresses + + def get_external_addr(self, mixdepth): + if self.bci is not None and hasattr(self.bci, 'import_addresses'): + addr = self.wallet.get_external_addr(mixdepth) + self.bci.import_addresses([addr], + self.wallet.get_wallet_name()) + return addr + + def __getattr__(self, attr): + # any method not present here is passed + # to the wallet: + return getattr(self.wallet, attr) diff --git a/jmclient/jmclient/wallet_utils.py b/jmclient/jmclient/wallet_utils.py index 20a435b..ff7af4e 100644 --- a/jmclient/jmclient/wallet_utils.py +++ b/jmclient/jmclient/wallet_utils.py @@ -13,9 +13,10 @@ from numbers import Integral from collections import Counter from itertools import islice from jmclient import (get_network, WALLET_IMPLEMENTATIONS, Storage, podle, - jm_single, BitcoinCoreInterface, JsonRpcError, sync_wallet, WalletError, + jm_single, BitcoinCoreInterface, JsonRpcError, WalletError, VolatileStorage, StoragePasswordError, is_segwit_mode, SegwitLegacyWallet, LegacyWallet, SegwitWallet, is_native_segwit_mode) +from jmclient.wallet_service import WalletService from jmbase.support import get_password, jmprint from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH from .output import fmt_utxo @@ -77,12 +78,12 @@ def get_wallettool_parser(): default=1, help=('History method verbosity, 0 (least) to 6 (most), ' '<=2 batches earnings, even values also list TXIDs')) - parser.add_option('--fast', + parser.add_option('--recoversync', action='store_true', - dest='fastsync', + dest='recoversync', default=False, - help=('choose to do fast wallet sync, only for Core and ' - 'only for previously synced wallet')) + help=('choose to do detailed wallet sync, ' + 'used for recovering on new Core instance.')) parser.add_option('-H', '--hd', action='store', @@ -328,22 +329,22 @@ def get_tx_info(txid): rpctx.get('blocktime', 0), txd -def get_imported_privkey_branch(wallet, m, showprivkey): +def get_imported_privkey_branch(wallet_service, m, showprivkey): entries = [] - for path in wallet.yield_imported_paths(m): - addr = wallet.get_addr_path(path) - script = wallet.get_script_path(path) + for path in wallet_service.yield_imported_paths(m): + addr = wallet_service.get_addr_path(path) + script = wallet_service.get_script_path(path) balance = 0.0 - for data in wallet.get_utxos_by_mixdepth_( - include_disabled=True)[m].values(): + for data in wallet_service.get_utxos_by_mixdepth(include_disabled=True, + hexfmt=False)[m].values(): if script == data['script']: balance += data['value'] used = ('used' if balance > 0.0 else 'empty') if showprivkey: - wip_privkey = wallet.get_wif_path(path) + wip_privkey = wallet_service.get_wif_path(path) else: wip_privkey = '' - entries.append(WalletViewEntry(wallet.get_path_repr(path), m, -1, + entries.append(WalletViewEntry(wallet_service.get_path_repr(path), m, -1, 0, addr, [balance, balance], used=used, priv=wip_privkey)) @@ -377,7 +378,7 @@ def wallet_showutxos(wallet, showprivkey): return json.dumps(unsp, indent=4) -def wallet_display(wallet, gaplimit, showprivkey, displayall=False, +def wallet_display(wallet_service, gaplimit, showprivkey, displayall=False, serialized=True, summarized=False): """build the walletview object, then return its serialization directly if serialized, @@ -412,45 +413,45 @@ def wallet_display(wallet, gaplimit, showprivkey, displayall=False, acctlist = [] # TODO - either optionally not show disabled utxos, or # mark them differently in display (labels; colors) - utxos = wallet.get_utxos_by_mixdepth_(include_disabled=True) - for m in range(wallet.mixdepth + 1): + utxos = wallet_service.get_utxos_by_mixdepth(include_disabled=True, hexfmt=False) + for m in range(wallet_service.mixdepth + 1): branchlist = [] for forchange in [0, 1]: entrylist = [] if forchange == 0: # users would only want to hand out the xpub for externals - xpub_key = wallet.get_bip32_pub_export(m, forchange) + xpub_key = wallet_service.get_bip32_pub_export(m, forchange) else: xpub_key = "" - unused_index = wallet.get_next_unused_index(m, forchange) + unused_index = wallet_service.get_next_unused_index(m, forchange) for k in range(unused_index + gaplimit): - path = wallet.get_path(m, forchange, k) - addr = wallet.get_addr_path(path) + path = wallet_service.get_path(m, forchange, k) + addr = wallet_service.get_addr_path(path) balance, used = get_addr_status( path, utxos[m], k >= unused_index, forchange) if showprivkey: - privkey = wallet.get_wif_path(path) + privkey = wallet_service.get_wif_path(path) else: privkey = '' if (displayall or balance > 0 or (used == 'new' and forchange == 0)): entrylist.append(WalletViewEntry( - wallet.get_path_repr(path), m, forchange, k, addr, + wallet_service.get_path_repr(path), m, forchange, k, addr, [balance, balance], priv=privkey, used=used)) - wallet.set_next_index(m, forchange, unused_index) - path = wallet.get_path_repr(wallet.get_path(m, forchange)) + wallet_service.set_next_index(m, forchange, unused_index) + path = wallet_service.get_path_repr(wallet_service.get_path(m, forchange)) branchlist.append(WalletViewBranch(path, m, forchange, entrylist, xpub=xpub_key)) - ipb = get_imported_privkey_branch(wallet, m, showprivkey) + ipb = get_imported_privkey_branch(wallet_service, m, showprivkey) if ipb: branchlist.append(ipb) #get the xpub key of the whole account - xpub_account = wallet.get_bip32_pub_export(mixdepth=m) - path = wallet.get_path_repr(wallet.get_path(m)) + xpub_account = wallet_service.get_bip32_pub_export(mixdepth=m) + path = wallet_service.get_path_repr(wallet_service.get_path(m)) acctlist.append(WalletViewAccount(path, m, branchlist, xpub=xpub_account)) - path = wallet.get_path_repr(wallet.get_path()) + path = wallet_service.get_path_repr(wallet_service.get_path()) walletview = WalletView(path, acctlist) if serialized: return walletview.serialize(summarize=summarized) @@ -598,7 +599,7 @@ def wallet_fetch_history(wallet, options): tx_db.execute("CREATE TABLE transactions(txid TEXT, " "blockhash TEXT, blocktime INTEGER, conflicts INTEGER);") jm_single().debug_silence[0] = True - wallet_name = jm_single().bc_interface.get_wallet_name(wallet) + wallet_name = wallet.get_wallet_name() buf = range(1000) t = 0 while len(buf) == 1000: @@ -850,8 +851,8 @@ def wallet_fetch_history(wallet, options): jmprint(('BUG ERROR: wallet balance (%s) does not match balance from ' + 'history (%s)') % (sat_to_str(total_wallet_balance), sat_to_str(balance))) - wallet_utxo_count = sum(map(len, wallet.get_utxos_by_mixdepth_( - include_disabled=True).values())) + wallet_utxo_count = sum(map(len, wallet.get_utxos_by_mixdepth( + include_disabled=True, hexfmt=False).values())) if utxo_count + unconfirmed_utxo_count != wallet_utxo_count: jmprint(('BUG ERROR: wallet utxo count (%d) does not match utxo count from ' + 'history (%s)') % (wallet_utxo_count, utxo_count)) @@ -969,11 +970,11 @@ def display_utxos_for_disable_choice_default(utxos_enabled, utxos_disabled): disable = False if chosen_idx <= disabled_max else True return ulist[chosen_idx], disable -def get_utxos_enabled_disabled(wallet, md): +def get_utxos_enabled_disabled(wallet_service, md): """ Returns dicts for enabled and disabled separately """ - utxos_enabled = wallet.get_utxos_by_mixdepth_()[md] - utxos_all = wallet.get_utxos_by_mixdepth_(include_disabled=True)[md] + utxos_enabled = wallet_service.get_utxos_by_mixdepth(hexfmt=False)[md] + utxos_all = wallet_service.get_utxos_by_mixdepth(include_disabled=True, hexfmt=False)[md] utxos_disabled_keyset = set(utxos_all).difference(set(utxos_enabled)) utxos_disabled = {} for u in utxos_disabled_keyset: @@ -1207,28 +1208,34 @@ def wallet_tool_main(wallet_root_path): wallet_path, seed, options.mixdepth, read_only=read_only, gap_limit=options.gaplimit) + # this object is only to respect the layering, + # the service will not be started since this is a synchronous script: + wallet_service = WalletService(wallet) + if method not in noscan_methods: # if nothing was configured, we override bitcoind's options so that # unconfirmed balance is included in the wallet display by default if 'listunspent_args' not in jm_single().config.options('POLICY'): jm_single().config.set('POLICY','listunspent_args', '[0]') - while not jm_single().bc_interface.wallet_synced: - sync_wallet(wallet, fast=options.fastsync) + while True: + if wallet_service.sync_wallet(fast = not options.recoversync): + break + #Now the wallet/data is prepared, execute the script according to the method if method == "display": - return wallet_display(wallet, options.gaplimit, options.showprivkey) + return wallet_display(wallet_service, options.gaplimit, options.showprivkey) elif method == "displayall": - return wallet_display(wallet, options.gaplimit, options.showprivkey, + return wallet_display(wallet_service, options.gaplimit, options.showprivkey, displayall=True) elif method == "summary": - return wallet_display(wallet, options.gaplimit, options.showprivkey, summarized=True) + return wallet_display(wallet_service, options.gaplimit, options.showprivkey, summarized=True) elif method == "history": if not isinstance(jm_single().bc_interface, BitcoinCoreInterface): jmprint('showing history only available when using the Bitcoin Core ' + 'blockchain interface', "error") sys.exit(0) else: - return wallet_fetch_history(wallet, options) + return wallet_fetch_history(wallet_service, options) elif method == "generate": retval = wallet_generate_recover("generate", wallet_root_path, mixdepth=options.mixdepth) @@ -1238,22 +1245,22 @@ def wallet_tool_main(wallet_root_path): mixdepth=options.mixdepth) return "Recovered wallet OK" if retval else "Failed" elif method == "showutxos": - return wallet_showutxos(wallet, options.showprivkey) + return wallet_showutxos(wallet_service, options.showprivkey) elif method == "showseed": - return wallet_showseed(wallet) + return wallet_showseed(wallet_service) elif method == "dumpprivkey": - return wallet_dumpprivkey(wallet, options.hd_path) + return wallet_dumpprivkey(wallet_service, options.hd_path) elif method == "importprivkey": #note: must be interactive (security) if options.mixdepth is None: parser.error("You need to specify a mixdepth with -m") - wallet_importprivkey(wallet, options.mixdepth, + wallet_importprivkey(wallet_service, options.mixdepth, map_key_type(options.key_type)) return "Key import completed." elif method == "signmessage": - return wallet_signmessage(wallet, options.hd_path, args[2]) + return wallet_signmessage(wallet_service, options.hd_path, args[2]) elif method == "freeze": - return wallet_freezeutxo(wallet, options.mixdepth) + return wallet_freezeutxo(wallet_service, options.mixdepth) else: parser.error("Unknown wallet-tool method: " + method) sys.exit(0) diff --git a/jmclient/jmclient/yieldgenerator.py b/jmclient/jmclient/yieldgenerator.py index 6214d45..5b2d05e 100644 --- a/jmclient/jmclient/yieldgenerator.py +++ b/jmclient/jmclient/yieldgenerator.py @@ -12,11 +12,13 @@ from twisted.python.log import startLogging from optparse import OptionParser from jmbase import get_log from jmclient import Maker, jm_single, load_program_config, \ - sync_wallet, JMClientProtocolFactory, start_reactor, calc_cj_fee + JMClientProtocolFactory, start_reactor, \ + calc_cj_fee, WalletService from .wallet_utils import open_test_wallet_maybe, get_wallet_path jlog = get_log() +MAX_MIX_DEPTH = 5 class YieldGenerator(Maker): """A maker for the purposes of generating a yield from held @@ -26,8 +28,8 @@ class YieldGenerator(Maker): __metaclass__ = abc.ABCMeta statement_file = os.path.join('logs', 'yigen-statement.csv') - def __init__(self, wallet): - Maker.__init__(self, wallet) + def __init__(self, wallet_service): + Maker.__init__(self, wallet_service) self.tx_unconfirm_timestamp = {} if not os.path.isfile(self.statement_file): self.log_statement( @@ -44,7 +46,7 @@ class YieldGenerator(Maker): self.income_statement.write(','.join(data) + '\n') self.income_statement.close() - def on_tx_unconfirmed(self, offer, txid, removed_utxos): + def on_tx_unconfirmed(self, offer, txid): self.tx_unconfirm_timestamp[offer["cjaddr"]] = int(time.time()) newoffers = self.create_my_orders() @@ -70,10 +72,10 @@ class YieldGeneratorBasic(YieldGenerator): It will often (but not always) reannounce orders after transactions, thus is somewhat suboptimal in giving more information to spies. """ - def __init__(self, wallet, offerconfig): + def __init__(self, wallet_service, offerconfig): self.txfee, self.cjfee_a, self.cjfee_r, self.ordertype, self.minsize \ = offerconfig - super(YieldGeneratorBasic,self).__init__(wallet) + super(YieldGeneratorBasic,self).__init__(wallet_service) def create_my_orders(self): mix_balance = self.get_available_mixdepths() @@ -129,10 +131,9 @@ class YieldGeneratorBasic(YieldGenerator): return None, None, None jlog.info('sending output to address=' + str(cj_addr)) - change_addr = self.wallet.get_internal_addr(mixdepth, - jm_single().bc_interface) + change_addr = self.wallet_service.get_internal_addr(mixdepth) - utxos = self.wallet.select_utxos(mixdepth, total_amount) + utxos = self.wallet_service.select_utxos(mixdepth, total_amount, minconfs=1) my_total_in = sum([va['value'] for va in utxos.values()]) real_cjfee = calc_cj_fee(offer["ordertype"], offer["cjfee"], amount) change_value = my_total_in - amount - offer["txfee"] + real_cjfee @@ -140,8 +141,8 @@ class YieldGeneratorBasic(YieldGenerator): jlog.debug(('change value={} below dust threshold, ' 'finding new utxos').format(change_value)) try: - utxos = self.wallet.select_utxos( - mixdepth, total_amount + jm_single().DUST_THRESHOLD) + utxos = self.wallet_service.select_utxos(mixdepth, + total_amount + jm_single().DUST_THRESHOLD, minconfs=1) except Exception: jlog.info('dont have the required UTXOs to make a ' 'output above the dust threshold, quitting') @@ -149,7 +150,7 @@ class YieldGeneratorBasic(YieldGenerator): return utxos, cj_addr, change_addr - def on_tx_confirmed(self, offer, confirmations, txid): + def on_tx_confirmed(self, offer, txid, confirmations): if offer["cjaddr"] in self.tx_unconfirm_timestamp: confirm_time = int(time.time()) - self.tx_unconfirm_timestamp[ offer["cjaddr"]] @@ -162,12 +163,13 @@ class YieldGeneratorBasic(YieldGenerator): offer["utxos"]), sum([av['value'] for av in offer["utxos"].values( )]), real_cjfee, real_cjfee - offer["offer"]["txfee"], round( confirm_time / 60.0, 2), '']) - return self.on_tx_unconfirmed(offer, txid, None) + return self.on_tx_unconfirmed(offer, txid) def get_available_mixdepths(self): """Returns the mixdepth/balance dict from the wallet that contains all available inputs for offers.""" - return self.wallet.get_balance_by_mixdepth(verbose=False) + return self.wallet_service.get_balance_by_mixdepth(verbose=False, + minconfs=1) def select_input_mixdepth(self, available, offer, amount): """Returns the mixdepth from which the given order should spend the @@ -182,8 +184,8 @@ class YieldGeneratorBasic(YieldGenerator): an order spending from the given input mixdepth. Can return None if there is no suitable output, in which case the order is aborted.""" - cjoutmix = (input_mixdepth + 1) % (self.wallet.mixdepth + 1) - return self.wallet.get_internal_addr(cjoutmix, jm_single().bc_interface) + cjoutmix = (input_mixdepth + 1) % (self.wallet_service.mixdepth + 1) + return self.wallet_service.get_internal_addr(cjoutmix) def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffer', @@ -209,12 +211,12 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffe parser.add_option('-g', '--gap-limit', action='store', type="int", dest='gaplimit', default=gaplimit, help='gap limit for wallet, default='+str(gaplimit)) - parser.add_option('--fast', + parser.add_option('--recoversync', action='store_true', - dest='fastsync', + dest='recoversync', default=False, - help=('choose to do fast wallet sync, only for Core and ' - 'only for previously synced wallet')) + help=('choose to do detailed wallet sync, ' + 'used for recovering on new Core instance.')) parser.add_option('-m', '--mixdepth', action='store', type='int', dest='mixdepth', default=None, help="highest mixdepth to use") @@ -248,13 +250,12 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffe wallet_path, wallet_name, options.mixdepth, gap_limit=options.gaplimit) - if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == "electrum-server": - jm_single().bc_interface.synctype = "with-script" + wallet_service = WalletService(wallet) + while not wallet_service.synced: + wallet_service.sync_wallet(fast=not options.recoversync) + wallet_service.startService() - while not jm_single().bc_interface.wallet_synced: - sync_wallet(wallet, fast=options.fastsync) - - maker = ygclass(wallet, [options.txfee, cjfee_a, cjfee_r, + maker = ygclass(wallet_service, [options.txfee, cjfee_a, cjfee_r, options.ordertype, options.minsize]) jlog.info('starting yield generator') clientfactory = JMClientProtocolFactory(maker, proto_type="MAKER") diff --git a/jmclient/test/commontest.py b/jmclient/test/commontest.py index b3d4a41..bbf21f4 100644 --- a/jmclient/test/commontest.py +++ b/jmclient/test/commontest.py @@ -12,7 +12,8 @@ from decimal import Decimal from jmbase import get_log from jmclient import ( jm_single, open_test_wallet_maybe, estimate_tx_fee, - BlockchainInterface, get_p2sh_vbyte, BIP32Wallet, SegwitLegacyWallet) + BlockchainInterface, get_p2sh_vbyte, BIP32Wallet, + SegwitLegacyWallet, WalletService) from jmbase.support import chunks import jmbitcoin as btc @@ -30,24 +31,15 @@ class DummyBlockchainInterface(BlockchainInterface): self.fake_query_results = None self.qusfail = False + def rpc(self, a, b): + return None def sync_addresses(self, wallet): pass def sync_unspent(self, wallet): pass - def import_addresses(self, addr_list, wallet_name): + def import_addresses(self, addr_list, wallet_name, restart_cb=None): pass - def outputs_watcher(self, wallet_name, notifyaddr, - tx_output_set, uf, cf, tf): - pass - def tx_watcher(self, txd, ucf, cf, sf, c, n): - pass - def add_tx_notify(self, - txd, - unconfirmfun, - confirmfun, - notifyaddr, - timeoutfun=None, - vb=None): + def is_address_imported(self, addr): pass def pushtx(self, txhex): @@ -127,7 +119,7 @@ def binarize_tx(tx): def make_sign_and_push(ins_full, - wallet, + wallet_service, amount, output_addr=None, change_addr=None, @@ -136,11 +128,12 @@ def make_sign_and_push(ins_full, """Utility function for easily building transactions from wallets """ + assert isinstance(wallet_service, WalletService) total = sum(x['value'] for x in ins_full.values()) ins = list(ins_full.keys()) #random output address and change addr - output_addr = wallet.get_new_addr(1, 1) if not output_addr else output_addr - change_addr = wallet.get_new_addr(1, 0) if not change_addr else change_addr + output_addr = wallet_service.get_new_addr(1, 1) if not output_addr else output_addr + change_addr = wallet_service.get_new_addr(0, 1) if not change_addr else change_addr fee_est = estimate_tx_fee(len(ins), 2) if estimate_fee else 10000 outs = [{'value': amount, 'address': output_addr}, {'value': total - amount - fee_est, @@ -150,15 +143,18 @@ def make_sign_and_push(ins_full, scripts = {} for index, ins in enumerate(de_tx['ins']): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - script = wallet.addr_to_script(ins_full[utxo]['address']) + script = wallet_service.addr_to_script(ins_full[utxo]['address']) scripts[index] = (script, ins_full[utxo]['value']) binarize_tx(de_tx) - de_tx = wallet.sign_tx(de_tx, scripts, hashcode=hashcode) + de_tx = wallet_service.sign_tx(de_tx, scripts, hashcode=hashcode) #pushtx returns False on any error push_succeed = jm_single().bc_interface.pushtx(btc.serialize(de_tx)) if push_succeed: - removed = wallet.remove_old_utxos(de_tx) - return btc.txhash(btc.serialize(de_tx)) + txid = btc.txhash(btc.serialize(de_tx)) + # in normal operation this happens automatically + # but in some tests there is no monitoring loop: + wallet_service.process_new_tx(de_tx, txid) + return txid else: return False @@ -194,17 +190,17 @@ def make_wallets(n, w = open_test_wallet_maybe(seeds[i], seeds[i], mixdepths - 1, test_wallet_cls=wallet_cls) - + wallet_service = WalletService(w) wallets[i + start_index] = {'seed': seeds[i], - 'wallet': w} + 'wallet': wallet_service} for j in range(mixdepths): for k in range(wallet_structures[i][j]): deviation = sdev_amt * random.random() amt = mean_amt - sdev_amt / 2.0 + deviation if amt < 0: amt = 0.001 amt = float(Decimal(amt).quantize(Decimal(10)**-8)) - jm_single().bc_interface.grab_coins( - w.get_new_addr(j, populate_internal), amt) + jm_single().bc_interface.grab_coins(wallet_service.get_new_addr( + j, populate_internal), amt) return wallets diff --git a/jmclient/test/test_blockchaininterface.py b/jmclient/test/test_blockchaininterface.py index 1efdd18..cc4c523 100644 --- a/jmclient/test/test_blockchaininterface.py +++ b/jmclient/test/test_blockchaininterface.py @@ -9,16 +9,16 @@ from commontest import create_wallet_for_sync import pytest from jmbase import get_log -from jmclient import load_program_config, jm_single, sync_wallet +from jmclient import load_program_config, jm_single log = get_log() -def sync_test_wallet(fast, wallet): +def sync_test_wallet(fast, wallet_service): sync_count = 0 - jm_single().bc_interface.wallet_synced = False - while not jm_single().bc_interface.wallet_synced: - sync_wallet(wallet, fast=fast) + wallet_service.synced = False + while not wallet_service.synced: + wallet_service.sync_wallet(fast=fast) sync_count += 1 # avoid infinite loop assert sync_count < 10 @@ -27,15 +27,15 @@ def sync_test_wallet(fast, wallet): @pytest.mark.parametrize('fast', (False, True)) def test_empty_wallet_sync(setup_wallets, fast): - wallet = create_wallet_for_sync([0, 0, 0, 0, 0], ['test_empty_wallet_sync']) + wallet_service = create_wallet_for_sync([0, 0, 0, 0, 0], ['test_empty_wallet_sync']) - sync_test_wallet(fast, wallet) + sync_test_wallet(fast, wallet_service) broken = True - for md in range(wallet.max_mixdepth + 1): + for md in range(wallet_service.max_mixdepth + 1): for internal in (True, False): broken = False - assert 0 == wallet.get_next_unused_index(md, internal) + assert 0 == wallet_service.get_next_unused_index(md, internal) assert not broken @@ -44,106 +44,115 @@ def test_empty_wallet_sync(setup_wallets, fast): (True, False), (True, True))) def test_sequentially_used_wallet_sync(setup_wallets, fast, internal): used_count = [1, 3, 6, 2, 23] - wallet = create_wallet_for_sync( + wallet_service = create_wallet_for_sync( used_count, ['test_sequentially_used_wallet_sync'], populate_internal=internal) - sync_test_wallet(fast, wallet) + sync_test_wallet(fast, wallet_service) broken = True for md in range(len(used_count)): broken = False - assert used_count[md] == wallet.get_next_unused_index(md, internal) + assert used_count[md] == wallet_service.get_next_unused_index(md, internal) assert not broken -@pytest.mark.parametrize('fast', (False, True)) +@pytest.mark.parametrize('fast', (False,)) def test_gap_used_wallet_sync(setup_wallets, fast): + """ After careful examination this test now only includes the Recovery sync. + Note: pre-Aug 2019, because of a bug, this code was not in fact testing both + Fast and Recovery sync, but only Recovery (twice). Also, the scenario set + out in this test (where coins are funded to a wallet which has no index-cache, + and initially no imports) is only appropriate for recovery-mode sync, not for + fast-mode (the now default). + """ used_count = [1, 3, 6, 2, 23] - wallet = create_wallet_for_sync(used_count, ['test_gap_used_wallet_sync']) - wallet.gap_limit = 20 + wallet_service = create_wallet_for_sync(used_count, ['test_gap_used_wallet_sync']) + wallet_service.gap_limit = 20 for md in range(len(used_count)): x = -1 for x in range(md): - assert x <= wallet.gap_limit, "test broken" + assert x <= wallet_service.gap_limit, "test broken" # create some unused addresses - wallet.get_new_script(md, True) - wallet.get_new_script(md, False) + wallet_service.get_new_script(md, True) + wallet_service.get_new_script(md, False) used_count[md] += x + 2 - jm_single().bc_interface.grab_coins(wallet.get_new_addr(md, True), 1) - jm_single().bc_interface.grab_coins(wallet.get_new_addr(md, False), 1) + jm_single().bc_interface.grab_coins(wallet_service.get_new_addr(md, True), 1) + jm_single().bc_interface.grab_coins(wallet_service.get_new_addr(md, False), 1) # reset indices to simulate completely unsynced wallet - for md in range(wallet.max_mixdepth + 1): - wallet.set_next_index(md, True, 0) - wallet.set_next_index(md, False, 0) - - sync_test_wallet(fast, wallet) + for md in range(wallet_service.max_mixdepth + 1): + wallet_service.set_next_index(md, True, 0) + wallet_service.set_next_index(md, False, 0) + sync_test_wallet(fast, wallet_service) broken = True for md in range(len(used_count)): broken = False - assert md + 1 == wallet.get_next_unused_index(md, True) - assert used_count[md] == wallet.get_next_unused_index(md, False) + assert md + 1 == wallet_service.get_next_unused_index(md, True) + assert used_count[md] == wallet_service.get_next_unused_index(md, False) assert not broken -@pytest.mark.parametrize('fast', (False, True)) +@pytest.mark.parametrize('fast', (False,)) def test_multigap_used_wallet_sync(setup_wallets, fast): + """ See docstring for test_gap_used_wallet_sync; exactly the + same applies here. + """ start_index = 5 used_count = [start_index, 0, 0, 0, 0] - wallet = create_wallet_for_sync(used_count, ['test_multigap_used_wallet_sync']) - wallet.gap_limit = 5 + wallet_service = create_wallet_for_sync(used_count, ['test_multigap_used_wallet_sync']) + wallet_service.gap_limit = 5 mixdepth = 0 for w in range(5): - for x in range(int(wallet.gap_limit * 0.6)): - assert x <= wallet.gap_limit, "test broken" + for x in range(int(wallet_service.gap_limit * 0.6)): + assert x <= wallet_service.gap_limit, "test broken" # create some unused addresses - wallet.get_new_script(mixdepth, True) - wallet.get_new_script(mixdepth, False) + wallet_service.get_new_script(mixdepth, True) + wallet_service.get_new_script(mixdepth, False) used_count[mixdepth] += x + 2 - jm_single().bc_interface.grab_coins(wallet.get_new_addr(mixdepth, True), 1) - jm_single().bc_interface.grab_coins(wallet.get_new_addr(mixdepth, False), 1) + jm_single().bc_interface.grab_coins(wallet_service.get_new_addr(mixdepth, True), 1) + jm_single().bc_interface.grab_coins(wallet_service.get_new_addr(mixdepth, False), 1) # reset indices to simulate completely unsynced wallet - for md in range(wallet.max_mixdepth + 1): - wallet.set_next_index(md, True, 0) - wallet.set_next_index(md, False, 0) + for md in range(wallet_service.max_mixdepth + 1): + wallet_service.set_next_index(md, True, 0) + wallet_service.set_next_index(md, False, 0) - sync_test_wallet(fast, wallet) + sync_test_wallet(fast, wallet_service) - assert used_count[mixdepth] - start_index == wallet.get_next_unused_index(mixdepth, True) - assert used_count[mixdepth] == wallet.get_next_unused_index(mixdepth, False) + assert used_count[mixdepth] - start_index == wallet_service.get_next_unused_index(mixdepth, True) + assert used_count[mixdepth] == wallet_service.get_next_unused_index(mixdepth, False) @pytest.mark.parametrize('fast', (False, True)) def test_retain_unused_indices_wallet_sync(setup_wallets, fast): used_count = [0, 0, 0, 0, 0] - wallet = create_wallet_for_sync(used_count, ['test_retain_unused_indices_wallet_sync']) + wallet_service = create_wallet_for_sync(used_count, ['test_retain_unused_indices_wallet_sync']) for x in range(9): - wallet.get_new_script(0, 1) + wallet_service.get_new_script(0, 1) - sync_test_wallet(fast, wallet) + sync_test_wallet(fast, wallet_service) - assert wallet.get_next_unused_index(0, 1) == 9 + assert wallet_service.get_next_unused_index(0, 1) == 9 @pytest.mark.parametrize('fast', (False, True)) def test_imported_wallet_sync(setup_wallets, fast): used_count = [0, 0, 0, 0, 0] - wallet = create_wallet_for_sync(used_count, ['test_imported_wallet_sync']) - source_wallet = create_wallet_for_sync(used_count, ['test_imported_wallet_sync_origin']) + wallet_service = create_wallet_for_sync(used_count, ['test_imported_wallet_sync']) + source_wallet_service = create_wallet_for_sync(used_count, ['test_imported_wallet_sync_origin']) - address = source_wallet.get_new_addr(0, 1) - wallet.import_private_key(0, source_wallet.get_wif(0, 1, 0)) + address = source_wallet_service.get_internal_addr(0) + wallet_service.import_private_key(0, source_wallet_service.get_wif(0, 1, 0)) txid = binascii.unhexlify(jm_single().bc_interface.grab_coins(address, 1)) - sync_test_wallet(fast, wallet) + sync_test_wallet(fast, wallet_service) - assert wallet._utxos.have_utxo(txid, 0) == 0 + assert wallet_service._utxos.have_utxo(txid, 0) == 0 @pytest.fixture(scope='module') diff --git a/jmclient/test/test_client_protocol.py b/jmclient/test/test_client_protocol.py index e8df0b9..4e4549d 100644 --- a/jmclient/test/test_client_protocol.py +++ b/jmclient/test/test_client_protocol.py @@ -6,7 +6,7 @@ from builtins import * from jmbase import get_log from jmclient import load_program_config, Taker,\ - JMClientProtocolFactory, jm_single, Maker + JMClientProtocolFactory, jm_single, Maker, WalletService from jmclient.client_protocol import JMTakerClientProtocol from twisted.python.log import msg as tmsg from twisted.internet import protocol, reactor, task @@ -83,11 +83,15 @@ class DummyWallet(object): def get_wallet_id(self): return 'aaaa' +#class DummyWalletService(object): +# wallet = DummyWallet() +# def register_callbacks(self, callbacks, unconfirmed=True): +# pass class DummyMaker(Maker): def __init__(self): self.aborted = False - self.wallet = DummyWallet() + self.wallet_service = WalletService(DummyWallet()) self.offerlist = self.create_my_orders() def try_to_create_my_orders(self): @@ -125,10 +129,10 @@ class DummyMaker(Maker): # utxos, cj_addr, change_addr return [], '', '' - def on_tx_unconfirmed(self, cjorder, txid, removed_utxos): + def on_tx_unconfirmed(self, cjorder, txid): return [], [] - def on_tx_confirmed(self, cjorder, confirmations, txid): + def on_tx_confirmed(self, cjorder, txid, confirmations): return [], [] @@ -278,7 +282,7 @@ class TrialTestJMClientProto(unittest.TestCase): self.addCleanup(self.client.transport.loseConnection) clientfactories = [] takers = [DummyTaker( - None, ["a", "b"], callbacks=( + WalletService(DummyWallet()), ["a", "b"], callbacks=( None, None, dummy_taker_finished)) for _ in range(len(params))] for i, p in enumerate(params): takers[i].set_fail_init(p[0]) diff --git a/jmclient/test/test_coinjoin.py b/jmclient/test/test_coinjoin.py index ac562fe..1031d35 100644 --- a/jmclient/test/test_coinjoin.py +++ b/jmclient/test/test_coinjoin.py @@ -12,8 +12,8 @@ import pytest from twisted.internet import reactor from jmbase import get_log -from jmclient import load_program_config, jm_single, \ - YieldGeneratorBasic, Taker, sync_wallet, LegacyWallet, SegwitLegacyWallet +from jmclient import load_program_config, jm_single,\ + YieldGeneratorBasic, Taker, LegacyWallet, SegwitLegacyWallet from jmclient.podle import set_commitment_file from commontest import make_wallets, binarize_tx from test_taker import dummy_filter_orderbook @@ -30,18 +30,21 @@ def make_wallets_to_list(make_wallets_data): assert all(wallets) return wallets - -def sync_wallets(wallets): - for w in wallets: - w.gap_limit = 0 - jm_single().bc_interface.wallet_synced = False +def sync_wallets(wallet_services, fast=True): + for wallet_service in wallet_services: + wallet_service.synced = False + wallet_service.gap_limit = 0 for x in range(20): - if jm_single().bc_interface.wallet_synced: + if wallet_service.synced: break - sync_wallet(w) + wallet_service.sync_wallet(fast=fast) else: assert False, "Failed to sync wallet" - + # because we don't run the monitoring loops for the + # wallet services, we need to update them on the latest + # block manually: + for wallet_service in wallet_services: + wallet_service.update_blockheight() def create_orderbook(makers): orderbook = [] @@ -119,15 +122,17 @@ def test_simple_coinjoin(monkeypatch, tmpdir, setup_cj, wallet_cls): set_commitment_file(str(tmpdir.join('commitments.json'))) MAKER_NUM = 3 - wallets = make_wallets_to_list(make_wallets( + wallet_services = make_wallets_to_list(make_wallets( MAKER_NUM + 1, wallet_structures=[[4, 0, 0, 0, 0]] * (MAKER_NUM + 1), mean_amt=1, wallet_cls=wallet_cls)) jm_single().bc_interface.tickchain() - sync_wallets(wallets) + jm_single().bc_interface.tickchain() + + sync_wallets(wallet_services) makers = [YieldGeneratorBasic( - wallets[i], + wallet_services[i], [0, 2000, 0, 'swabsoffer', 10**7]) for i in range(MAKER_NUM)] orderbook = create_orderbook(makers) @@ -136,7 +141,7 @@ def test_simple_coinjoin(monkeypatch, tmpdir, setup_cj, wallet_cls): cj_amount = int(1.1 * 10**8) # mixdepth, amount, counterparties, dest_addr, waittime schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0)] - taker = create_taker(wallets[-1], schedule, monkeypatch) + taker = create_taker(wallet_services[-1], schedule, monkeypatch) active_orders, maker_data = init_coinjoin(taker, makers, orderbook, cj_amount) @@ -156,21 +161,22 @@ def test_coinjoin_mixdepth_wrap_taker(monkeypatch, tmpdir, setup_cj): set_commitment_file(str(tmpdir.join('commitments.json'))) MAKER_NUM = 3 - wallets = make_wallets_to_list(make_wallets( + wallet_services = make_wallets_to_list(make_wallets( MAKER_NUM + 1, wallet_structures=[[4, 0, 0, 0, 0]] * MAKER_NUM + [[0, 0, 0, 0, 3]], mean_amt=1)) - for w in wallets: - assert w.max_mixdepth == 4 + for wallet_service in wallet_services: + assert wallet_service.max_mixdepth == 4 jm_single().bc_interface.tickchain() jm_single().bc_interface.tickchain() - sync_wallets(wallets) + + sync_wallets(wallet_services) cj_fee = 2000 makers = [YieldGeneratorBasic( - wallets[i], + wallet_services[i], [0, cj_fee, 0, 'swabsoffer', 10**7]) for i in range(MAKER_NUM)] orderbook = create_orderbook(makers) @@ -179,7 +185,7 @@ def test_coinjoin_mixdepth_wrap_taker(monkeypatch, tmpdir, setup_cj): cj_amount = int(1.1 * 10**8) # mixdepth, amount, counterparties, dest_addr, waittime schedule = [(4, cj_amount, MAKER_NUM, 'INTERNAL', 0)] - taker = create_taker(wallets[-1], schedule, monkeypatch) + taker = create_taker(wallet_services[-1], schedule, monkeypatch) active_orders, maker_data = init_coinjoin(taker, makers, orderbook, cj_amount) @@ -193,11 +199,12 @@ def test_coinjoin_mixdepth_wrap_taker(monkeypatch, tmpdir, setup_cj): tx = btc.deserialize(txdata[2]) binarize_tx(tx) - w = wallets[-1] - w.remove_old_utxos_(tx) - w.add_new_utxos_(tx, b'\x00' * 32) # fake txid + wallet_service = wallet_services[-1] + # TODO change for new tx monitoring: + wallet_service.remove_old_utxos_(tx) + wallet_service.add_new_utxos_(tx, b'\x00' * 32) # fake txid - balances = w.get_balance_by_mixdepth() + balances = wallet_service.get_balance_by_mixdepth() assert balances[0] == cj_amount # <= because of tx fee assert balances[4] <= 3 * 10**8 - cj_amount - (cj_fee * MAKER_NUM) @@ -210,21 +217,22 @@ def test_coinjoin_mixdepth_wrap_maker(monkeypatch, tmpdir, setup_cj): set_commitment_file(str(tmpdir.join('commitments.json'))) MAKER_NUM = 2 - wallets = make_wallets_to_list(make_wallets( + wallet_services = make_wallets_to_list(make_wallets( MAKER_NUM + 1, wallet_structures=[[0, 0, 0, 0, 4]] * MAKER_NUM + [[3, 0, 0, 0, 0]], mean_amt=1)) - for w in wallets: - assert w.max_mixdepth == 4 + for wallet_service in wallet_services: + assert wallet_service.max_mixdepth == 4 jm_single().bc_interface.tickchain() jm_single().bc_interface.tickchain() - sync_wallets(wallets) + + sync_wallets(wallet_services) cj_fee = 2000 makers = [YieldGeneratorBasic( - wallets[i], + wallet_services[i], [0, cj_fee, 0, 'swabsoffer', 10**7]) for i in range(MAKER_NUM)] orderbook = create_orderbook(makers) @@ -233,7 +241,7 @@ def test_coinjoin_mixdepth_wrap_maker(monkeypatch, tmpdir, setup_cj): cj_amount = int(1.1 * 10**8) # mixdepth, amount, counterparties, dest_addr, waittime schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0)] - taker = create_taker(wallets[-1], schedule, monkeypatch) + taker = create_taker(wallet_services[-1], schedule, monkeypatch) active_orders, maker_data = init_coinjoin(taker, makers, orderbook, cj_amount) @@ -248,43 +256,46 @@ def test_coinjoin_mixdepth_wrap_maker(monkeypatch, tmpdir, setup_cj): binarize_tx(tx) for i in range(MAKER_NUM): - w = wallets[i] - w.remove_old_utxos_(tx) - w.add_new_utxos_(tx, b'\x00' * 32) # fake txid + wallet_service = wallet_services[i] + # TODO as above re: monitoring + wallet_service.remove_old_utxos_(tx) + wallet_service.add_new_utxos_(tx, b'\x00' * 32) # fake txid - balances = w.get_balance_by_mixdepth() + balances = wallet_service.get_balance_by_mixdepth() assert balances[0] == cj_amount assert balances[4] == 4 * 10**8 - cj_amount + cj_fee @pytest.mark.parametrize('wallet_cls,wallet_cls_sec', ( (SegwitLegacyWallet, LegacyWallet), - (LegacyWallet, SegwitLegacyWallet) + #(LegacyWallet, SegwitLegacyWallet) )) def test_coinjoin_mixed_maker_addresses(monkeypatch, tmpdir, setup_cj, wallet_cls, wallet_cls_sec): set_commitment_file(str(tmpdir.join('commitments.json'))) MAKER_NUM = 2 - wallets = make_wallets_to_list(make_wallets( + wallet_services = make_wallets_to_list(make_wallets( MAKER_NUM + 1, wallet_structures=[[1, 0, 0, 0, 0]] * MAKER_NUM + [[3, 0, 0, 0, 0]], mean_amt=1, wallet_cls=wallet_cls)) - wallets_sec = make_wallets_to_list(make_wallets( + wallet_services_sec = make_wallets_to_list(make_wallets( MAKER_NUM, wallet_structures=[[1, 0, 0, 0, 0]] * MAKER_NUM, mean_amt=1, wallet_cls=wallet_cls_sec)) for i in range(MAKER_NUM): - wif = wallets_sec[i].get_wif(0, False, 0) - wallets[i].import_private_key(0, wif, key_type=wallets_sec[i].TYPE) + wif = wallet_services_sec[i].get_wif(0, False, 0) + wallet_services[i].wallet.import_private_key(0, wif, + key_type=wallet_services_sec[i].wallet.TYPE) jm_single().bc_interface.tickchain() jm_single().bc_interface.tickchain() - sync_wallets(wallets) + + sync_wallets(wallet_services, fast=False) makers = [YieldGeneratorBasic( - wallets[i], + wallet_services[i], [0, 2000, 0, 'swabsoffer', 10**7]) for i in range(MAKER_NUM)] orderbook = create_orderbook(makers) @@ -292,7 +303,7 @@ def test_coinjoin_mixed_maker_addresses(monkeypatch, tmpdir, setup_cj, cj_amount = int(1.1 * 10**8) # mixdepth, amount, counterparties, dest_addr, waittime schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0)] - taker = create_taker(wallets[-1], schedule, monkeypatch) + taker = create_taker(wallet_services[-1], schedule, monkeypatch) active_orders, maker_data = init_coinjoin(taker, makers, orderbook, cj_amount) diff --git a/jmclient/test/test_maker.py b/jmclient/test/test_maker.py index b86d1e2..268f461 100644 --- a/jmclient/test/test_maker.py +++ b/jmclient/test/test_maker.py @@ -6,7 +6,7 @@ from builtins import * # noqa: F401 import jmbitcoin as btc from jmclient import Maker, get_p2sh_vbyte, get_p2pk_vbyte, \ - load_program_config, jm_single + load_program_config, jm_single, WalletService import jmclient from commontest import DummyBlockchainInterface from test_taker import DummyWallet @@ -116,7 +116,7 @@ def test_verify_unsigned_tx_sw_valid(setup_env_nodeps): p2pkh_gen = address_p2pkh_generator() wallet = DummyWallet() - maker = OfflineMaker(wallet) + maker = OfflineMaker(WalletService(wallet)) cj_addr, cj_script = next(p2sh_gen) changeaddr, cj_change_script = next(p2sh_gen) @@ -149,7 +149,7 @@ def test_verify_unsigned_tx_nonsw_valid(setup_env_nodeps): p2pkh_gen = address_p2pkh_generator() wallet = DummyWallet() - maker = OfflineMaker(wallet) + maker = OfflineMaker(WalletService(wallet)) cj_addr, cj_script = next(p2pkh_gen) changeaddr, cj_change_script = next(p2pkh_gen) diff --git a/jmclient/test/test_payjoin.py b/jmclient/test/test_payjoin.py index e068666..267c13d 100644 --- a/jmclient/test/test_payjoin.py +++ b/jmclient/test/test_payjoin.py @@ -12,17 +12,17 @@ import pytest from twisted.internet import reactor from jmbase import get_log from jmclient import cryptoengine -from jmclient import (load_program_config, jm_single, sync_wallet, +from jmclient import (load_program_config, jm_single, P2EPMaker, P2EPTaker, LegacyWallet, SegwitLegacyWallet, SegwitWallet) from commontest import make_wallets -from test_coinjoin import make_wallets_to_list, sync_wallets, create_orderbook +from test_coinjoin import make_wallets_to_list, create_orderbook, sync_wallets testdir = os.path.dirname(os.path.realpath(__file__)) log = get_log() -def create_taker(wallet, schedule, monkeypatch): - taker = P2EPTaker("fakemaker", wallet, schedule, +def create_taker(wallet_service, schedule, monkeypatch): + taker = P2EPTaker("fakemaker", wallet_service, schedule, callbacks=(None, None, None)) return taker @@ -32,13 +32,13 @@ def dummy_user_check(message): log.info(message) return True -def getbals(wallet, mixdepth): +def getbals(wallet_service, mixdepth): """ Retrieves balances for a mixdepth and the 'next' """ - bbm = wallet.get_balance_by_mixdepth() - return (bbm[mixdepth], bbm[(mixdepth + 1) % (wallet.mixdepth + 1)]) + bbm = wallet_service.get_balance_by_mixdepth() + return (bbm[mixdepth], bbm[(mixdepth + 1) % (wallet_service.mixdepth + 1)]) -def final_checks(wallets, amount, txfee, tsb, msb, source_mixdepth=0): +def final_checks(wallet_services, amount, txfee, tsb, msb, source_mixdepth=0): """We use this to check that the wallet contents are as we've expected according to the test case. amount is the payment amount going from taker to maker. @@ -48,10 +48,9 @@ def final_checks(wallets, amount, txfee, tsb, msb, source_mixdepth=0): of two entries, source and destination mixdepth respectively. """ jm_single().bc_interface.tickchain() - for wallet in wallets: - sync_wallet(wallet) - takerbals = getbals(wallets[1], source_mixdepth) - makerbals = getbals(wallets[0], source_mixdepth) + sync_wallets(wallet_services) + takerbals = getbals(wallet_services[1], source_mixdepth) + makerbals = getbals(wallet_services[0], source_mixdepth) # is the payment received? maker_newcoin_amt = makerbals[1] - msb[1] if not maker_newcoin_amt >= amount: @@ -97,23 +96,23 @@ def test_simple_payjoin(monkeypatch, tmpdir, setup_cj, wallet_cls, def raise_exit(i): raise Exception("sys.exit called") monkeypatch.setattr(sys, 'exit', raise_exit) - wallets = [] - wallets.append(make_wallets_to_list(make_wallets( + wallet_services = [] + wallet_services.append(make_wallets_to_list(make_wallets( 1, wallet_structures=[wallet_structures[0]], mean_amt=mean_amt, wallet_cls=wallet_cls[0]))[0]) - wallets.append(make_wallets_to_list(make_wallets( + wallet_services.append(make_wallets_to_list(make_wallets( 1, wallet_structures=[wallet_structures[1]], mean_amt=mean_amt, wallet_cls=wallet_cls[1]))[0]) jm_single().bc_interface.tickchain() - sync_wallets(wallets) + sync_wallets(wallet_services) # For accounting purposes, record the balances # at the start. - msb = getbals(wallets[0], 0) - tsb = getbals(wallets[1], 0) + msb = getbals(wallet_services[0], 0) + tsb = getbals(wallet_services[1], 0) cj_amount = int(1.1 * 10**8) - maker = P2EPMaker(wallets[0], 0, cj_amount) + maker = P2EPMaker(wallet_services[0], 0, cj_amount) destaddr = maker.destination_addr monkeypatch.setattr(maker, 'user_check', dummy_user_check) # TODO use this to sanity check behaviour @@ -123,7 +122,7 @@ def test_simple_payjoin(monkeypatch, tmpdir, setup_cj, wallet_cls, # mixdepth, amount, counterparties, dest_addr, waittime; # in payjoin we only pay attention to the first two entries. schedule = [(0, cj_amount, 1, destaddr, 0)] - taker = create_taker(wallets[-1], schedule, monkeypatch) + taker = create_taker(wallet_services[-1], schedule, monkeypatch) monkeypatch.setattr(taker, 'user_check', dummy_user_check) init_data = taker.initialize(orderbook) # the P2EPTaker.initialize() returns: @@ -147,7 +146,7 @@ def test_simple_payjoin(monkeypatch, tmpdir, setup_cj, wallet_cls, assert False # Although the above OK is proof that a transaction went through, # it doesn't prove it was a good transaction! Here do balance checks: - assert final_checks(wallets, cj_amount, taker.total_txfee, tsb, msb) + assert final_checks(wallet_services, cj_amount, taker.total_txfee, tsb, msb) @pytest.fixture(scope='module') def setup_cj(): diff --git a/jmclient/test/test_podle.py b/jmclient/test/test_podle.py index f188972..eb70eb6 100644 --- a/jmclient/test/test_podle.py +++ b/jmclient/test/test_podle.py @@ -196,14 +196,14 @@ def test_podle_error_string(setup_podle): ('fakepriv2', 'fakeutxo2')] to = ['tooold1', 'tooold2'] ts = ['toosmall1', 'toosmall2'] - wallet = make_wallets(1, [[1, 0, 0, 0, 0]])[0]['wallet'] + wallet_service = make_wallets(1, [[1, 0, 0, 0, 0]])[0]['wallet'] cjamt = 100 tua = "3" tuamtper = "20" errmgsheader, errmsg = generate_podle_error_string(priv_utxo_pairs, to, ts, - wallet, + wallet_service, cjamt, tua, tuamtper) @@ -212,7 +212,7 @@ def test_podle_error_string(setup_podle): y = [x[1] for x in priv_utxo_pairs] assert all([errmsg.find(x) != -1 for x in to + ts + y]) #ensure OK with nothing - errmgsheader, errmsg = generate_podle_error_string([], [], [], wallet, + errmgsheader, errmsg = generate_podle_error_string([], [], [], wallet_service, cjamt, tua, tuamtper) @pytest.fixture(scope="module") diff --git a/jmclient/test/test_storage.py b/jmclient/test/test_storage.py index 7c2d63c..313e277 100644 --- a/jmclient/test/test_storage.py +++ b/jmclient/test/test_storage.py @@ -84,7 +84,6 @@ def test_storage_invalid(): MockStorage(b'garbagefile', __file__, b'password') pytest.fail("Non-wallet file, encrypted") - def test_storage_readonly(): s = MockStorage(None, 'nonexistant', b'password', create=True) s = MockStorage(s.file_data, __file__, b'password', read_only=True) diff --git a/jmclient/test/test_taker.py b/jmclient/test/test_taker.py index d2bbe2c..a9707cb 100644 --- a/jmclient/test/test_taker.py +++ b/jmclient/test/test_taker.py @@ -15,7 +15,7 @@ import struct from base64 import b64encode from jmclient import load_program_config, jm_single, set_commitment_file,\ get_commitment_file, SegwitLegacyWallet, Taker, VolatileStorage,\ - get_network + get_network, WalletService from taker_test_data import t_utxos_by_mixdepth, t_orderbook,\ t_maker_response, t_chosen_orders, t_dummy_ext @@ -35,14 +35,15 @@ class DummyWallet(SegwitLegacyWallet): txid, index = txid.split(':') path = (b'dummy', md, i) self._utxos.add_utxo(binascii.unhexlify(txid), int(index), - path, data['value'], md) + path, data['value'], md, 1) script = self._ENGINE.address_to_script(data['address']) self._script_map[script] = path - def get_utxos_by_mixdepth(self, verbose=True): + def get_utxos_by_mixdepth(self, verbose=True, includeheight=False): return t_utxos_by_mixdepth - def get_utxos_by_mixdepth_(self, verbose=True): + def get_utxos_by_mixdepth_(self, verbose=True, include_disabled=False, + includeheight=False): utxos = self.get_utxos_by_mixdepth(verbose) utxos_conv = {} @@ -59,7 +60,8 @@ class DummyWallet(SegwitLegacyWallet): return utxos_conv - def select_utxos(self, mixdepth, amount): + def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None, + maxheight=None): if amount > self.get_balance_by_mixdepth()[mixdepth]: raise Exception("Not enough funds") return t_utxos_by_mixdepth[mixdepth] @@ -126,10 +128,12 @@ def get_taker(schedule=None, schedule_len=0, on_finished=None, print("Using schedule: " + str(schedule)) on_finished_callback = on_finished if on_finished else taker_finished filter_orders_callback = filter_orders if filter_orders else dummy_filter_orderbook - return Taker(DummyWallet(), schedule, + taker = Taker(WalletService(DummyWallet()), schedule, callbacks=[filter_orders_callback, None, on_finished_callback]) + taker.wallet_service.current_blockheight = 10**6 + return taker -def test_filter_rejection(createcmtdata): +def test_filter_rejection(setup_taker): def filter_orders_reject(orders_feesl, cjamount): print("calling filter orders rejection") return False @@ -149,7 +153,7 @@ def test_filter_rejection(createcmtdata): (True, False), (False, True), ]) -def test_make_commitment(createcmtdata, failquery, external): +def test_make_commitment(setup_taker, failquery, external): def clean_up(): jm_single().config.set("POLICY", "taker_utxo_age", old_taker_utxo_age) jm_single().config.set("POLICY", "taker_utxo_amtpercent", old_taker_utxo_amtpercent) @@ -175,7 +179,7 @@ def test_make_commitment(createcmtdata, failquery, external): taker.make_commitment() clean_up() -def test_not_found_maker_utxos(createcmtdata): +def test_not_found_maker_utxos(setup_taker): taker = get_taker([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)]) orderbook = copy.deepcopy(t_orderbook) res = taker.initialize(orderbook) @@ -187,7 +191,7 @@ def test_not_found_maker_utxos(createcmtdata): assert res[1] == "Not enough counterparties responded to fill, giving up" jm_single().bc_interface.setQUSFail(False) -def test_auth_pub_not_found(createcmtdata): +def test_auth_pub_not_found(setup_taker): taker = get_taker([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)]) orderbook = copy.deepcopy(t_orderbook) res = taker.initialize(orderbook) @@ -239,7 +243,7 @@ def test_auth_pub_not_found(createcmtdata): ([(0, 0, 5, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)], False, False, 2, False, ["J659UPUSLLjHJpaB", "J65z23xdjxJjC7er", 0], None), #test inadequate for sweep ]) -def test_taker_init(createcmtdata, schedule, highfee, toomuchcoins, minmakers, +def test_taker_init(setup_taker, schedule, highfee, toomuchcoins, minmakers, notauthed, ignored, nocommit): #these tests do not trigger utxo_retries oldtakerutxoretries = jm_single().config.get("POLICY", "taker_utxo_retries") @@ -349,7 +353,7 @@ def test_taker_init(createcmtdata, schedule, highfee, toomuchcoins, minmakers, taker.prepare_my_bitcoin_data() with pytest.raises(NotImplementedError) as e_info: a = taker.coinjoin_address() - taker.wallet.inject_addr_get_failure = True + taker.wallet_service.wallet.inject_addr_get_failure = True taker.my_cj_addr = "dummy" assert not taker.prepare_my_bitcoin_data() #clean up @@ -365,6 +369,8 @@ def test_unconfirm_confirm(schedule_len): and merely update schedule index for confirm (useful for schedules/tumbles). This tests that the on_finished callback correctly reports the fromtx variable as "False" once the schedule is complete. + The exception to the above is that the txd passed in must match + self.latest_tx, so we use a dummy value here for that. """ test_unconfirm_confirm.txflag = True def finished_for_confirms(res, fromtx=False, waittime=0, txdetails=None): @@ -372,13 +378,14 @@ def test_unconfirm_confirm(schedule_len): test_unconfirm_confirm.txflag = fromtx taker = get_taker(schedule_len=schedule_len, on_finished=finished_for_confirms) - taker.unconfirm_callback("a", "b") + taker.latest_tx = {"outs": "blah"} + taker.unconfirm_callback({"ins": "foo", "outs": "blah"}, "b") for i in range(schedule_len-1): taker.schedule_index += 1 - fromtx = taker.confirm_callback("a", "b", 1) + fromtx = taker.confirm_callback({"ins": "foo", "outs": "blah"}, "b", 1) assert test_unconfirm_confirm.txflag taker.schedule_index += 1 - fromtx = taker.confirm_callback("a", "b", 1) + fromtx = taker.confirm_callback({"ins": "foo", "outs": "blah"}, "b", 1) assert not test_unconfirm_confirm.txflag @pytest.mark.parametrize( @@ -387,7 +394,7 @@ def test_unconfirm_confirm(schedule_len): ("mrcNu71ztWjAQA6ww9kHiW3zBWSQidHXTQ", [(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)]) ]) -def test_on_sig(createcmtdata, dummyaddr, schedule): +def test_on_sig(setup_taker, dummyaddr, schedule): #plan: create a new transaction with known inputs and dummy outputs; #then, create a signature with various inputs, pass in in b64 to on_sig. #in order for it to verify, the DummyBlockchainInterface will have to @@ -472,7 +479,12 @@ def test_auth_counterparty(schedule): assert not taker.auth_counterparty(sig_tweaked, auth_pub, maker_pub) @pytest.fixture(scope="module") -def createcmtdata(request): +def setup_taker(request): + def clean(): + from twisted.internet import reactor + for dc in reactor.getDelayedCalls(): + dc.cancel() + request.addfinalizer(clean) def cmtdatateardown(): shutil.rmtree("cmtdata") request.addfinalizer(cmtdatateardown) diff --git a/jmclient/test/test_tx_creation.py b/jmclient/test/test_tx_creation.py index 76d7736..91cc0a7 100644 --- a/jmclient/test/test_tx_creation.py +++ b/jmclient/test/test_tx_creation.py @@ -13,7 +13,7 @@ from commontest import make_wallets, make_sign_and_push import jmbitcoin as bitcoin import pytest from jmbase import get_log -from jmclient import load_program_config, jm_single, sync_wallet,\ +from jmclient import load_program_config, jm_single,\ get_p2pk_vbyte log = get_log() @@ -36,14 +36,14 @@ def test_create_p2sh_output_tx(setup_tx_creation, nw, wallet_structures, mean_amt, sdev_amt, amount, pubs, k): wallets = make_wallets(nw, wallet_structures, mean_amt, sdev_amt) for w in wallets.values(): - sync_wallet(w['wallet'], fast=True) + w['wallet'].sync_wallet(fast=True) for k, w in enumerate(wallets.values()): - wallet = w['wallet'] - ins_full = wallet.select_utxos(0, amount) + wallet_service = w['wallet'] + ins_full = wallet_service.select_utxos(0, amount) script = bitcoin.mk_multisig_script(pubs, k) output_addr = bitcoin.script_to_address(script, vbyte=196) txid = make_sign_and_push(ins_full, - wallet, + wallet_service, amount, output_addr=output_addr) assert txid @@ -81,16 +81,16 @@ def test_all_same_priv(setup_tx_creation): #recipient priv = "aa"*32 + "01" addr = bitcoin.privkey_to_address(priv, magicbyte=get_p2pk_vbyte()) - wallet = make_wallets(1, [[1,0,0,0,0]], 1)[0]['wallet'] + wallet_service = make_wallets(1, [[1,0,0,0,0]], 1)[0]['wallet'] #make another utxo on the same address - addrinwallet = wallet.get_addr(0,0,0) + addrinwallet = wallet_service.get_addr(0,0,0) jm_single().bc_interface.grab_coins(addrinwallet, 1) - sync_wallet(wallet, fast=True) - insfull = wallet.select_utxos(0, 110000000) + wallet_service.sync_wallet(fast=True) + insfull = wallet_service.select_utxos(0, 110000000) outs = [{"address": addr, "value": 1000000}] ins = list(insfull.keys()) tx = bitcoin.mktx(ins, outs) - tx = bitcoin.signall(tx, wallet.get_key_from_addr(addrinwallet)) + tx = bitcoin.signall(tx, wallet_service.get_key_from_addr(addrinwallet)) @pytest.mark.parametrize( "signall", @@ -101,9 +101,9 @@ def test_all_same_priv(setup_tx_creation): def test_verify_tx_input(setup_tx_creation, signall): priv = "aa"*32 + "01" addr = bitcoin.privkey_to_address(priv, magicbyte=get_p2pk_vbyte()) - wallet = make_wallets(1, [[2,0,0,0,0]], 1)[0]['wallet'] - sync_wallet(wallet, fast=True) - insfull = wallet.select_utxos(0, 110000000) + wallet_service = make_wallets(1, [[2,0,0,0,0]], 1)[0]['wallet'] + wallet_service.sync_wallet(fast=True) + insfull = wallet_service.select_utxos(0, 110000000) print(insfull) outs = [{"address": addr, "value": 1000000}] ins = list(insfull.keys()) @@ -115,14 +115,14 @@ def test_verify_tx_input(setup_tx_creation, signall): for index, ins in enumerate(desertx['ins']): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) ad = insfull[utxo]['address'] - priv = wallet.get_key_from_addr(ad) + priv = wallet_service.get_key_from_addr(ad) privdict[utxo] = priv tx = bitcoin.signall(tx, privdict) else: for index, ins in enumerate(desertx['ins']): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) ad = insfull[utxo]['address'] - priv = wallet.get_key_from_addr(ad) + priv = wallet_service.get_key_from_addr(ad) tx = bitcoin.sign(tx, index, priv) desertx2 = bitcoin.deserialize(tx) print(desertx2) @@ -143,28 +143,28 @@ def test_absurd_fees(setup_tx_creation): """ jm_single().bc_interface.absurd_fees = True #pay into it - wallet = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet'] - sync_wallet(wallet, fast=True) + wallet_service = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet'] + wallet_service.sync_wallet(fast=True) amount = 350000000 - ins_full = wallet.select_utxos(0, amount) + ins_full = wallet_service.select_utxos(0, amount) with pytest.raises(ValueError) as e_info: - txid = make_sign_and_push(ins_full, wallet, amount, estimate_fee=True) + txid = make_sign_and_push(ins_full, wallet_service, amount, estimate_fee=True) def test_create_sighash_txs(setup_tx_creation): #non-standard hash codes: for sighash in [bitcoin.SIGHASH_ANYONECANPAY + bitcoin.SIGHASH_SINGLE, bitcoin.SIGHASH_NONE, bitcoin.SIGHASH_SINGLE]: - wallet = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet'] - sync_wallet(wallet, fast=True) + wallet_service = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet'] + wallet_service.sync_wallet(fast=True) amount = 350000000 - ins_full = wallet.select_utxos(0, amount) + ins_full = wallet_service.select_utxos(0, amount) print("using hashcode: " + str(sighash)) - txid = make_sign_and_push(ins_full, wallet, amount, hashcode=sighash) + txid = make_sign_and_push(ins_full, wallet_service, amount, hashcode=sighash) assert txid #trigger insufficient funds with pytest.raises(Exception) as e_info: - fake_utxos = wallet.select_utxos(4, 1000000000) + fake_utxos = wallet_service.select_utxos(4, 1000000000) def test_spend_p2sh_utxos(setup_tx_creation): @@ -174,11 +174,11 @@ def test_spend_p2sh_utxos(setup_tx_creation): script = bitcoin.mk_multisig_script(pubs, 2) msig_addr = bitcoin.p2sh_scriptaddr(script, magicbyte=196) #pay into it - wallet = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet'] - sync_wallet(wallet, fast=True) + wallet_service = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet'] + wallet_service.sync_wallet(fast=True) amount = 350000000 - ins_full = wallet.select_utxos(0, amount) - txid = make_sign_and_push(ins_full, wallet, amount, output_addr=msig_addr) + ins_full = wallet_service.select_utxos(0, amount) + txid = make_sign_and_push(ins_full, wallet_service, amount, output_addr=msig_addr) assert txid #wait for mining time.sleep(1) @@ -186,7 +186,7 @@ def test_spend_p2sh_utxos(setup_tx_creation): msig_in = txid + ":0" ins = [msig_in] #random output address and change addr - output_addr = wallet.get_new_addr(1, 1) + output_addr = wallet_service.get_internal_addr(1) amount2 = amount - 50000 outs = [{'value': amount2, 'address': output_addr}] tx = bitcoin.mktx(ins, outs) @@ -205,19 +205,19 @@ def test_spend_p2wpkh(setup_tx_creation): scriptPubKeys = [bitcoin.pubkey_to_p2wpkh_script(pub) for pub in pubs] addresses = [bitcoin.pubkey_to_p2wpkh_address(pub) for pub in pubs] #pay into it - wallet = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet'] - sync_wallet(wallet, fast=True) + wallet_service = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet'] + wallet_service.sync_wallet(fast=True) amount = 35000000 p2wpkh_ins = [] for addr in addresses: - ins_full = wallet.select_utxos(0, amount) - txid = make_sign_and_push(ins_full, wallet, amount, output_addr=addr) + ins_full = wallet_service.select_utxos(0, amount) + txid = make_sign_and_push(ins_full, wallet_service, amount, output_addr=addr) assert txid p2wpkh_ins.append(txid + ":0") #wait for mining time.sleep(1) #random output address - output_addr = wallet.get_new_addr(1, 1) + output_addr = wallet_service.get_internal_addr(1) amount2 = amount*3 - 50000 outs = [{'value': amount2, 'address': output_addr}] tx = bitcoin.mktx(p2wpkh_ins, outs) @@ -249,19 +249,19 @@ def test_spend_p2wsh(setup_tx_creation): scriptPubKeys = [bitcoin.pubkeys_to_p2wsh_script(pubs[i:i+2]) for i in [0, 2]] addresses = [bitcoin.pubkeys_to_p2wsh_address(pubs[i:i+2]) for i in [0, 2]] #pay into it - wallet = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet'] - sync_wallet(wallet, fast=True) + wallet_service = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet'] + wallet_service.sync_wallet(fast=True) amount = 35000000 p2wsh_ins = [] for addr in addresses: - ins_full = wallet.select_utxos(0, amount) - txid = make_sign_and_push(ins_full, wallet, amount, output_addr=addr) + ins_full = wallet_service.select_utxos(0, amount) + txid = make_sign_and_push(ins_full, wallet_service, amount, output_addr=addr) assert txid p2wsh_ins.append(txid + ":0") #wait for mining time.sleep(1) #random output address and change addr - output_addr = wallet.get_new_addr(1, 1) + output_addr = wallet_service.get_internal_addr(1) amount2 = amount*2 - 50000 outs = [{'value': amount2, 'address': output_addr}] tx = bitcoin.mktx(p2wsh_ins, outs) diff --git a/jmclient/test/test_utxomanager.py b/jmclient/test/test_utxomanager.py index b9bdecf..3a0ffe6 100644 --- a/jmclient/test/test_utxomanager.py +++ b/jmclient/test/test_utxomanager.py @@ -32,11 +32,11 @@ def test_utxomanager_persist(setup_env_nodeps): mixdepth = 0 value = 500 - um.add_utxo(txid, index, path, value, mixdepth) - um.add_utxo(txid, index+1, path, value, mixdepth+1) + um.add_utxo(txid, index, path, value, mixdepth, 1) + um.add_utxo(txid, index+1, path, value, mixdepth+1, 2) # the third utxo will be disabled and we'll check if # the disablement persists in the storage across UM instances - um.add_utxo(txid, index+2, path, value, mixdepth+1) + um.add_utxo(txid, index+2, path, value, mixdepth+1, 3) um.disable_utxo(txid, index+2) um.save() @@ -103,20 +103,24 @@ def test_utxomanager_select(setup_env_nodeps): mixdepth = 0 value = 500 - um.add_utxo(txid, index, path, value, mixdepth) + um.add_utxo(txid, index, path, value, mixdepth, 100) assert len(um.select_utxos(mixdepth, value)) == 1 assert len(um.select_utxos(mixdepth+1, value)) == 0 - um.add_utxo(txid, index+1, path, value, mixdepth) + um.add_utxo(txid, index+1, path, value, mixdepth, None) assert len(um.select_utxos(mixdepth, value)) == 2 # ensure that added utxos that are disabled do not # get used by the selector - um.add_utxo(txid, index+2, path, value, mixdepth) + um.add_utxo(txid, index+2, path, value, mixdepth, 101) um.disable_utxo(txid, index+2) assert len(um.select_utxos(mixdepth, value)) == 2 + # ensure that unconfirmed coins are not selected if + # dis-requested: + assert len(um.select_utxos(mixdepth, value, maxheight=105)) == 1 + @pytest.fixture def setup_env_nodeps(monkeypatch): diff --git a/jmclient/test/test_wallet.py b/jmclient/test/test_wallet.py index ee41bdf..5520f83 100644 --- a/jmclient/test/test_wallet.py +++ b/jmclient/test/test_wallet.py @@ -14,7 +14,7 @@ from jmbase import get_log from jmclient import load_program_config, jm_single, \ SegwitLegacyWallet,BIP32Wallet, BIP49Wallet, LegacyWallet,\ VolatileStorage, get_network, cryptoengine, WalletError,\ - SegwitWallet + SegwitWallet, WalletService from test_blockchaininterface import sync_test_wallet testdir = os.path.dirname(os.path.realpath(__file__)) @@ -52,7 +52,7 @@ def fund_wallet_addr(wallet, addr, value_btc=1): txin_id = jm_single().bc_interface.grab_coins(addr, value_btc) txinfo = jm_single().bc_interface.rpc('gettransaction', [txin_id]) txin = btc.deserialize(unhexlify(txinfo['hex'])) - utxo_in = wallet.add_new_utxos_(txin, unhexlify(txin_id)) + utxo_in = wallet.add_new_utxos_(txin, unhexlify(txin_id), 1) assert len(utxo_in) == 1 return list(utxo_in.keys())[0] @@ -407,7 +407,7 @@ def test_add_new_utxos(setup_wallet): for s in tx_scripts])) binarize_tx(tx) txid = b'\x01' * 32 - added = wallet.add_new_utxos_(tx, txid) + added = wallet.add_new_utxos_(tx, txid, 1) assert len(added) == len(scripts) added_scripts = {x['script'] for x in added.values()} @@ -429,7 +429,7 @@ def test_remove_old_utxos(setup_wallet): for i in range(3): txin = jm_single().bc_interface.grab_coins( wallet.get_internal_addr(1), 1) - wallet.add_utxo(unhexlify(txin), 0, wallet.get_script(1, 1, i), 10**8) + wallet.add_utxo(unhexlify(txin), 0, wallet.get_script(1, 1, i), 10**8, 1) inputs = wallet.select_utxos_(0, 10**8) inputs.update(wallet.select_utxos_(1, 2 * 10**8)) @@ -466,7 +466,7 @@ def test_initialize_twice(setup_wallet): def test_is_known(setup_wallet): wallet = get_populated_wallet(num=0) script = wallet.get_new_script(1, True) - addr = wallet.get_new_addr(2, False) + addr = wallet.get_external_addr(2) assert wallet.is_known_script(script) assert wallet.is_known_addr(addr) @@ -651,7 +651,7 @@ def test_wallet_mixdepth_decrease(setup_wallet): VolatileStorage(data=storage_data), mixdepth=new_mixdepth) assert new_wallet.max_mixdepth == max_mixdepth assert new_wallet.mixdepth == new_mixdepth - sync_test_wallet(True, new_wallet) + sync_test_wallet(True, WalletService(new_wallet)) assert max_mixdepth not in new_wallet.get_balance_by_mixdepth() assert max_mixdepth not in new_wallet.get_utxos_by_mixdepth() diff --git a/jmclient/test/test_wallets.py b/jmclient/test/test_wallets.py index f0a81bd..1a2efbf 100644 --- a/jmclient/test/test_wallets.py +++ b/jmclient/test/test_wallets.py @@ -13,38 +13,38 @@ import json import pytest from jmbase import get_log from jmclient import ( - load_program_config, jm_single, sync_wallet, + load_program_config, jm_single, estimate_tx_fee, BitcoinCoreInterface, Mnemonic) from taker_test_data import t_raw_signed_tx testdir = os.path.dirname(os.path.realpath(__file__)) log = get_log() -def do_tx(wallet, amount): - ins_full = wallet.select_utxos(0, amount) - cj_addr = wallet.get_internal_addr(1) - change_addr = wallet.get_internal_addr(0) - wallet.update_cache_index() +def do_tx(wallet_service, amount): + ins_full = wallet_service.select_utxos(0, amount) + cj_addr = wallet_service.get_internal_addr(1) + change_addr = wallet_service.get_internal_addr(0) + wallet_service.save_wallet() txid = make_sign_and_push(ins_full, - wallet, + wallet_service, amount, output_addr=cj_addr, change_addr=change_addr, estimate_fee=True) assert txid time.sleep(2) #blocks - jm_single().bc_interface.sync_unspent(wallet) + wallet_service.sync_unspent() return txid def test_query_utxo_set(setup_wallets): load_program_config() jm_single().bc_interface.tick_forward_chain_interval = 1 - wallet = create_wallet_for_sync([2, 3, 0, 0, 0], + wallet_service = create_wallet_for_sync([2, 3, 0, 0, 0], ["wallet4utxo.json", "4utxo", [2, 3]]) - sync_wallet(wallet, fast=True) - txid = do_tx(wallet, 90000000) - txid2 = do_tx(wallet, 20000000) + wallet_service.sync_wallet(fast=True) + txid = do_tx(wallet_service, 90000000) + txid2 = do_tx(wallet_service, 20000000) print("Got txs: ", txid, txid2) res1 = jm_single().bc_interface.query_utxo_set(txid + ":0", includeunconf=True) res2 = jm_single().bc_interface.query_utxo_set( diff --git a/jmclient/test/test_yieldgenerator.py b/jmclient/test/test_yieldgenerator.py index 8b75b0e..52cf568 100644 --- a/jmclient/test/test_yieldgenerator.py +++ b/jmclient/test/test_yieldgenerator.py @@ -5,7 +5,8 @@ from builtins import * # noqa: F401 import unittest from jmclient import load_program_config, jm_single,\ - SegwitLegacyWallet, VolatileStorage, YieldGeneratorBasic, get_network + SegwitLegacyWallet, VolatileStorage, YieldGeneratorBasic, \ + get_network, WalletService class CustomUtxoWallet(SegwitLegacyWallet): @@ -37,7 +38,7 @@ class CustomUtxoWallet(SegwitLegacyWallet): # script, and make it fit the required length (32 bytes). txid = tx['outs'][0]['script'] + b'x' * 32 txid = txid[:32] - self.add_new_utxos_(tx, txid) + self.add_new_utxos_(tx, txid, 1) def assert_utxos_from_mixdepth(self, utxos, expected): """Asserts that the list of UTXOs (as returned from UTXO selection @@ -56,7 +57,7 @@ def create_yg_basic(balances, txfee=0, cjfee_a=0, cjfee_r=0, wallet = CustomUtxoWallet(balances) offerconfig = (txfee, cjfee_a, cjfee_r, ordertype, minsize) - yg = YieldGeneratorBasic(wallet, offerconfig) + yg = YieldGeneratorBasic(WalletService(wallet), offerconfig) # We don't need any of the logic from Maker, including the waiting # loop. Just stop it, so that it does not linger around and create @@ -140,9 +141,9 @@ class OidToOrderTests(unittest.TestCase): yg = create_yg_basic([10, 1000, 2000]) utxos, cj_addr, change_addr = self.call_oid_to_order(yg, 500) self.assertEqual(len(utxos), 1) - yg.wallet.assert_utxos_from_mixdepth(utxos, 1) - self.assertEqual(yg.wallet.get_addr_mixdepth(cj_addr), 2) - self.assertEqual(yg.wallet.get_addr_mixdepth(change_addr), 1) + yg.wallet_service.wallet.assert_utxos_from_mixdepth(utxos, 1) + self.assertEqual(yg.wallet_service.wallet.get_addr_mixdepth(cj_addr), 2) + self.assertEqual(yg.wallet_service.wallet.get_addr_mixdepth(change_addr), 1) def test_not_enough_balance_with_dust_threshold(self): # 410 is exactly the size of the change output. So it will be @@ -158,12 +159,12 @@ class OidToOrderTests(unittest.TestCase): # over the threshold. jm_single().DUST_THRESHOLD = 410 yg = create_yg_basic([10, 1000, 10], txfee=100, cjfee_a=10) - yg.wallet.add_utxo_at_mixdepth(1, 500) + yg.wallet_service.wallet.add_utxo_at_mixdepth(1, 500) utxos, cj_addr, change_addr = self.call_oid_to_order(yg, 500) self.assertEqual(len(utxos), 2) - yg.wallet.assert_utxos_from_mixdepth(utxos, 1) - self.assertEqual(yg.wallet.get_addr_mixdepth(cj_addr), 2) - self.assertEqual(yg.wallet.get_addr_mixdepth(change_addr), 1) + yg.wallet_service.wallet.assert_utxos_from_mixdepth(utxos, 1) + self.assertEqual(yg.wallet_service.wallet.get_addr_mixdepth(cj_addr), 2) + self.assertEqual(yg.wallet_service.wallet.get_addr_mixdepth(change_addr), 1) class OfferReannouncementTests(unittest.TestCase): @@ -171,7 +172,7 @@ class OfferReannouncementTests(unittest.TestCase): def call_on_tx_unconfirmed(self, yg): """Calls yg.on_tx_unconfirmed with fake arguments.""" - return yg.on_tx_unconfirmed({'cjaddr': 'addr'}, 'txid', []) + return yg.on_tx_unconfirmed({'cjaddr': 'addr'}, 'txid') def create_yg_and_offer(self, maxsize): """Constructs a fake yg instance that has an offer with the given diff --git a/scripts/add-utxo.py b/scripts/add-utxo.py index d61ad6f..aaafd3a 100644 --- a/scripts/add-utxo.py +++ b/scripts/add-utxo.py @@ -19,7 +19,7 @@ from optparse import OptionParser from jmbase import jmprint import jmbitcoin as btc from jmclient import load_program_config, jm_single, get_p2pk_vbyte,\ - open_wallet, sync_wallet, add_external_commitments, update_commitments,\ + open_wallet, WalletService, add_external_commitments, update_commitments,\ PoDLE, get_podle_commitments, get_utxo_info, validate_utxo_data, quit,\ get_wallet_path @@ -146,12 +146,12 @@ def main(): help='only validate the provided utxos (file or command line), not add', default=False ) - parser.add_option('--fast', + parser.add_option('--recoversync', action='store_true', - dest='fastsync', + dest='recoversync', default=False, - help=('choose to do fast wallet sync, only for Core and ' - 'only for previously synced wallet')) + help=('choose to do detailed wallet sync, ' + 'used for recovering on new Core instance.')) (options, args) = parser.parse_args() load_program_config() #TODO; sort out "commit file location" global so this script can @@ -179,16 +179,18 @@ def main(): if options.loadwallet: wallet_path = get_wallet_path(options.loadwallet, None) wallet = open_wallet(wallet_path, gap_limit=options.gaplimit) - while not jm_single().bc_interface.wallet_synced: - sync_wallet(wallet, fast=options.fastsync) + wallet_service = WalletService(wallet) + while True: + if wallet_service.sync_wallet(fast=not options.recoversync): + break # minor note: adding a utxo from an external wallet for commitments, we # default to not allowing disabled utxos to avoid a privacy leak, so the # user would have to explicitly enable. - for md, utxos in wallet.get_utxos_by_mixdepth_().items(): + for md, utxos in wallet_service.get_utxos_by_mixdepth(hexfmt=False).items(): for (txid, index), utxo in utxos.items(): txhex = binascii.hexlify(txid).decode('ascii') + ':' + str(index) - wif = wallet.get_wif_path(utxo['path']) + wif = wallet_service.get_wif_path(utxo['path']) utxo_data.append((txhex, wif)) elif options.in_file: diff --git a/scripts/cli_options.py b/scripts/cli_options.py index f60d9ea..8f0bb75 100644 --- a/scripts/cli_options.py +++ b/scripts/cli_options.py @@ -33,12 +33,12 @@ def add_common_options(parser): 'for the total transaction fee, default=dynamically estimated, note that this is adjusted ' 'based on the estimated fee calculated after tx construction, based on ' 'policy set in joinmarket.cfg.') - parser.add_option('--fast', + parser.add_option('--recoversync', action='store_true', - dest='fastsync', + dest='recoversync', default=False, - help=('choose to do fast wallet sync, only for Core and ' - 'only for previously synced wallet')) + help=('choose to do detailed wallet sync, ' + 'used for recovering on new Core instance.')) parser.add_option( '-x', '--max-cj-fee-abs', @@ -358,6 +358,18 @@ def get_tumbler_parser(): default=9, help= 'maximum amount of times to re-create a transaction before giving up, default 9') + # note that this is used slightly differently in tumbler from sendpayment, + # hence duplicated: + parser.add_option('-A', + '--amtmixdepths', + action='store', + type='int', + dest='amtmixdepths', + help='number of mixdepths ever used in wallet, ' + 'only to be used if mixdepths higher than ' + 'mixdepthsrc + number of mixdepths to tumble ' + 'have been used.', + default=-1) add_common_options(parser) return parser diff --git a/scripts/joinmarket-qt.py b/scripts/joinmarket-qt.py index 6c409fd..5b118d7 100644 --- a/scripts/joinmarket-qt.py +++ b/scripts/joinmarket-qt.py @@ -72,7 +72,7 @@ from jmclient import load_program_config, get_network,\ open_test_wallet_maybe, get_wallet_path, get_p2sh_vbyte, get_p2pk_vbyte,\ jm_single, validate_address, weighted_order_choose, Taker,\ JMClientProtocolFactory, start_reactor, get_schedule, schedule_to_text,\ - get_blockchain_interface_instance, direct_send,\ + get_blockchain_interface_instance, direct_send, WalletService,\ RegtestBitcoinCoreInterface, tumbler_taker_finished_update,\ get_tumble_log, restart_wait, tumbler_filter_orders_callback,\ wallet_generate_recover_bip39, wallet_display, get_utxos_enabled_disabled @@ -621,12 +621,23 @@ class SpendTab(QWidget): makercount = int(self.widgets[1][1].text()) mixdepth = int(self.widgets[2][1].text()) if makercount == 0: - txid = direct_send(w.wallet, amount, mixdepth, + txid = direct_send(w.wallet_service, amount, mixdepth, destaddr, accept_callback=self.checkDirectSend, info_callback=self.infoDirectSend) if not txid: self.giveUp() else: + # since direct_send() assumes a one-shot processing, it does + # not add a callback for confirmation, so that event could + # get lost; we do that here to ensure that the confirmation + # event is noticed: + def qt_directsend_callback(rtxd, rtxid, confs): + if rtxid == txid: + return True + return False + w.wallet_service.active_txids.append(txid) + w.wallet_service.register_callbacks([qt_directsend_callback], + txid, cb_type="confirmed") self.persistTxToHistory(destaddr, self.direct_send_amount, txid) self.cleanUp() return @@ -640,7 +651,7 @@ class SpendTab(QWidget): self.startJoin() def startJoin(self): - if not w.wallet: + if not w.wallet_service: JMQtMessageBox(self, "Cannot start without a loaded wallet.", mbtype="crit", title="Error") return @@ -654,7 +665,7 @@ class SpendTab(QWidget): check_offers_callback = None destaddrs = self.tumbler_destaddrs if self.tumbler_options else [] - self.taker = Taker(w.wallet, + self.taker = Taker(w.wallet_service, self.spendstate.loaded_schedule, order_chooser=weighted_order_choose, callbacks=[check_offers_callback, @@ -797,7 +808,7 @@ class SpendTab(QWidget): #TODO prob best to completely fold multiple and tumble to reduce #complexity/duplication if self.spendstate.typestate == 'multiple' and not self.tumbler_options: - self.taker.wallet.update_cache_index() + self.taker.wallet_service.save_wallet() return if fromtx: if res: @@ -811,12 +822,6 @@ class SpendTab(QWidget): self.nextTxTimer.start(int(waittime*60*1000)) #QtCore.QTimer.singleShot(int(self.taker_finished_waittime*60*1000), # self.startNextTransaction) - #see note above re multiple/tumble duplication - if self.spendstate.typestate == 'multiple' and \ - not self.tumbler_options: - txd, txid = txdetails - self.taker.wallet.remove_old_utxos(txd) - self.taker.wallet.add_new_utxos(txd, txid) else: if self.tumbler_options: w.statusBar().showMessage("Transaction failed, trying again...") @@ -915,7 +920,7 @@ class SpendTab(QWidget): if len(self.widgets[i][1].text()) == 0: JMQtMessageBox(self, errs[i - 1], mbtype='warn', title="Error") return False - if not w.wallet: + if not w.wallet_service: JMQtMessageBox(self, "There is no wallet loaded.", mbtype='warn', @@ -1037,13 +1042,13 @@ class CoinsTab(QWidget): self.cTW.addChild(m_item) self.cTW.show() - if not w.wallet: + if not w.wallet_service: show_blank() return utxos_enabled = {} utxos_disabled = {} for i in range(jm_single().config.getint("GUI", "max_mix_depth")): - utxos_e, utxos_d = get_utxos_enabled_disabled(w.wallet, i) + utxos_e, utxos_d = get_utxos_enabled_disabled(w.wallet_service, i) if utxos_e != {}: utxos_enabled[i] = utxos_e if utxos_d != {}: @@ -1070,7 +1075,7 @@ class CoinsTab(QWidget): # txid:index, btc, address t = btc.safe_hexlify(k[0])+":"+str(k[1]) s = "{0:.08f}".format(v['value']/1e8) - a = w.wallet.script_to_addr(v["script"]) + a = w.wallet_service.script_to_addr(v["script"]) item = QTreeWidgetItem([t, s, a]) item.setFont(0, QFont(MONOSPACE_FONT)) #if rows[i][forchange][j][3] != 'new': @@ -1080,7 +1085,7 @@ class CoinsTab(QWidget): def toggle_utxo_disable(self, txid, idx): txid_bytes = btc.safe_from_hex(txid) - w.wallet.toggle_disable_utxo(txid_bytes, idx) + w.wallet_service.toggle_disable_utxo(txid_bytes, idx) self.updateUtxos() def create_menu(self, position): @@ -1179,30 +1184,42 @@ class JMWalletTab(QWidget): if xpub_exists: menu.addAction("Copy extended pubkey to clipboard", lambda: app.clipboard().setText(xpub)) - menu.addAction("Resync wallet from blockchain", - lambda: w.resyncWallet()) #TODO add more items to context menu - menu.exec_(self.walletTree.viewport().mapToGlobal(position)) + if address_valid or xpub_exists: + menu.exec_(self.walletTree.viewport().mapToGlobal(position)) def openQRCodePopup(self, address): popup = BitcoinQRCodePopup(self, address) popup.show() def updateWalletInfo(self, walletinfo=None): + nm = jm_single().config.getint("GUI", "max_mix_depth") l = self.walletTree + + # before deleting, note whether items were expanded + esrs = [] + for i in range(l.topLevelItemCount()): + tli = l.invisibleRootItem().child(i) + # must check top and also the two subitems (branches): + expandedness = tuple( + x.isExpanded() for x in [tli, tli.child(0), tli.child(1)]) + esrs.append(expandedness) l.clear() if walletinfo: self.mainwindow = self.parent().parent().parent() rows, mbalances, xpubs, total_bal = walletinfo if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == "regtest": - self.wallet_name = self.mainwindow.wallet.seed + self.wallet_name = self.mainwindow.testwalletname else: - self.wallet_name = os.path.basename(self.mainwindow.wallet._storage.path) + self.wallet_name = os.path.basename( + self.mainwindow.wallet_service.get_storage_location()) + if total_bal is None: + total_bal = " (syncing..)" self.label1.setText("CURRENT WALLET: " + self.wallet_name + ', total balance: ' + total_bal) - self.walletTree.show() + l.show() - for i in range(jm_single().config.getint("GUI", "max_mix_depth")): + for i in range(nm): if walletinfo: mdbalance = mbalances[i] else: @@ -1210,6 +1227,10 @@ class JMWalletTab(QWidget): m_item = QTreeWidgetItem(["Mixdepth " + str(i) + " , balance: " + mdbalance, '', '', '', '']) l.addChild(m_item) + # if expansion states existed, reinstate them: + if len(esrs) == nm: + m_item.setExpanded(esrs[i][0]) + for forchange in [0, 1]: heading = "EXTERNAL" if forchange == 0 else "INTERNAL" if walletinfo and heading == "EXTERNAL": @@ -1217,8 +1238,11 @@ class JMWalletTab(QWidget): heading += heading_end seq_item = QTreeWidgetItem([heading, '', '', '', '']) m_item.addChild(seq_item) + # by default, external is expanded, but remember user choice: if not forchange: seq_item.setExpanded(True) + if len(esrs) == nm: + seq_item.setExpanded(esrs[i][forchange+1]) if not walletinfo: item = QTreeWidgetItem(['None', '', '', '']) seq_item.addChild(item) @@ -1238,7 +1262,14 @@ class JMMainWindow(QMainWindow): def __init__(self, reactor): super(JMMainWindow, self).__init__() - self.wallet = None + # the wallet service that encapsulates + # the wallet we will interact with + self.wallet_service = None + + # the monitoring loop that queries + # the walletservice to update the GUI + self.walletRefresh = None + self.reactor = reactor self.initUI() @@ -1294,7 +1325,7 @@ class JMMainWindow(QMainWindow): msgbox.setWindowTitle(appWindowTitle) label1 = QLabel() label1.setText( - "" + "" + "Read more about Joinmarket

" + "

".join( ["Joinmarket core software version: " + JM_CORE_VERSION, "JoinmarketQt version: " + JM_GUI_VERSION, @@ -1321,7 +1352,7 @@ class JMMainWindow(QMainWindow): msgbox.exec_() def exportPrivkeysJson(self): - if not self.wallet: + if not self.wallet_service: JMQtMessageBox(self, "No wallet loaded.", mbtype='crit', @@ -1351,7 +1382,7 @@ class JMMainWindow(QMainWindow): #option for anyone with gaplimit troubles, although #that is a complete mess for a user, mostly changing #the gaplimit in the Settings tab should address it. - rows = get_wallet_printout(self.wallet) + rows = get_wallet_printout(self.wallet_service) addresses = [] for forchange in rows[0]: for mixdepth in forchange: @@ -1365,7 +1396,7 @@ class JMMainWindow(QMainWindow): time.sleep(0.1) if done: break - priv = self.wallet.get_key_from_addr(addr) + priv = self.wallet_service.get_key_from_addr(addr) private_keys[addr] = btc.wif_compressed_privkey( priv, vbyte=get_p2pk_vbyte()) @@ -1519,7 +1550,7 @@ class JMMainWindow(QMainWindow): if firstarg: wallet_path = get_wallet_path(str(firstarg), None) try: - self.wallet = open_test_wallet_maybe(wallet_path, str(firstarg), + wallet = open_test_wallet_maybe(wallet_path, str(firstarg), None, ask_for_password=False, password=pwd.encode('utf-8') if pwd else None, gap_limit=jm_single().config.getint("GUI", "gaplimit")) except Exception as e: @@ -1528,58 +1559,40 @@ class JMMainWindow(QMainWindow): mbtype='warn', title="Error") return False - self.wallet.seed = str(firstarg) + # only used for GUI display on regtest: + self.testwalletname = wallet.seed = str(firstarg) if 'listunspent_args' not in jm_single().config.options('POLICY'): jm_single().config.set('POLICY', 'listunspent_args', '[0]') - assert self.wallet, "No wallet loaded" - reactor.callLater(0, self.syncWalletUpdate, True, restart_cb) + assert wallet, "No wallet loaded" + + # shut down any existing wallet service + # monitoring loops + if self.wallet_service is not None: + if self.wallet_service.isRunning(): + self.wallet_service.stopService() + if self.walletRefresh is not None: + self.walletRefresh.stop() + + self.wallet_service = WalletService(wallet) + self.wallet_service.add_restart_callback(restart_cb) + self.wallet_service.startService() + self.walletRefresh = task.LoopingCall(self.updateWalletInfo) + self.walletRefresh.start(5.0) + self.statusBar().showMessage("Reading wallet from blockchain ...") return True - def syncWalletUpdate(self, fast, restart_cb=None): - if restart_cb: - fast=False - #Special syncing condition for Electrum - iselectrum = jm_single().config.get("BLOCKCHAIN", - "blockchain_source") == "electrum-server" - if iselectrum: - jm_single().bc_interface.synctype = "with-script" - - jm_single().bc_interface.sync_wallet(self.wallet, fast=fast, - restart_cb=restart_cb) - - if iselectrum: - #sync_wallet only initialises, we must manually call its entry - #point here (because we can't use connectionMade as a trigger) - jm_single().bc_interface.sync_addresses(self.wallet) - self.wait_for_sync_loop = task.LoopingCall(self.updateWalletInfo) - self.wait_for_sync_loop.start(0.2) - else: - self.updateWalletInfo() - def updateWalletInfo(self): - if jm_single().config.get("BLOCKCHAIN", - "blockchain_source") == "electrum-server": - if not jm_single().bc_interface.wallet_synced: - return - self.wait_for_sync_loop.stop() t = self.centralWidget().widget(0) - if not self.wallet: #failure to sync in constructor means object is not created + if not self.wallet_service: #failure to sync in constructor means object is not created newstmsg = "Unable to sync wallet - see error in console." + elif not self.wallet_service.synced: + return else: - t.updateWalletInfo(get_wallet_printout(self.wallet)) + t.updateWalletInfo(get_wallet_printout(self.wallet_service)) newstmsg = "Wallet synced successfully." self.statusBar().showMessage(newstmsg) - def resyncWallet(self): - if not self.wallet: - JMQtMessageBox(self, - "No wallet loaded", - mbtype='warn', - title="Error") - return - self.loadWalletFromBlockchain() - def generateWallet(self): log.debug('generating wallet') if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == "regtest": @@ -1689,8 +1702,8 @@ class JMMainWindow(QMainWindow): self.loadWalletFromBlockchain(self.walletname, pwd=self.textpassword, restart_cb=restart_cb) -def get_wallet_printout(wallet): - """Given a joinmarket wallet, retrieve the list of +def get_wallet_printout(wallet_service): + """Given a WalletService object, retrieve the list of addresses and corresponding balances to be displayed. We retrieve a WalletView abstraction, and iterate over sub-objects to arrange the per-mixdepth and per-address lists. @@ -1701,7 +1714,7 @@ def get_wallet_printout(wallet): xpubs: [[xpubext, xpubint], ...] Bitcoin amounts returned are in btc, not satoshis """ - walletview = wallet_display(wallet, jm_single().config.getint("GUI", + walletview = wallet_display(wallet_service, jm_single().config.getint("GUI", "gaplimit"), False, serialized=False) rows = [] mbalances = [] @@ -1718,7 +1731,10 @@ def get_wallet_printout(wallet): entry.serialize_wallet_position(), entry.serialize_amounts(), entry.serialize_extra_data()]) - return (rows, mbalances, xpubs, walletview.get_fmt_balance()) + # in case the wallet is not yet synced, don't return an incorrect + # 0 balance, but signal incompleteness: + total_bal = walletview.get_fmt_balance() if wallet_service.synced else None + return (rows, mbalances, xpubs, total_bal) ################################ config_load_error = False diff --git a/scripts/receive-payjoin.py b/scripts/receive-payjoin.py index 9e90792..7eba1c6 100644 --- a/scripts/receive-payjoin.py +++ b/scripts/receive-payjoin.py @@ -20,12 +20,12 @@ def receive_payjoin_main(makerclass): parser.add_option('-g', '--gap-limit', action='store', type="int", dest='gaplimit', default=6, help='gap limit for wallet, default=6') - parser.add_option('--fast', + parser.add_option('--recoversync', action='store_true', - dest='fastsync', + dest='recoversync', default=False, - help=('choose to do fast wallet sync, only for Core and ' - 'only for previously synced wallet')) + help=('choose to do detailed wallet sync, ' + 'used for recovering on new Core instance.')) parser.add_option('-m', '--mixdepth', action='store', type='int', dest='mixdepth', default=0, help="mixdepth to source coins from") @@ -68,7 +68,7 @@ def receive_payjoin_main(makerclass): jm_single().bc_interface.synctype = "with-script" while not jm_single().bc_interface.wallet_synced: - sync_wallet(wallet, fast=options.fastsync) + sync_wallet(wallet, fast=not options.recoversync) maker = makerclass(wallet, options.mixdepth, receiving_amount) diff --git a/scripts/sendpayment.py b/scripts/sendpayment.py index b0b6630..7f37414 100644 --- a/scripts/sendpayment.py +++ b/scripts/sendpayment.py @@ -16,7 +16,7 @@ import pprint from jmclient import Taker, P2EPTaker, load_program_config, get_schedule,\ JMClientProtocolFactory, start_reactor, validate_address, jm_single,\ - sync_wallet, estimate_tx_fee, direct_send,\ + estimate_tx_fee, direct_send, WalletService,\ open_test_wallet_maybe, get_wallet_path from twisted.python.log import startLogging from jmbase.support import get_log, set_logging_level, jmprint @@ -128,19 +128,20 @@ def main(): wallet_path = get_wallet_path(wallet_name, None) wallet = open_test_wallet_maybe( wallet_path, wallet_name, max_mix_depth, gap_limit=options.gaplimit) + wallet_service = WalletService(wallet) + # in this script, we need the wallet synced before + # logic processing for some paths, so do it now: + while not wallet_service.synced: + wallet_service.sync_wallet(fast=not options.recoversync) + # the sync call here will now be a no-op: + wallet_service.startService() - if jm_single().config.get("BLOCKCHAIN", - "blockchain_source") == "electrum-server" and options.makercount != 0: - jm_single().bc_interface.synctype = "with-script" - #wallet sync will now only occur on reactor start if we're joining. - while not jm_single().bc_interface.wallet_synced: - sync_wallet(wallet, fast=options.fastsync) # From the estimated tx fees, check if the expected amount is a # significant value compared the the cj amount total_cj_amount = amount if total_cj_amount == 0: - total_cj_amount = wallet.get_balance_by_mixdepth()[options.mixdepth] + total_cj_amount = wallet_service.get_balance_by_mixdepth()[options.mixdepth] if total_cj_amount == 0: raise ValueError("No confirmed coins in the selected mixdepth. Quitting") exp_tx_fees_ratio = ((1 + options.makercount) * options.txfee) / total_cj_amount @@ -155,7 +156,7 @@ def main(): .format(exp_tx_fees_ratio)) if options.makercount == 0 and not options.p2ep: - direct_send(wallet, amount, mixdepth, destaddr, options.answeryes) + direct_send(wallet_service, amount, mixdepth, destaddr, options.answeryes) return if wallet.get_txtype() == 'p2pkh': @@ -193,8 +194,6 @@ def main(): if fromtx: if res: txd, txid = txdetails - taker.wallet.remove_old_utxos(txd) - taker.wallet.add_new_utxos(txd, txid) reactor.callLater(waittime*60, clientfactory.getClient().clientStart) else: @@ -259,10 +258,10 @@ def main(): txdetails=None): log.error("PayJoin payment was NOT made, timed out.") reactor.stop() - taker = P2EPTaker(options.p2ep, wallet, schedule, + taker = P2EPTaker(options.p2ep, wallet_service, schedule, callbacks=(None, None, p2ep_on_finished_callback)) else: - taker = Taker(wallet, + taker = Taker(wallet_service, schedule, order_chooser=chooseOrdersFunc, max_cj_fee=maxcjfee, diff --git a/scripts/tumbler.py b/scripts/tumbler.py index 9b72684..6d8188c 100644 --- a/scripts/tumbler.py +++ b/scripts/tumbler.py @@ -10,8 +10,8 @@ import pprint from twisted.python.log import startLogging from jmclient import Taker, load_program_config, get_schedule,\ JMClientProtocolFactory, start_reactor, jm_single, get_wallet_path,\ - open_test_wallet_maybe, sync_wallet, get_tumble_schedule,\ - schedule_to_text, estimate_tx_fee, restart_waiter,\ + open_test_wallet_maybe, get_tumble_schedule,\ + schedule_to_text, estimate_tx_fee, restart_waiter, WalletService,\ get_tumble_log, tumbler_taker_finished_update,\ tumbler_filter_orders_callback, validate_address from jmbase.support import get_log, jmprint @@ -38,13 +38,17 @@ def main(): #Load the wallet wallet_name = args[0] max_mix_depth = options['mixdepthsrc'] + options['mixdepthcount'] + if options['amtmixdepths'] > max_mix_depth: + max_mix_depth = options['amtmixdepths'] wallet_path = get_wallet_path(wallet_name, None) wallet = open_test_wallet_maybe(wallet_path, wallet_name, max_mix_depth) - if jm_single().config.get("BLOCKCHAIN", - "blockchain_source") == "electrum-server": - jm_single().bc_interface.synctype = "with-script" - while not jm_single().bc_interface.wallet_synced: - sync_wallet(wallet, fast=options['fastsync']) + wallet_service = WalletService(wallet) + # in this script, we need the wallet synced before + # logic processing for some paths, so do it now: + while not wallet_service.synced: + wallet_service.sync_wallet(fast=not options['recoversync']) + # the sync call here will now be a no-op: + wallet_service.startService() maxcjfee = get_max_cj_fee_values(jm_single().config, options_org) log.info("Using maximum coinjoin fee limits per maker of {:.4%}, {} sat" @@ -128,7 +132,7 @@ def main(): max_mix_to_tumble = min(options['mixdepthsrc']+options['mixdepthcount'], \ max_mix_depth) for i in range(options['mixdepthsrc'], max_mix_to_tumble): - total_tumble_amount += wallet.get_balance_by_mixdepth()[i] + total_tumble_amount += wallet_service.get_balance_by_mixdepth()[i] if total_tumble_amount == 0: raise ValueError("No confirmed coins in the selected mixdepth(s). Quitting") exp_tx_fees_ratio = (involved_parties * options['txfee']) \ @@ -164,7 +168,7 @@ def main(): reactor.callLater(waittime*60, clientfactory.getClient().clientStart) #instantiate Taker with given schedule and run - taker = Taker(wallet, + taker = Taker(wallet_service, schedule, order_chooser=options['order_choose_fn'], max_cj_fee=maxcjfee, diff --git a/scripts/yg-privacyenhanced.py b/scripts/yg-privacyenhanced.py index 2eef844..03631c9 100644 --- a/scripts/yg-privacyenhanced.py +++ b/scripts/yg-privacyenhanced.py @@ -34,8 +34,8 @@ jlog = get_log() class YieldGeneratorPrivacyEnhanced(YieldGeneratorBasic): - def __init__(self, wallet, offerconfig): - super(YieldGeneratorPrivacyEnhanced, self).__init__(wallet, offerconfig) + def __init__(self, wallet_service, offerconfig): + super(YieldGeneratorPrivacyEnhanced, self).__init__(wallet_service, offerconfig) def create_my_orders(self): mix_balance = self.get_available_mixdepths() diff --git a/test/common.py b/test/common.py index 3d15ee4..6dc8fce 100644 --- a/test/common.py +++ b/test/common.py @@ -15,27 +15,28 @@ sys.path.insert(0, os.path.join(data_dir)) from jmbase import get_log from jmclient import open_test_wallet_maybe, BIP32Wallet, SegwitLegacyWallet, \ - estimate_tx_fee, jm_single + estimate_tx_fee, jm_single, WalletService import jmbitcoin as btc from jmbase import chunks log = get_log() def make_sign_and_push(ins_full, - wallet, + wallet_service, amount, output_addr=None, change_addr=None, hashcode=btc.SIGHASH_ALL, estimate_fee = False): """Utility function for easily building transactions - from wallets + from wallets. """ + assert isinstance(wallet_service, WalletService) total = sum(x['value'] for x in ins_full.values()) ins = ins_full.keys() #random output address and change addr - output_addr = wallet.get_new_addr(1, 1) if not output_addr else output_addr - change_addr = wallet.get_new_addr(1, 0) if not change_addr else change_addr + output_addr = wallet_service.get_new_addr(1, 1) if not output_addr else output_addr + change_addr = wallet_service.get_new_addr(0, 1) if not change_addr else change_addr fee_est = estimate_tx_fee(len(ins), 2) if estimate_fee else 10000 outs = [{'value': amount, 'address': output_addr}, {'value': total - amount - fee_est, @@ -46,7 +47,7 @@ def make_sign_and_push(ins_full, for index, ins in enumerate(de_tx['ins']): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) addr = ins_full[utxo]['address'] - priv = wallet.get_key_from_addr(addr) + priv = wallet_service.get_key_from_addr(addr) if index % 2: priv = binascii.unhexlify(priv) tx = btc.sign(tx, index, priv, hashcode=hashcode) @@ -99,15 +100,16 @@ def make_wallets(n, w = open_test_wallet_maybe(seeds[i], seeds[i], mixdepths - 1, test_wallet_cls=walletclass) + wallet_service = WalletService(w) wallets[i + start_index] = {'seed': seeds[i].decode('ascii'), - 'wallet': w} + 'wallet': wallet_service} for j in range(mixdepths): for k in range(wallet_structures[i][j]): deviation = sdev_amt * random.random() amt = mean_amt - sdev_amt / 2.0 + deviation if amt < 0: amt = 0.001 amt = float(Decimal(amt).quantize(Decimal(10)**-8)) - jm_single().bc_interface.grab_coins(w.get_external_addr(j), amt) + jm_single().bc_interface.grab_coins(wallet_service.get_new_addr(j, 1), amt) return wallets diff --git a/test/test_full_coinjoin.py b/test/test_full_coinjoin.py index e332461..a223618 100644 --- a/test/test_full_coinjoin.py +++ b/test/test_full_coinjoin.py @@ -57,7 +57,7 @@ def test_cj(setup_full_coinjoin, num_ygs, wallet_structures, mean_amt, wallet = wallets[num_ygs]['wallet'] sync_wallet(wallet, fast=True) # grab a dest addr from the wallet - destaddr = wallet.get_new_addr(4, 0) + destaddr = wallet.get_external_addr(4) coinjoin_amt = 20000000 schedule = [[1, coinjoin_amt, 2, destaddr, 0.0, False]] diff --git a/test/test_segwit.py b/test/test_segwit.py index 3c76f0b..2aa281e 100644 --- a/test/test_segwit.py +++ b/test/test_segwit.py @@ -60,14 +60,14 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, MIXDEPTH = 0 # set up wallets and inputs - nsw_wallet = make_wallets(1, wallet_structure, in_amt, + nsw_wallet_service = make_wallets(1, wallet_structure, in_amt, walletclass=LegacyWallet)[0]['wallet'] - jm_single().bc_interface.sync_wallet(nsw_wallet, fast=True) - sw_wallet = make_wallets(1, [[len(segwit_ins), 0, 0, 0, 0]], segwit_amt)[0]['wallet'] - jm_single().bc_interface.sync_wallet(sw_wallet, fast=True) + nsw_wallet_service.sync_wallet(fast=True) + sw_wallet_service = make_wallets(1, [[len(segwit_ins), 0, 0, 0, 0]], segwit_amt)[0]['wallet'] + sw_wallet_service.sync_wallet(fast=True) - nsw_utxos = nsw_wallet.get_utxos_by_mixdepth_()[MIXDEPTH] - sw_utxos = sw_wallet.get_utxos_by_mixdepth_()[MIXDEPTH] + nsw_utxos = nsw_wallet_service.get_utxos_by_mixdepth(hexfmt=False)[MIXDEPTH] + sw_utxos = sw_wallet_service.get_utxos_by_mixdepth(hexfmt=False)[MIXDEPTH] assert len(o_ins) <= len(nsw_utxos), "sync failed" assert len(segwit_ins) <= len(sw_utxos), "sync failed" @@ -102,8 +102,8 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, FEE = 50000 assert FEE < total_amt_in_sat - amount, "test broken, not enough funds" - cj_script = nsw_wallet.get_new_script(MIXDEPTH + 1, True) - change_script = nsw_wallet.get_new_script(MIXDEPTH, True) + cj_script = nsw_wallet_service.get_new_script(MIXDEPTH + 1, True) + change_script = nsw_wallet_service.get_new_script(MIXDEPTH, True) change_amt = total_amt_in_sat - amount - FEE tx_outs = [ @@ -115,22 +115,21 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, # import new addresses to bitcoind jm_single().bc_interface.import_addresses( - [nsw_wallet.script_to_addr(x) - for x in [cj_script, change_script]], - jm_single().bc_interface.get_wallet_name(nsw_wallet)) + [nsw_wallet_service.script_to_addr(x) + for x in [cj_script, change_script]], nsw_wallet_service.get_wallet_name()) # sign tx scripts = {} for nsw_in_index in o_ins: inp = nsw_ins[nsw_in_index][1] scripts[nsw_in_index] = (inp['script'], inp['value']) - tx = nsw_wallet.sign_tx(tx, scripts) + tx = nsw_wallet_service.sign_tx(tx, scripts) scripts = {} for sw_in_index in segwit_ins: inp = sw_ins[sw_in_index][1] scripts[sw_in_index] = (inp['script'], inp['value']) - tx = sw_wallet.sign_tx(tx, scripts) + tx = sw_wallet_service.sign_tx(tx, scripts) print(tx) @@ -139,8 +138,8 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, assert txid balances = jm_single().bc_interface.get_received_by_addr( - [nsw_wallet.script_to_addr(cj_script), - nsw_wallet.script_to_addr(change_script)], None)['data'] + [nsw_wallet_service.script_to_addr(cj_script), + nsw_wallet_service.script_to_addr(change_script)], None)['data'] assert balances[0]['balance'] == amount assert balances[1]['balance'] == change_amt diff --git a/test/ygrunner.py b/test/ygrunner.py index 76587b2..e0bb18e 100644 --- a/test/ygrunner.py +++ b/test/ygrunner.py @@ -19,7 +19,7 @@ import pytest import random from jmbase import jmprint from jmclient import YieldGeneratorBasic, load_program_config, jm_single,\ - sync_wallet, JMClientProtocolFactory, start_reactor, SegwitWallet,\ + JMClientProtocolFactory, start_reactor, SegwitWallet,\ SegwitLegacyWallet, cryptoengine @@ -109,21 +109,21 @@ def test_start_ygs(setup_ygrunner, num_ygs, wallet_structures, mean_amt, # TODO add Legacy walletclass = SegwitLegacyWallet - wallets = make_wallets(num_ygs + 1, + wallet_services = make_wallets(num_ygs + 1, wallet_structures=wallet_structures, mean_amt=mean_amt, walletclass=walletclass) #the sendpayment bot uses the last wallet in the list - wallet = wallets[num_ygs]['wallet'] - jmprint("\n\nTaker wallet seed : " + wallets[num_ygs]['seed']) + wallet_service = wallet_services[num_ygs]['wallet'] + jmprint("\n\nTaker wallet seed : " + wallet_services[num_ygs]['seed']) # for manual audit if necessary, show the maker's wallet seeds # also (note this audit should be automated in future, see # test_full_coinjoin.py in this directory) jmprint("\n\nMaker wallet seeds: ") for i in range(num_ygs): - jmprint("Maker seed: " + wallets[i]['seed']) + jmprint("Maker seed: " + wallet_services[i]['seed']) jmprint("\n") - sync_wallet(wallet, fast=True) + wallet_service.sync_wallet(fast=True) txfee = 1000 cjfee_a = 4200 cjfee_r = '0.001' @@ -138,8 +138,9 @@ def test_start_ygs(setup_ygrunner, num_ygs, wallet_structures, mean_amt, for i in range(num_ygs): cfg = [txfee, cjfee_a, cjfee_r, ordertype, minsize] - sync_wallet(wallets[i]["wallet"], fast=True) - yg = ygclass(wallets[i]["wallet"], cfg) + wallet_service_yg = wallet_services[i]["wallet"] + wallet_service_yg.startService() + yg = ygclass(wallet_service_yg, cfg) if malicious: yg.set_maliciousness(malicious, mtype="tx") clientfactory = JMClientProtocolFactory(yg, proto_type="MAKER")