Browse Source

Merge #359: Improvements to wallet syncing and monitoring

c654de0 Wallet and blockchain refactoring (AdamISZ)
master
AdamISZ 6 years ago
parent
commit
9e6fb98002
No known key found for this signature in database
GPG Key ID: 141001A1AF77F20B
  1. 3
      jmbase/jmbase/__init__.py
  2. 3
      jmbase/jmbase/support.py
  3. 14
      jmbitcoin/jmbitcoin/secp256k1_transaction.py
  4. 5
      jmclient/jmclient/__init__.py
  5. 606
      jmclient/jmclient/blockchaininterface.py
  6. 90
      jmclient/jmclient/client_protocol.py
  7. 68
      jmclient/jmclient/maker.py
  8. 16
      jmclient/jmclient/output.py
  9. 6
      jmclient/jmclient/storage.py
  10. 112
      jmclient/jmclient/taker.py
  11. 26
      jmclient/jmclient/taker_utils.py
  12. 187
      jmclient/jmclient/wallet.py
  13. 661
      jmclient/jmclient/wallet_service.py
  14. 99
      jmclient/jmclient/wallet_utils.py
  15. 53
      jmclient/jmclient/yieldgenerator.py
  16. 46
      jmclient/test/commontest.py
  17. 113
      jmclient/test/test_blockchaininterface.py
  18. 14
      jmclient/test/test_client_protocol.py
  19. 95
      jmclient/test/test_coinjoin.py
  20. 6
      jmclient/test/test_maker.py
  21. 41
      jmclient/test/test_payjoin.py
  22. 6
      jmclient/test/test_podle.py
  23. 1
      jmclient/test/test_storage.py
  24. 46
      jmclient/test/test_taker.py
  25. 78
      jmclient/test/test_tx_creation.py
  26. 16
      jmclient/test/test_utxomanager.py
  27. 12
      jmclient/test/test_wallet.py
  28. 24
      jmclient/test/test_wallets.py
  29. 23
      jmclient/test/test_yieldgenerator.py
  30. 20
      scripts/add-utxo.py
  31. 20
      scripts/cli_options.py
  32. 162
      scripts/joinmarket-qt.py
  33. 10
      scripts/receive-payjoin.py
  34. 25
      scripts/sendpayment.py
  35. 22
      scripts/tumbler.py
  36. 4
      scripts/yg-privacyenhanced.py
  37. 18
      test/common.py
  38. 2
      test/test_full_coinjoin.py
  39. 29
      test/test_segwit.py
  40. 17
      test/ygrunner.py

3
jmbase/jmbase/__init__.py

@ -4,6 +4,7 @@ from builtins import *
from .support import (get_log, chunks, debug_silence, jmprint,
joinmarket_alert, core_alert, get_password,
set_logging_level, set_logging_color)
set_logging_level, set_logging_color,
JM_WALLET_NAME_PREFIX)
from .commands import *

3
jmbase/jmbase/support.py

@ -5,6 +5,9 @@ from builtins import * # noqa: F401
import logging
from getpass import getpass
# global Joinmarket constants
JM_WALLET_NAME_PREFIX = "joinmarket-wallet-"
from chromalog.log import (
ColorizingStreamHandler,
ColorizingFormatter,

14
jmbitcoin/jmbitcoin/secp256k1_transaction.py

@ -9,6 +9,8 @@ import copy
import re
import os
import struct
# note, only used for non-cryptographic randomness:
import random
from jmbitcoin.secp256k1_main import *
from jmbitcoin.bech32 import *
@ -871,3 +873,15 @@ def mktx(ins, outs, version=1, locktime=0):
txobj["outs"].append(outobj)
return serialize(txobj)
def make_shuffled_tx(ins, outs, deser=True, version=1, locktime=0):
""" Simple utility to ensure transaction
inputs and outputs are randomly ordered.
Can possibly be replaced by BIP69 in future
"""
random.shuffle(ins)
random.shuffle(outs)
tx = mktx(ins, outs, version=version, locktime=locktime)
if deser:
return deserialize(tx)
else:
return tx

5
jmclient/jmclient/__init__.py

@ -16,7 +16,7 @@ from .taker import Taker, P2EPTaker
from .wallet import (Mnemonic, estimate_tx_fee, WalletError, BaseWallet, ImportWalletMixin,
BIP39WalletMixin, BIP32Wallet, BIP49Wallet, LegacyWallet,
SegwitWallet, SegwitLegacyWallet, UTXOManager,
WALLET_IMPLEMENTATIONS, make_shuffled_tx)
WALLET_IMPLEMENTATIONS)
from .storage import (Argon2Hash, Storage, StorageError,
StoragePasswordError, VolatileStorage)
from .cryptoengine import BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, EngineError
@ -24,7 +24,7 @@ from .configure import (
load_program_config, get_p2pk_vbyte, jm_single, get_network,
validate_address, get_irc_mchannels, get_blockchain_interface_instance,
get_p2sh_vbyte, set_config, is_segwit_mode, is_native_segwit_mode)
from .blockchaininterface import (BlockchainInterface, sync_wallet,
from .blockchaininterface import (BlockchainInterface,
RegtestBitcoinCoreInterface, BitcoinCoreInterface)
from .electruminterface import ElectrumInterface
from .client_protocol import (JMTakerClientProtocol, JMClientProtocolFactory,
@ -46,6 +46,7 @@ from .wallet_utils import (
wallet_tool_main, wallet_generate_recover_bip39, open_wallet,
open_test_wallet_maybe, create_wallet, get_wallet_cls, get_wallet_path,
wallet_display, get_utxos_enabled_disabled)
from .wallet_service import WalletService
from .maker import Maker, P2EPMaker
from .yieldgenerator import YieldGenerator, YieldGeneratorBasic, ygmain
# Set default logging handler to avoid "No handler found" warnings.

606
jmclient/jmclient/blockchaininterface.py

@ -3,34 +3,21 @@ from __future__ import (absolute_import, division,
from builtins import * # noqa: F401
import abc
import ast
import random
import sys
import time
import binascii
from copy import deepcopy
from decimal import Decimal
from twisted.internet import reactor, task
import jmbitcoin as btc
from jmclient.jsonrpc import JsonRpcConnectionError, JsonRpcError
from jmclient.configure import get_p2pk_vbyte, jm_single
from jmclient.configure import jm_single
from jmbase.support import get_log, jmprint
log = get_log()
# an inaccessible blockheight; consider rewriting in 1900 years
INF_HEIGHT = 10**8
def sync_wallet(wallet, fast=False):
"""Wrapper function to choose fast syncing where it's
both possible and requested.
"""
if fast and (
isinstance(jm_single().bc_interface, BitcoinCoreInterface) or isinstance(
jm_single().bc_interface, RegtestBitcoinCoreInterface)):
jm_single().bc_interface.sync_wallet(wallet, fast=True)
else:
jm_single().bc_interface.sync_wallet(wallet)
log = get_log()
class BlockchainInterface(object):
__metaclass__ = abc.ABCMeta
@ -38,150 +25,12 @@ class BlockchainInterface(object):
def __init__(self):
pass
def sync_wallet(self, wallet, restart_cb=None):
"""Default behaviour is for Core and similar interfaces.
If address sync fails, flagged with wallet_synced value;
do not attempt to sync_unspent in that case.
"""
self.sync_addresses(wallet, restart_cb)
if self.wallet_synced:
self.sync_unspent(wallet)
@staticmethod
def get_wallet_name(wallet):
return 'joinmarket-wallet-' + wallet.get_wallet_id()
@abc.abstractmethod
def sync_addresses(self, wallet):
"""Finds which addresses have been used"""
@abc.abstractmethod
def sync_unspent(self, wallet):
"""Finds the unspent transaction outputs belonging to this wallet"""
def is_address_imported(self, addr):
try:
return self.rpc('getaccount', [addr]) != ''
except JsonRpcError:
return len(self.rpc('getaddressinfo', [addr])['labels']) > 0
def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr,
wallet_name=None, timeoutfun=None, spentfun=None, txid_flag=True,
n=0, c=1, vb=None):
"""Given a deserialized transaction txd,
callback functions for broadcast and confirmation of the transaction,
an address to import, and a callback function for timeout, set up
a polling loop to check for events on the transaction. Also optionally set
to trigger "confirmed" callback on number of confirmations c. Also checks
for spending (if spentfun is not None) of the outpoint n.
If txid_flag is True, we create a watcher loop on the txid (hence only
really usable in a segwit context, and only on fully formed transactions),
else we create a watcher loop on the output set of the transaction (taken
from the outs field of the txd).
"""
if not vb:
vb = get_p2pk_vbyte()
if isinstance(self, BitcoinCoreInterface) or isinstance(self,
RegtestBitcoinCoreInterface):
#This code ensures that a walletnotify is triggered, by
#ensuring that at least one of the output addresses is
#imported into the wallet (note the sweep special case, where
#none of the output addresses belong to me).
one_addr_imported = False
for outs in txd['outs']:
addr = btc.script_to_address(outs['script'], vb)
try:
if self.is_address_imported(addr):
one_addr_imported = True
break
except JsonRpcError as e:
log.debug("Failed to getaccount for address: " + addr)
log.debug("This is normal for bech32 addresses.")
continue
if not one_addr_imported:
try:
self.rpc('importaddress', [notifyaddr, 'joinmarket-notify', False])
except JsonRpcError as e:
#In edge case of address already controlled
#by another account, warn but do not quit in middle of tx.
#Can occur if destination is owned in Core wallet.
if e.code == -4 and e.message == "The wallet already " + \
"contains the private key for this address or script":
log.warn("WARNING: Failed to import address: " + notifyaddr)
#No other error should be possible
else:
raise
#Warning! In case of txid_flag false, this is *not* a valid txid,
#but only a hash of an incomplete transaction serialization.
txid = btc.txhash(btc.serialize(txd))
if not txid_flag:
tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']])
loop = task.LoopingCall(self.outputs_watcher, wallet_name, notifyaddr,
tx_output_set, unconfirmfun, confirmfun,
timeoutfun)
log.debug("Created watcher loop for address: " + notifyaddr)
loopkey = notifyaddr
else:
loop = task.LoopingCall(self.tx_watcher, txd, unconfirmfun, confirmfun,
spentfun, c, n)
log.debug("Created watcher loop for txid: " + txid)
loopkey = txid
self.tx_watcher_loops[loopkey] = [loop, False, False, False]
#Hardcoded polling interval, but in any case it can be very short.
loop.start(5.0)
#Give up on un-broadcast transactions and broadcast but not confirmed
#transactions as per settings in the config.
reactor.callLater(float(jm_single().config.get("TIMEOUT",
"unconfirm_timeout_sec")), self.tx_network_timeout, loopkey)
confirm_timeout_sec = int(jm_single().config.get(
"TIMEOUT", "confirm_timeout_hours")) * 3600
reactor.callLater(confirm_timeout_sec, self.tx_timeout, txd, loopkey, timeoutfun)
def tx_network_timeout(self, loopkey):
"""If unconfirm has not been called by the time this
is triggered, we abandon monitoring, assuming the tx has
not been broadcast.
"""
if not self.tx_watcher_loops[loopkey][1]:
log.info("Abandoning monitoring of un-broadcast tx for: " + str(loopkey))
if self.tx_watcher_loops[loopkey][0].running:
self.tx_watcher_loops[loopkey][0].stop()
def tx_timeout(self, txd, loopkey, timeoutfun):
"""Assuming we are watching for an already-broadcast
transaction, give up once this triggers if confirmation has not occurred.
"""
if not loopkey in self.tx_watcher_loops:
#Occurs if the tx has already confirmed before this
return
if not self.tx_watcher_loops[loopkey][2]:
#Not confirmed after prescribed timeout in hours; give up
log.info("Timed out waiting for confirmation of: " + str(loopkey))
if self.tx_watcher_loops[loopkey][0].running:
self.tx_watcher_loops[loopkey][0].stop()
if timeoutfun:
timeoutfun(txd, loopkey)
@abc.abstractmethod
def outputs_watcher(self, wallet_name, notifyaddr, tx_output_set,
unconfirmfun, confirmfun, timeoutfun):
"""Given a key for the watcher loop (notifyaddr), a wallet name (account),
a set of outputs, and unconfirm, confirm and timeout callbacks,
check to see if a transaction matching that output set has appeared in
the wallet. Call the callbacks and update the watcher loop state.
End the loop when the confirmation has been seen (no spent monitoring here).
"""
@abc.abstractmethod
def tx_watcher(self, txd, unconfirmfun, confirmfun, spentfun, c, n):
"""Called at a polling interval, checks if the given deserialized
transaction (which must be fully signed) is (a) broadcast, (b) confirmed
and (c) spent from at index n, and notifies confirmation if number
of confs = c.
TODO: Deal with conflicts correctly. Here just abandons monitoring.
"""
@abc.abstractmethod
def pushtx(self, txhex):
"""pushes tx to the network, returns False if failed"""
@ -233,16 +82,6 @@ class ElectrumWalletInterface(BlockchainInterface): #pragma: no cover
def sync_unspent(self, wallet):
log.debug("Dummy electrum interface, no sync unspent")
def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr):
log.debug("Dummy electrum interface, no add tx notify")
def outputs_watcher(self, wallet_name, notifyaddr,
tx_output_set, uf, cf, tf):
log.debug("Dummy electrum interface, no outputs watcher")
def tx_watcher(self, txd, ucf, cf, sf, c, n):
log.debug("Dummy electrum interface, no tx watcher")
def pushtx(self, txhex, timeout=10):
#synchronous send
from electrum.transaction import Transaction
@ -319,7 +158,6 @@ class BitcoinCoreInterface(BlockchainInterface):
def __init__(self, jsonRpc, network):
super(BitcoinCoreInterface, self).__init__()
self.jsonRpc = jsonRpc
self.fast_sync_called = False
blockchainInfo = self.jsonRpc.call("getblockchaininfo", [])
actualNet = blockchainInfo['chain']
@ -327,13 +165,6 @@ class BitcoinCoreInterface(BlockchainInterface):
if netmap[actualNet] != network:
raise Exception('wrong network configured')
self.txnotify_fun = []
self.wallet_synced = False
#task.LoopingCall objects that track transactions, keyed by txids.
#Format: {"txid": (loop, unconfirmed true/false, confirmed true/false,
#spent true/false), ..}
self.tx_watcher_loops = {}
def get_block(self, blockheight):
"""Returns full serialized block at a given height.
"""
@ -346,7 +177,7 @@ class BitcoinCoreInterface(BlockchainInterface):
def rpc(self, method, args):
if method not in ['importaddress', 'walletpassphrase', 'getaccount',
'gettransaction', 'getrawtransaction', 'gettxout',
'importmulti']:
'importmulti', 'listtransactions', 'getblockcount']:
log.debug('rpc: ' + method + " " + str(args))
res = self.jsonRpc.call(method, args)
return res
@ -357,8 +188,6 @@ class BitcoinCoreInterface(BlockchainInterface):
of another account/label (see console output), and quits.
Do NOT use for in-run imports, use rpc('importaddress',..) instead.
"""
log.debug('importing ' + str(len(addr_list)) +
' addresses with label ' + wallet_name)
requests = []
for addr in addr_list:
requests.append({
@ -395,7 +224,7 @@ class BitcoinCoreInterface(BlockchainInterface):
but in some cases a rescan is not required (if the address is known
to be new/unused). For that case use import_addresses instead.
"""
self.import_addresses(addr_list, wallet_name, restart_cb)
self.import_addresses(addr_list, wallet_name)
if jm_single().config.get("BLOCKCHAIN",
"blockchain_source") != 'regtest': #pragma: no cover
#Exit conditions cannot be included in tests
@ -409,218 +238,6 @@ class BitcoinCoreInterface(BlockchainInterface):
jmprint(restart_msg, "important")
sys.exit(0)
def sync_wallet(self, wallet, fast=False, restart_cb=None):
#trigger fast sync if the index_cache is available
#(and not specifically disabled).
if fast:
self.sync_wallet_fast(wallet)
self.fast_sync_called = True
return
super(BitcoinCoreInterface, self).sync_wallet(wallet, restart_cb=restart_cb)
self.fast_sync_called = False
def sync_wallet_fast(self, wallet):
"""Exploits the fact that given an index_cache,
all addresses necessary should be imported, so we
can just list all used addresses to find the right
index values.
"""
self.get_address_usages(wallet)
self.sync_unspent(wallet)
def get_address_usages(self, wallet):
"""Use rpc `listaddressgroupings` to locate all used
addresses in the account (whether spent or unspent outputs).
This will not result in a full sync if working with a new
Bitcoin Core instance, in which case "fast" should have been
specifically disabled by the user.
"""
wallet_name = self.get_wallet_name(wallet)
agd = self.rpc('listaddressgroupings', [])
#flatten all groups into a single list; then, remove duplicates
fagd = (tuple(item) for sublist in agd for item in sublist)
#"deduplicated flattened address grouping data" = dfagd
dfagd = set(fagd)
used_addresses = set()
for addr_info in dfagd:
if len(addr_info) < 3 or addr_info[2] != wallet_name:
continue
used_addresses.add(addr_info[0])
#for a first run, import first chunk
if not used_addresses:
log.info("Detected new wallet, performing initial import")
# delegate inital address import to sync_addresses
# this should be fast because "getaddressesbyaccount" should return
# an empty list in this case
self.sync_addresses(wallet)
self.wallet_synced = True
return
#Wallet has been used; scan forwards.
log.debug("Fast sync in progress. Got this many used addresses: " + str(
len(used_addresses)))
#Need to have wallet.index point to the last used address
#Algo:
# 1. Scan batch 1 of each branch, record matched wallet addresses.
# 2. Check if all addresses in 'used addresses' have been matched, if
# so, break.
# 3. Repeat the above for batch 2, 3.. up to max 20 batches.
# 4. If after all 20 batches not all used addresses were matched,
# quit with error.
# 5. Calculate used indices.
# 6. If all used addresses were matched, set wallet index to highest
# found index in each branch and mark wallet sync complete.
#Rationale for this algo:
# Retrieving addresses is a non-zero computational load, so batching
# and then caching allows a small sync to complete *reasonably*
# quickly while a larger one is not really negatively affected.
# The downside is another free variable, batch size, but this need
# not be exposed to the user; it is not the same as gap limit, in fact,
# the concept of gap limit does not apply to this kind of sync, which
# *assumes* that the most recent usage of addresses is indeed recorded.
remaining_used_addresses = used_addresses.copy()
addresses, saved_indices = self._collect_addresses_init(wallet)
for addr in addresses:
remaining_used_addresses.discard(addr)
BATCH_SIZE = 100
MAX_ITERATIONS = 20
current_indices = deepcopy(saved_indices)
for j in range(MAX_ITERATIONS):
if not remaining_used_addresses:
break
for addr in \
self._collect_addresses_gap(wallet, gap_limit=BATCH_SIZE):
remaining_used_addresses.discard(addr)
# increase wallet indices for next iteration
for md in current_indices:
current_indices[md][0] += BATCH_SIZE
current_indices[md][1] += BATCH_SIZE
self._rewind_wallet_indices(wallet, current_indices,
current_indices)
else:
self._rewind_wallet_indices(wallet, saved_indices, saved_indices)
raise Exception("Failed to sync in fast mode after 20 batches; "
"please re-try wallet sync without --fast flag.")
# creating used_indices on-the-fly would be more efficient, but the
# overall performance gain is probably negligible
used_indices = self._get_used_indices(wallet, used_addresses)
self._rewind_wallet_indices(wallet, used_indices, saved_indices)
self.wallet_synced = True
def sync_addresses(self, wallet, restart_cb=None):
log.debug("requesting detailed wallet history")
wallet_name = self.get_wallet_name(wallet)
addresses, saved_indices = self._collect_addresses_init(wallet)
try:
imported_addresses = set(self.rpc('getaddressesbyaccount',
[wallet_name]))
except JsonRpcError:
if wallet_name in self.rpc('listlabels', []):
imported_addresses = set(self.rpc('getaddressesbylabel',
[wallet_name]).keys())
else:
imported_addresses = set()
if not addresses.issubset(imported_addresses):
self.add_watchonly_addresses(addresses - imported_addresses,
wallet_name, restart_cb)
return
used_addresses_gen = (tx['address']
for tx in self._yield_transactions(wallet_name)
if tx['category'] == 'receive')
used_indices = self._get_used_indices(wallet, used_addresses_gen)
log.debug("got used indices: {}".format(used_indices))
gap_limit_used = not self._check_gap_indices(wallet, used_indices)
self._rewind_wallet_indices(wallet, used_indices, saved_indices)
new_addresses = self._collect_addresses_gap(wallet)
if not new_addresses.issubset(imported_addresses):
log.debug("Syncing iteration finished, additional step required")
self.add_watchonly_addresses(new_addresses - imported_addresses,
wallet_name, restart_cb)
self.wallet_synced = False
elif gap_limit_used:
log.debug("Syncing iteration finished, additional step required")
self.wallet_synced = False
else:
log.debug("Wallet successfully synced")
self._rewind_wallet_indices(wallet, used_indices, saved_indices)
self.wallet_synced = True
@staticmethod
def _rewind_wallet_indices(wallet, used_indices, saved_indices):
for md in used_indices:
for int_type in (0, 1):
index = max(used_indices[md][int_type],
saved_indices[md][int_type])
wallet.set_next_index(md, int_type, index, force=True)
@staticmethod
def _get_used_indices(wallet, addr_gen):
indices = {x: [0, 0] for x in range(wallet.max_mixdepth + 1)}
for addr in addr_gen:
if not wallet.is_known_addr(addr):
continue
md, internal, index = wallet.get_details(
wallet.addr_to_path(addr))
if internal not in (0, 1):
assert internal == 'imported'
continue
indices[md][internal] = max(indices[md][internal], index + 1)
return indices
@staticmethod
def _check_gap_indices(wallet, used_indices):
for md in used_indices:
for internal in (0, 1):
if used_indices[md][internal] >\
max(wallet.get_next_unused_index(md, internal), 0):
return False
return True
@staticmethod
def _collect_addresses_init(wallet):
addresses = set()
saved_indices = dict()
for md in range(wallet.max_mixdepth + 1):
saved_indices[md] = [0, 0]
for internal in (0, 1):
next_unused = wallet.get_next_unused_index(md, internal)
for index in range(next_unused):
addresses.add(wallet.get_addr(md, internal, index))
for index in range(wallet.gap_limit):
addresses.add(wallet.get_new_addr(md, internal))
wallet.set_next_index(md, internal, next_unused)
saved_indices[md][internal] = next_unused
for path in wallet.yield_imported_paths(md):
addresses.add(wallet.get_addr_path(path))
return addresses, saved_indices
@staticmethod
def _collect_addresses_gap(wallet, gap_limit=None):
gap_limit = gap_limit or wallet.gap_limit
addresses = set()
for md in range(wallet.max_mixdepth + 1):
for internal in (True, False):
old_next = wallet.get_next_unused_index(md, internal)
for index in range(gap_limit):
addresses.add(wallet.get_new_addr(md, internal))
wallet.set_next_index(md, internal, old_next)
return addresses
def _yield_transactions(self, wallet_name):
batch_size = 1000
iteration = 0
@ -634,47 +251,6 @@ class BitcoinCoreInterface(BlockchainInterface):
return
iteration += 1
def start_unspent_monitoring(self, wallet):
self.unspent_monitoring_loop = task.LoopingCall(self.sync_unspent, wallet)
self.unspent_monitoring_loop.start(1.0)
def stop_unspent_monitoring(self):
self.unspent_monitoring_loop.stop()
def sync_unspent(self, wallet):
st = time.time()
wallet_name = self.get_wallet_name(wallet)
wallet.reset_utxos()
listunspent_args = []
if 'listunspent_args' in jm_single().config.options('POLICY'):
listunspent_args = ast.literal_eval(jm_single().config.get(
'POLICY', 'listunspent_args'))
unspent_list = self.rpc('listunspent', listunspent_args)
for u in unspent_list:
if not wallet.is_known_addr(u['address']):
continue
self._add_unspent_utxo(wallet, u)
et = time.time()
log.debug('bitcoind sync_unspent took ' + str((et - st)) + 'sec')
self.wallet_synced = True
@staticmethod
def _add_unspent_utxo(wallet, utxo):
"""
Add a UTXO as returned by rpc's listunspent call to the wallet.
params:
wallet: wallet
utxo: single utxo dict as returned by listunspent
"""
txid = binascii.unhexlify(utxo['txid'])
script = binascii.unhexlify(utxo['scriptPubKey'])
value = int(Decimal(str(utxo['amount'])) * Decimal('1e8'))
wallet.add_utxo(txid, int(utxo['vout']), script, value)
def get_deser_from_gettransaction(self, rpcretval):
"""Get full transaction deserialization from a call
to `gettransaction`
@ -686,156 +262,36 @@ class BitcoinCoreInterface(BlockchainInterface):
hexval = str(rpcretval["hex"])
return btc.deserialize(hexval)
def outputs_watcher(self, wallet_name, notifyaddr, tx_output_set,
unconfirmfun, confirmfun, timeoutfun):
"""Given a key for the watcher loop (notifyaddr), a wallet name (label),
a set of outputs, and unconfirm, confirm and timeout callbacks,
check to see if a transaction matching that output set has appeared in
the wallet. Call the callbacks and update the watcher loop state.
End the loop when the confirmation has been seen (no spent monitoring here).
def list_transactions(self, num):
""" Return a list of the last `num` transactions seen
in the wallet (under any label/account).
"""
wl = self.tx_watcher_loops[notifyaddr]
txlist = self.rpc("listtransactions", ["*", 100, 0, True])
for tx in txlist[::-1]:
#changed syntax in 0.14.0; allow both syntaxes
try:
res = self.rpc("gettransaction", [tx["txid"], True])
except:
try:
res = self.rpc("gettransaction", [tx["txid"], 1])
except JsonRpcError as e:
#This should never happen (gettransaction is a wallet rpc).
log.warn("Failed gettransaction call; JsonRpcError")
res = None
except Exception as e:
log.warn("Failed gettransaction call; unexpected error:")
log.warn(str(e))
res = None
if not res:
continue
if "confirmations" not in res:
log.debug("Malformed gettx result: " + str(res))
return
txd = self.get_deser_from_gettransaction(res)
if txd is None:
continue
txos = set([(sv['script'], sv['value']) for sv in txd['outs']])
if not txos == tx_output_set:
continue
#Here we have found a matching transaction in the wallet.
real_txid = btc.txhash(btc.serialize(txd))
if not wl[1] and res["confirmations"] == 0:
log.debug("Tx: " + str(real_txid) + " seen on network.")
unconfirmfun(txd, real_txid)
wl[1] = True
return
if not wl[2] and res["confirmations"] > 0:
log.debug("Tx: " + str(real_txid) + " has " + str(
res["confirmations"]) + " confirmations.")
confirmfun(txd, real_txid, res["confirmations"])
wl[2] = True
wl[0].stop()
return
if res["confirmations"] < 0:
log.debug("Tx: " + str(real_txid) + " has a conflict. Abandoning.")
wl[0].stop()
return
return self.rpc("listtransactions", ["*", num, 0, True])
def tx_watcher(self, txd, unconfirmfun, confirmfun, spentfun, c, n):
"""Called at a polling interval, checks if the given deserialized
transaction (which must be fully signed) is (a) broadcast, (b) confirmed
and (c) spent from at index n, and notifies confirmation if number
of confs = c.
TODO: Deal with conflicts correctly. Here just abandons monitoring.
def get_transaction(self, txid):
""" Returns a serialized transaction for txid txid,
in hex as returned by Bitcoin Core rpc, or None
if no transaction can be retrieved. Works also for
watch-only wallets.
"""
txid = btc.txhash(btc.serialize(txd))
wl = self.tx_watcher_loops[txid]
#changed syntax in 0.14.0; allow both syntaxes
try:
res = self.rpc('gettransaction', [txid, True])
except JsonRpcError as e:
return
if not res:
return
res = self.rpc("gettransaction", [txid, True])
except:
try:
res = self.rpc("gettransaction", [txid, 1])
except JsonRpcError as e:
#This should never happen (gettransaction is a wallet rpc).
log.warn("Failed gettransaction call; JsonRpcError")
return None
except Exception as e:
log.warn("Failed gettransaction call; unexpected error:")
log.warn(str(e))
return None
if "confirmations" not in res:
log.debug("Malformed gettx result: " + str(res))
return
if not wl[1] and res["confirmations"] == 0:
log.debug("Tx: " + str(txid) + " seen on network.")
unconfirmfun(txd, txid)
wl[1] = True
return
if not wl[2] and res["confirmations"] > 0:
log.debug("Tx: " + str(txid) + " has " + str(
res["confirmations"]) + " confirmations.")
confirmfun(txd, txid, res["confirmations"])
if c <= res["confirmations"]:
wl[2] = True
#Note we do not stop the monitoring loop when
#confirmations occur, since we are also monitoring for spending.
return
if res["confirmations"] < 0:
log.debug("Tx: " + str(txid) + " has a conflict. Abandoning.")
wl[0].stop()
return
if not spentfun or wl[3]:
return
#To trigger the spent callback, we check if this utxo outpoint appears in
#listunspent output with 0 or more confirmations. Note that this requires
#we have added the destination address to the watch-only wallet, otherwise
#that outpoint will not be returned by listunspent.
res2 = self.rpc('listunspent', [0, 999999])
if not res2:
return
txunspent = False
for r in res2:
if "txid" not in r:
continue
if txid == r["txid"] and n == r["vout"]:
txunspent = True
break
if not txunspent:
#We need to find the transaction which spent this one;
#assuming the address was added to the wallet, then this
#transaction must be in the recent list retrieved via listunspent.
#For each one, use gettransaction to check its inputs.
#This is a bit expensive, but should only occur once.
txlist = self.rpc("listtransactions", ["*", 1000, 0, True])
for tx in txlist[::-1]:
#changed syntax in 0.14.0; allow both syntaxes
try:
res = self.rpc("gettransaction", [tx["txid"], True])
except:
try:
res = self.rpc("gettransaction", [tx["txid"], 1])
except:
#This should never happen (gettransaction is a wallet rpc).
log.info("Failed any gettransaction call")
res = None
if not res:
continue
deser = self.get_deser_from_gettransaction(res)
if deser is None:
continue
for vin in deser["ins"]:
if not "outpoint" in vin:
#coinbases
continue
if vin["outpoint"]["hash"] == txid and vin["outpoint"]["index"] == n:
#recover the deserialized form of the spending transaction.
log.info("We found a spending transaction: " + \
btc.txhash(binascii.unhexlify(res["hex"])))
res2 = self.rpc("gettransaction", [tx["txid"], True])
spending_deser = self.get_deser_from_gettransaction(res2)
if not spending_deser:
log.info("ERROR: could not deserialize spending tx.")
#Should never happen, it's a parsing bug.
#No point continuing to monitor, we just hope we
#can extract the secret by scanning blocks.
wl[3] = True
return
spentfun(spending_deser, vin["outpoint"]["hash"])
wl[3] = True
return
log.warning("Malformed gettx result: " + str(res))
return None
return res
def pushtx(self, txhex):
try:

90
jmclient/jmclient/client_protocol.py

@ -18,9 +18,8 @@ import hashlib
import os
import sys
from jmbase import get_log
from jmclient import (jm_single, get_irc_mchannels, get_p2sh_vbyte,
from jmclient import (jm_single, get_irc_mchannels,
RegtestBitcoinCoreInterface)
from .output import fmt_tx_data
import jmbitcoin as btc
jlog = get_log()
@ -263,72 +262,79 @@ class JMMakerClientProtocol(JMClientProtocol):
self.finalized_offers[nick] = offer
tx = btc.deserialize(txhex)
self.finalized_offers[nick]["txd"] = tx
jm_single().bc_interface.add_tx_notify(tx, self.unconfirm_callback,
self.confirm_callback, offer["cjaddr"],
wallet_name=jm_single().bc_interface.get_wallet_name(
self.client.wallet),
txid_flag=False,
vb=get_p2sh_vbyte())
txid = btc.txhash(btc.serialize(tx))
# we index the callback by the out-set of the transaction,
# because the txid is not known until all scriptSigs collected
# (hence this is required for Makers, but not Takers).
# For more info see WalletService.transaction_monitor():
txinfo = tuple((x["script"], x["value"]) for x in tx["outs"])
self.client.wallet_service.register_callbacks([self.unconfirm_callback],
txinfo, "unconfirmed")
self.client.wallet_service.register_callbacks([self.confirm_callback],
txinfo, "confirmed")
task.deferLater(reactor, float(jm_single().config.getint("TIMEOUT",
"unconfirm_timeout_sec")),
self.client.wallet_service.check_callback_called,
txinfo, self.unconfirm_callback, "unconfirmed",
"transaction with outputs: " + str(txinfo) + " not broadcast.")
d = self.callRemote(commands.JMTXSigs,
nick=nick,
sigs=json.dumps(sigs))
self.defaultCallbacks(d)
return {"accepted": True}
def unconfirm_callback(self, txd, txid):
#find the offer for this tx
offerinfo = None
def tx_match(self, txd):
for k,v in iteritems(self.finalized_offers):
#Tx considered defined by its output set
if v["txd"]["outs"] == txd["outs"]:
offerinfo = v
break
else:
return False
return offerinfo
def unconfirm_callback(self, txd, txid):
#find the offer for this tx
offerinfo = self.tx_match(txd)
if not offerinfo:
jlog.info("Failed to find notified unconfirmed transaction: " + txid)
return
removed_utxos = self.client.wallet.remove_old_utxos(txd)
jlog.info('saw tx on network, removed_utxos=\n{}'.format('\n'.join(
'{} - {}'.format(u, fmt_tx_data(tx_data, self.client.wallet))
for u, tx_data in removed_utxos.items())))
return False
to_cancel, to_announce = self.client.on_tx_unconfirmed(offerinfo,
txid, removed_utxos)
txid)
self.client.modify_orders(to_cancel, to_announce)
txinfo = tuple((x["script"], x["value"]) for x in txd["outs"])
confirm_timeout_sec = float(jm_single().config.get(
"TIMEOUT", "confirm_timeout_hours")) * 3600
task.deferLater(reactor, confirm_timeout_sec,
self.client.wallet_service.check_callback_called,
txinfo, self.confirm_callback, "confirmed",
"transaction with outputs " + str(txinfo) + " not confirmed.")
d = self.callRemote(commands.JMAnnounceOffers,
to_announce=json.dumps(to_announce),
to_cancel=json.dumps(to_cancel),
offerlist=json.dumps(self.client.offerlist))
self.defaultCallbacks(d)
return True
def confirm_callback(self, txd, txid, confirmations):
def confirm_callback(self, txd, txid, confirms):
#find the offer for this tx
offerinfo = None
for k,v in iteritems(self.finalized_offers):
#Tx considered defined by its output set
if v["txd"]["outs"] == txd["outs"]:
offerinfo = v
break
offerinfo = self.tx_match(txd)
if not offerinfo:
jlog.info("Failed to find notified unconfirmed transaction: " + txid)
return
jm_single().bc_interface.wallet_synced = False
jm_single().bc_interface.sync_unspent(self.client.wallet)
jlog.info('tx in a block: ' + txid)
self.wait_for_sync_loop = task.LoopingCall(self.modify_orders, offerinfo,
confirmations, txid)
self.wait_for_sync_loop.start(2.0)
def modify_orders(self, offerinfo, confirmations, txid):
if not jm_single().bc_interface.wallet_synced:
return
self.wait_for_sync_loop.stop()
return False
jlog.info('tx in a block: ' + txid + ' with ' + str(
confirms) + ' confirmations.')
to_cancel, to_announce = self.client.on_tx_confirmed(offerinfo,
confirmations, txid)
txid, confirms)
self.client.modify_orders(to_cancel, to_announce)
d = self.callRemote(commands.JMAnnounceOffers,
to_announce=json.dumps(to_announce),
to_cancel=json.dumps(to_cancel),
offerlist=json.dumps(self.client.offerlist))
to_announce=json.dumps(to_announce),
to_cancel=json.dumps(to_cancel),
offerlist=json.dumps(self.client.offerlist))
self.defaultCallbacks(d)
return True
class JMTakerClientProtocol(JMClientProtocol):

68
jmclient/jmclient/maker.py

@ -13,7 +13,8 @@ from binascii import unhexlify
from jmbitcoin import SerializationError, SerializationTruncationError
import jmbitcoin as btc
from jmclient.wallet import estimate_tx_fee, make_shuffled_tx
from jmclient.wallet import estimate_tx_fee
from jmclient.wallet_service import WalletService
from jmclient.configure import jm_single
from jmbase.support import get_log
from jmclient.support import calc_cj_fee, select_one_utxo
@ -24,9 +25,10 @@ from .cryptoengine import EngineError
jlog = get_log()
class Maker(object):
def __init__(self, wallet):
def __init__(self, wallet_service):
self.active_orders = {}
self.wallet = wallet
assert isinstance(wallet_service, WalletService)
self.wallet_service = wallet_service
self.nextoid = -1
self.offerlist = None
self.sync_wait_loop = task.LoopingCall(self.try_to_create_my_orders)
@ -40,7 +42,7 @@ class Maker(object):
is flagged as True. TODO: Use a deferred, probably.
Note that create_my_orders() is defined by subclasses.
"""
if not jm_single().bc_interface.wallet_synced:
if not self.wallet_service.synced:
return
self.offerlist = self.create_my_orders()
self.sync_wait_loop.stop()
@ -89,7 +91,7 @@ class Maker(object):
return reject(reason)
try:
if not self.wallet.pubkey_has_script(
if not self.wallet_service.pubkey_has_script(
unhexlify(cr_dict['P']), unhexlify(res[0]['script'])):
raise EngineError()
except EngineError:
@ -102,13 +104,14 @@ class Maker(object):
if not utxos:
#could not find funds
return (False,)
self.wallet.update_cache_index()
# for index update persistence:
self.wallet_service.save_wallet()
# Construct data for auth request back to taker.
# Need to choose an input utxo pubkey to sign with
# (no longer using the coinjoin pubkey from 0.2.0)
# Just choose the first utxo in self.utxos and retrieve key from wallet.
auth_address = utxos[list(utxos.keys())[0]]['address']
auth_key = self.wallet.get_key_from_addr(auth_address)
auth_key = self.wallet_service.get_key_from_addr(auth_address)
auth_pub = btc.privtopub(auth_key)
btc_sig = btc.ecdsa_sign(kphex, auth_key)
return (True, utxos, auth_pub, cj_addr, change_addr, btc_sig)
@ -137,11 +140,11 @@ class Maker(object):
utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
if utxo not in utxos:
continue
script = self.wallet.addr_to_script(utxos[utxo]['address'])
script = self.wallet_service.addr_to_script(utxos[utxo]['address'])
amount = utxos[utxo]['value']
our_inputs[index] = (script, amount)
txs = self.wallet.sign_tx(btc.deserialize(unhexlify(txhex)), our_inputs)
txs = self.wallet_service.sign_tx(btc.deserialize(unhexlify(txhex)), our_inputs)
for index in our_inputs:
sigmsg = unhexlify(txs['ins'][index]['script'])
if 'txinwitness' in txs['ins'][index]:
@ -268,13 +271,13 @@ class Maker(object):
"""
@abc.abstractmethod
def on_tx_unconfirmed(self, cjorder, txid, removed_utxos):
def on_tx_unconfirmed(self, cjorder, txid):
"""Performs action on receipt of transaction into the
mempool in the blockchain instance (e.g. announcing orders)
"""
@abc.abstractmethod
def on_tx_confirmed(self, cjorder, confirmations, txid):
def on_tx_confirmed(self, cjorder, txid, confirmations):
"""Performs actions on receipt of 1st confirmation of
a transaction into a block (e.g. announce orders)
"""
@ -292,15 +295,16 @@ class P2EPMaker(Maker):
and partially signed transaction, thus information leak to snoopers
is not possible.
"""
def __init__(self, wallet, mixdepth, amount):
def __init__(self, wallet_service, mixdepth, amount):
super(P2EPMaker, self).__init__(wallet_service)
self.receiving_amount = amount
self.mixdepth = mixdepth
# destination mixdepth must be different from that
# which we source coins from; use the standard "next"
dest_mixdepth = (self.mixdepth + 1) % wallet.max_mixdepth
dest_mixdepth = (self.mixdepth + 1) % self.wallet_service.max_mixdepth
# Select an unused destination in the external branch
self.destination_addr = wallet.get_new_addr(dest_mixdepth, 0)
super(P2EPMaker, self).__init__(wallet)
self.destination_addr = self.wallet_service.get_external_addr(
dest_mixdepth)
# Callback to request user permission (for e.g. GUI)
# args: (1) message, as string
# returns: True or False
@ -371,15 +375,19 @@ class P2EPMaker(Maker):
pass
def on_tx_unconfirmed(self, txd, txid):
""" For P2EP Maker there is no "offer", so
the second argument is repurposed as the deserialized
transaction.
"""
self.user_info("The transaction has been broadcast.")
self.user_info("Txid is: " + txid)
self.user_info("Transaction in detail: " + pprint.pformat(txd))
self.user_info("shutting down.")
reactor.stop()
def on_tx_confirmed(self, offer, confirmations, txid):
def on_tx_confirmed(self, txd, txid, confirmations):
# will not be reached except in testing
self.on_tx_unconfirmed(offer, confirmations)
self.on_tx_unconfirmed(txd, txid)
def on_tx_received(self, nick, txhex):
""" Called when the sender-counterparty has sent a transaction proposal.
@ -461,7 +469,7 @@ class P2EPMaker(Maker):
self.user_info("Network transaction fee of fallback tx is: " + str(
btc_fee) + " satoshis.")
fee_est = estimate_tx_fee(len(tx['ins']), len(tx['outs']),
txtype=self.wallet.get_txtype())
txtype=self.wallet_service.get_txtype())
fee_ok = False
if btc_fee > 0.3 * fee_est and btc_fee < 3 * fee_est:
fee_ok = True
@ -546,7 +554,7 @@ class P2EPMaker(Maker):
# sweeping dust ...
self.user_info("Choosing one coin at random")
try:
my_utxos = self.wallet.select_utxos(
my_utxos = self.wallet_service.select_utxos(
self.mixdepth, jm_single().DUST_THRESHOLD,
select_fn=select_one_utxo)
except:
@ -556,15 +564,15 @@ class P2EPMaker(Maker):
# get an approximate required amount assuming 4 inputs, which is
# fairly conservative (but guess by necessity).
fee_for_select = estimate_tx_fee(len(tx['ins']) + 4, 2,
txtype=self.wallet.get_txtype())
txtype=self.wallet_service.get_txtype())
approx_sum = max_sender_amt - self.receiving_amount + fee_for_select
try:
my_utxos = self.wallet.select_utxos(self.mixdepth, approx_sum)
my_utxos = self.wallet_service.select_utxos(self.mixdepth, approx_sum)
not_uih2 = True
except Exception:
# TODO probably not logical to always sweep here.
self.user_info("Sweeping all coins in this mixdepth.")
my_utxos = self.wallet.get_utxos_by_mixdepth()[self.mixdepth]
my_utxos = self.wallet_service.get_utxos_by_mixdepth()[self.mixdepth]
if my_utxos == {}:
return self.no_coins_fallback()
if not_uih2:
@ -582,7 +590,7 @@ class P2EPMaker(Maker):
new_destination_amount = self.receiving_amount + my_total_in
# estimate the required fee for the new version of the transaction
total_ins = len(tx["ins"]) + len(my_utxos.keys())
est_fee = estimate_tx_fee(total_ins, 2, txtype=self.wallet.get_txtype())
est_fee = estimate_tx_fee(total_ins, 2, txtype=self.wallet_service.get_txtype())
self.user_info("We estimated a fee of: " + str(est_fee))
new_change_amount = total_sender_input + my_total_in - \
new_destination_amount - est_fee
@ -601,7 +609,7 @@ class P2EPMaker(Maker):
# this call should never fail so no catch here.
currentblock = jm_single().bc_interface.rpc(
"getblockchaininfo", [])["blocks"]
new_tx = make_shuffled_tx(new_ins, new_outs, False, 2, currentblock)
new_tx = btc.make_shuffled_tx(new_ins, new_outs, False, 2, currentblock)
new_tx_deser = btc.deserialize(new_tx)
# sign our inputs before transfer
@ -610,16 +618,14 @@ class P2EPMaker(Maker):
utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
if utxo not in my_utxos:
continue
script = self.wallet.addr_to_script(my_utxos[utxo]['address'])
script = self.wallet_service.addr_to_script(my_utxos[utxo]['address'])
amount = my_utxos[utxo]['value']
our_inputs[index] = (script, amount)
txs = self.wallet.sign_tx(btc.deserialize(new_tx), our_inputs)
jm_single().bc_interface.add_tx_notify(txs,
self.on_tx_unconfirmed, self.on_tx_confirmed,
self.destination_addr,
wallet_name=jm_single().bc_interface.get_wallet_name(self.wallet),
txid_flag=False, vb=self.wallet._ENGINE.VBYTE)
txs = self.wallet_service.sign_tx(btc.deserialize(new_tx), our_inputs)
txinfo = tuple((x["script"], x["value"]) for x in txs["outs"])
self.wallet_service.register_callbacks([self.on_tx_unconfirmed], txinfo, "unconfirmed")
self.wallet_service.register_callbacks([self.on_tx_confirmed], txinfo, "confirmed")
# The blockchain interface just abandons monitoring if the transaction
# is not broadcast before the configured timeout; we want to take
# action in this case, so we add an additional callback to the reactor:

16
jmclient/jmclient/output.py

@ -4,11 +4,11 @@ from builtins import * # noqa: F401
from binascii import hexlify
def fmt_utxos(utxos, wallet, prefix=''):
def fmt_utxos(utxos, wallet_service, prefix=''):
output = []
for u in utxos:
utxo_str = '{}{} - {}'.format(
prefix, fmt_utxo(u), fmt_tx_data(utxos[u], wallet))
prefix, fmt_utxo(u), fmt_tx_data(utxos[u], wallet_service))
output.append(utxo_str)
return '\n'.join(output)
@ -17,13 +17,13 @@ def fmt_utxo(utxo):
return '{}:{}'.format(hexlify(utxo[0]).decode('ascii'), utxo[1])
def fmt_tx_data(tx_data, wallet):
def fmt_tx_data(tx_data, wallet_service):
return 'path: {}, address: {}, value: {}'.format(
wallet.get_path_repr(wallet.script_to_path(tx_data['script'])),
wallet.script_to_addr(tx_data['script']), tx_data['value'])
wallet_service.get_path_repr(wallet_service.script_to_path(tx_data['script'])),
wallet_service.script_to_addr(tx_data['script']), tx_data['value'])
def generate_podle_error_string(priv_utxo_pairs, to, ts, wallet, cjamount,
def generate_podle_error_string(priv_utxo_pairs, to, ts, wallet_service, cjamount,
taker_utxo_age, taker_utxo_amtpercent):
"""Gives detailed error information on why commitment sourcing failed.
"""
@ -64,9 +64,9 @@ def generate_podle_error_string(priv_utxo_pairs, to, ts, wallet, cjamount,
"with 'python add-utxo.py --help'\n\n")
errmsg += ("***\nFor reference, here are the utxos in your wallet:\n")
for md, utxos in wallet.get_utxos_by_mixdepth_().items():
for md, utxos in wallet_service.get_utxos_by_mixdepth(hexfmt=False).items():
if not utxos:
continue
errmsg += ("\nmixdepth {}:\n{}".format(
md, fmt_utxos(utxos, wallet, prefix=' ')))
md, fmt_utxos(utxos, wallet_service, prefix=' ')))
return (errmsgheader, errmsg)

6
jmclient/jmclient/storage.py

@ -217,6 +217,9 @@ class Storage(object):
with open(self.path, 'rb') as fh:
return fh.read()
def get_location(self):
return self.path
@staticmethod
def _serialize(data):
return bencoder.bencode(data)
@ -334,3 +337,6 @@ class VolatileStorage(Storage):
def _read_file(self):
return self.file_data
def get_location(self):
return None

112
jmclient/jmclient/taker.py

@ -7,17 +7,18 @@ from future.utils import iteritems
import base64
import pprint
import random
from twisted.internet import reactor
from twisted.internet import reactor, task
from binascii import hexlify, unhexlify
from jmbitcoin import SerializationError, SerializationTruncationError
import jmbitcoin as btc
from jmclient.configure import get_p2sh_vbyte, jm_single, validate_address
from jmclient.configure import jm_single, validate_address
from jmbase.support import get_log
from jmclient.support import (calc_cj_fee, weighted_order_choose, choose_orders,
choose_sweep_orders)
from jmclient.wallet import estimate_tx_fee, make_shuffled_tx
from jmclient.wallet import estimate_tx_fee
from jmclient.podle import generate_podle, get_podle_commitments, PoDLE
from jmclient.wallet_service import WalletService
from .output import generate_podle_error_string
from .cryptoengine import EngineError
@ -31,7 +32,7 @@ class JMTakerError(Exception):
class Taker(object):
def __init__(self,
wallet,
wallet_service,
schedule,
order_chooser=weighted_order_choose,
callbacks=None,
@ -80,7 +81,8 @@ class Taker(object):
None
"""
self.aborted = False
self.wallet = wallet
assert isinstance(wallet_service, WalletService)
self.wallet_service = wallet_service
self.schedule = schedule
self.order_chooser = order_chooser
self.max_cj_fee = max_cj_fee
@ -173,7 +175,7 @@ class Taker(object):
#mixdepth in tumble schedules:
if self.schedule_index == 0 or si[0] != self.schedule[
self.schedule_index - 1]:
self.mixdepthbal = self.wallet.get_balance_by_mixdepth(
self.mixdepthbal = self.wallet_service.get_balance_by_mixdepth(
)[self.mixdepth]
#reset to satoshis
self.cjamount = int(self.cjamount * self.mixdepthbal)
@ -183,15 +185,20 @@ class Taker(object):
self.cjamount = jm_single().mincjamount
self.n_counterparties = si[2]
self.my_cj_addr = si[3]
# for sweeps to external addresses we need an in-wallet import
# for the transaction monitor (this will be a no-op for txs to
# in-wallet addresses).
if self.cjamount == 0:
self.wallet_service.import_non_wallet_address(self.my_cj_addr)
#if destination is flagged "INTERNAL", choose a destination
#from the next mixdepth modulo the maxmixdepth
if self.my_cj_addr == "INTERNAL":
next_mixdepth = (self.mixdepth + 1) % (
self.wallet.mixdepth + 1)
self.wallet_service.mixdepth + 1)
jlog.info("Choosing a destination from mixdepth: " + str(
next_mixdepth))
self.my_cj_addr = self.wallet.get_internal_addr(next_mixdepth,
bci=jm_single().bc_interface)
self.my_cj_addr = self.wallet_service.get_internal_addr(next_mixdepth)
jlog.info("Chose destination address: " + self.my_cj_addr)
self.outputs = []
self.cjfee_total = 0
@ -277,8 +284,7 @@ class Taker(object):
self.my_change_addr = None
if self.cjamount != 0:
try:
self.my_change_addr = self.wallet.get_internal_addr(self.mixdepth,
bci=jm_single().bc_interface)
self.my_change_addr = self.wallet_service.get_internal_addr(self.mixdepth)
except:
self.taker_info_callback("ABORT", "Failed to get a change address")
return False
@ -291,15 +297,15 @@ class Taker(object):
total_amount = self.cjamount + self.total_cj_fee + self.total_txfee
jlog.info('total estimated amount spent = ' + str(total_amount))
try:
self.input_utxos = self.wallet.select_utxos(self.mixdepth,
total_amount)
self.input_utxos = self.wallet_service.select_utxos(self.mixdepth, total_amount,
minconfs=1)
except Exception as e:
self.taker_info_callback("ABORT",
"Unable to select sufficient coins: " + repr(e))
return False
else:
#sweep
self.input_utxos = self.wallet.get_utxos_by_mixdepth()[self.mixdepth]
self.input_utxos = self.wallet_service.get_utxos_by_mixdepth()[self.mixdepth]
#do our best to estimate the fee based on the number of
#our own utxos; this estimate may be significantly higher
#than the default set in option.txfee * makercount, where
@ -310,7 +316,7 @@ class Taker(object):
est_outs = 2*self.n_counterparties + 1
jlog.debug("Estimated outs: "+str(est_outs))
estimated_fee = estimate_tx_fee(est_ins, est_outs,
txtype=self.wallet.get_txtype())
txtype=self.wallet_service.get_txtype())
jlog.debug("We have a fee estimate: "+str(estimated_fee))
jlog.debug("And a requested fee of: "+str(
self.txfee_default * self.n_counterparties))
@ -387,7 +393,7 @@ class Taker(object):
auth_pub_bin = unhexlify(auth_pub)
for inp in utxo_data:
try:
if self.wallet.pubkey_has_script(
if self.wallet_service.pubkey_has_script(
auth_pub_bin, unhexlify(inp['script'])):
break
except EngineError:
@ -454,7 +460,7 @@ class Taker(object):
#Estimate fee per choice of next/3/6 blocks targetting.
estimated_fee = estimate_tx_fee(
len(sum(self.utxos.values(), [])), len(self.outputs) + 2,
txtype=self.wallet.get_txtype())
txtype=self.wallet_service.get_txtype())
jlog.info("Based on initial guess: " + str(self.total_txfee) +
", we estimated a miner fee of: " + str(estimated_fee))
#reset total
@ -495,7 +501,7 @@ class Taker(object):
for u in sum(self.utxos.values(), [])]
self.outputs.append({'address': self.coinjoin_address(),
'value': self.cjamount})
tx = make_shuffled_tx(self.utxo_tx, self.outputs, False)
tx = btc.make_shuffled_tx(self.utxo_tx, self.outputs, False)
jlog.info('obtained tx\n' + pprint.pformat(btc.deserialize(tx)))
self.latest_tx = btc.deserialize(tx)
@ -687,7 +693,7 @@ class Taker(object):
new_utxos_dict = {k: v for k, v in utxos.items() if k in new_utxos}
for k, v in iteritems(new_utxos_dict):
addr = v['address']
priv = self.wallet.get_key_from_addr(addr)
priv = self.wallet_service.get_key_from_addr(addr)
if priv: #can be null from create-unsigned
priv_utxo_pairs.append((priv, k))
return priv_utxo_pairs, too_old, too_small
@ -714,9 +720,9 @@ class Taker(object):
#in the transaction, about to be consumed, rather than use
#random utxos that will persist after. At this step we also
#allow use of external utxos in the json file.
if any(self.wallet.get_utxos_by_mixdepth_().values()):
if any(self.wallet_service.get_utxos_by_mixdepth(hexfmt=False).values()):
utxos = {}
for mdutxo in self.wallet.get_utxos_by_mixdepth().values():
for mdutxo in self.wallet_service.get_utxos_by_mixdepth().values():
utxos.update(mdutxo)
priv_utxo_pairs, to, ts = priv_utxo_pairs_from_utxos(
utxos, age, amt)
@ -740,7 +746,7 @@ class Taker(object):
"Commitment sourced OK")
else:
errmsgheader, errmsg = generate_podle_error_string(priv_utxo_pairs,
to, ts, self.wallet, self.cjamount,
to, ts, self.wallet_service, self.cjamount,
jm_single().config.get("POLICY", "taker_utxo_age"),
jm_single().config.get("POLICY", "taker_utxo_amtpercent"))
@ -766,10 +772,10 @@ class Taker(object):
utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
if utxo not in self.input_utxos.keys():
continue
script = self.wallet.addr_to_script(self.input_utxos[utxo]['address'])
script = self.wallet_service.addr_to_script(self.input_utxos[utxo]['address'])
amount = self.input_utxos[utxo]['value']
our_inputs[index] = (script, amount)
self.latest_tx = self.wallet.sign_tx(self.latest_tx, our_inputs)
self.latest_tx = self.wallet_service.sign_tx(self.latest_tx, our_inputs)
def push(self):
tx = btc.serialize(self.latest_tx)
@ -783,13 +789,21 @@ class Taker(object):
notify_addr = btc.address_to_script(self.my_cj_addr)
else:
notify_addr = self.my_cj_addr
#add the txnotify callbacks *before* pushing in case the
#walletnotify is triggered before the notify callbacks are set up;
#add the callbacks *before* pushing to ensure triggering;
#this does leave a dangling notify callback if the push fails, but
#that doesn't cause problems.
jm_single().bc_interface.add_tx_notify(self.latest_tx,
self.unconfirm_callback, self.confirm_callback,
notify_addr, vb=get_p2sh_vbyte())
self.wallet_service.register_callbacks([self.unconfirm_callback], self.txid,
"unconfirmed")
self.wallet_service.register_callbacks([self.confirm_callback], self.txid,
"confirmed")
task.deferLater(reactor,
float(jm_single().config.getint(
"TIMEOUT", "unconfirm_timeout_sec")),
self.wallet_service.check_callback_called,
self.txid, self.unconfirm_callback,
"unconfirmed",
"transaction with txid: " + str(self.txid) + " not broadcast.")
tx_broadcast = jm_single().config.get('POLICY', 'tx_broadcast')
nick_to_use = None
if tx_broadcast == 'self':
@ -820,22 +834,44 @@ class Taker(object):
self.self_sign()
return self.push()
def tx_match(self, txd):
# Takers process only in series, so this should not occur:
assert self.latest_tx is not None
# check if the transaction matches our created tx:
if txd['outs'] != self.latest_tx['outs']:
return False
return True
def unconfirm_callback(self, txd, txid):
if not self.tx_match(txd):
return False
jlog.info("Transaction seen on network, waiting for confirmation")
#To allow client to mark transaction as "done" (e.g. by persisting state)
self.on_finished_callback(True, fromtx="unconfirmed")
self.waiting_for_conf = True
confirm_timeout_sec = float(jm_single().config.get(
"TIMEOUT", "confirm_timeout_hours")) * 3600
task.deferLater(reactor, confirm_timeout_sec,
self.wallet_service.check_callback_called,
txid, self.confirm_callback, "confirmed",
"transaction with txid " + str(txid) + " not confirmed.")
return True
def confirm_callback(self, txd, txid, confirmations):
if not self.tx_match(txd):
return False
self.waiting_for_conf = False
if self.aborted:
#do not trigger on_finished processing (abort whole schedule)
return
#do not trigger on_finished processing (abort whole schedule),
# but we still return True as we have finished our listening
# for this tx:
return True
jlog.debug("Confirmed callback in taker, confs: " + str(confirmations))
fromtx=False if self.schedule_index + 1 == len(self.schedule) else True
waittime = self.schedule[self.schedule_index][4]
self.on_finished_callback(True, fromtx=fromtx, waittime=waittime,
txdetails=(txd, txid))
return True
class P2EPTaker(Taker):
""" The P2EP Taker will initialize its protocol directly
@ -845,8 +881,8 @@ class P2EPTaker(Taker):
improves the privacy of the operation.
"""
def __init__(self, counterparty, wallet, schedule, callbacks):
super(P2EPTaker, self).__init__(wallet, schedule, callbacks=callbacks)
def __init__(self, counterparty, wallet_service, schedule, callbacks):
super(P2EPTaker, self).__init__(wallet_service, schedule, callbacks=callbacks)
self.p2ep_receiver_nick = counterparty
# Callback to request user permission (for e.g. GUI)
# args: (1) message, as string
@ -951,7 +987,7 @@ class P2EPTaker(Taker):
# estimate the fee for the version of the transaction which is
# not coinjoined:
est_fee = estimate_tx_fee(len(self.input_utxos), 2,
txtype=self.wallet.get_txtype())
txtype=self.wallet_service.get_txtype())
my_change_value = my_total_in - self.cjamount - est_fee
if my_change_value <= 0:
# as discussed in initialize(), this should be an extreme edge case.
@ -973,7 +1009,7 @@ class P2EPTaker(Taker):
"getblockchaininfo", [])["blocks"]
# As for JM coinjoins, the `None` key is used for our own inputs
# to the transaction; this preparatory version contains only those.
tx = make_shuffled_tx(self.utxos[None], self.outputs,
tx = btc.make_shuffled_tx(self.utxos[None], self.outputs,
False, 2, currentblock)
jlog.info('Created proposed fallback tx\n' + pprint.pformat(
btc.deserialize(tx)))
@ -984,10 +1020,10 @@ class P2EPTaker(Taker):
dtx = btc.deserialize(tx)
for index, ins in enumerate(dtx['ins']):
utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
script = self.wallet.addr_to_script(self.input_utxos[utxo]['address'])
script = self.wallet_service.addr_to_script(self.input_utxos[utxo]['address'])
amount = self.input_utxos[utxo]['value']
our_inputs[index] = (script, amount)
self.signed_noncj_tx = btc.serialize(self.wallet.sign_tx(dtx, our_inputs))
self.signed_noncj_tx = btc.serialize(self.wallet_service.sign_tx(dtx, our_inputs))
self.taker_info_callback("INFO", "Built tx proposal, sending to receiver.")
return (True, [self.p2ep_receiver_nick], self.signed_noncj_tx)
@ -1153,7 +1189,7 @@ class P2EPTaker(Taker):
# in joinmarket.cfg; allow either (a) automatic agreement for any value within
# a range of 0.3 to 3x this figure, or (b) user to agree on prompt.
fee_est = estimate_tx_fee(len(tx['ins']), len(tx['outs']),
txtype=self.wallet.get_txtype())
txtype=self.wallet_service.get_txtype())
fee_ok = False
if btc_fee > 0.3 * fee_est and btc_fee < 3 * fee_est:
fee_ok = True

26
jmclient/jmclient/taker_utils.py

@ -20,7 +20,7 @@ Utility functions for tumbler-style takers;
Currently re-used by CLI script tumbler.py and joinmarket-qt
"""
def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False,
def direct_send(wallet_service, amount, mixdepth, destaddr, answeryes=False,
accept_callback=None, info_callback=None):
"""Send coins directly from one mixdepth to one destination address;
does not need IRC. Sweep as for normal sendpayment (set amount=0).
@ -46,12 +46,12 @@ def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False,
assert mixdepth >= 0
assert isinstance(amount, numbers.Integral)
assert amount >=0
assert isinstance(wallet, BaseWallet)
assert isinstance(wallet_service.wallet, BaseWallet)
from pprint import pformat
txtype = wallet.get_txtype()
txtype = wallet_service.get_txtype()
if amount == 0:
utxos = wallet.get_utxos_by_mixdepth()[mixdepth]
utxos = wallet_service.get_utxos_by_mixdepth()[mixdepth]
if utxos == {}:
log.error(
"There are no utxos in mixdepth: " + str(mixdepth) + ", quitting.")
@ -62,7 +62,7 @@ def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False,
else:
#8 inputs to be conservative
initial_fee_est = estimate_tx_fee(8,2, txtype=txtype)
utxos = wallet.select_utxos(mixdepth, amount + initial_fee_est)
utxos = wallet_service.select_utxos(mixdepth, amount + initial_fee_est)
if len(utxos) < 8:
fee_est = estimate_tx_fee(len(utxos), 2, txtype=txtype)
else:
@ -70,15 +70,14 @@ def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False,
total_inputs_val = sum([va['value'] for u, va in iteritems(utxos)])
changeval = total_inputs_val - fee_est - amount
outs = [{"value": amount, "address": destaddr}]
change_addr = wallet.get_internal_addr(mixdepth,
jm_single().bc_interface)
change_addr = wallet_service.get_internal_addr(mixdepth)
outs.append({"value": changeval, "address": change_addr})
#Now ready to construct transaction
log.info("Using a fee of : " + str(fee_est) + " satoshis.")
if amount != 0:
log.info("Using a change value of: " + str(changeval) + " satoshis.")
txsigned = sign_tx(wallet, mktx(list(utxos.keys()), outs), utxos)
txsigned = sign_tx(wallet_service, mktx(list(utxos.keys()), outs), utxos)
log.info("Got signed transaction:\n")
log.info(pformat(txsigned))
tx = serialize(txsigned)
@ -104,15 +103,15 @@ def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False,
return txid
def sign_tx(wallet, tx, utxos):
def sign_tx(wallet_service, tx, utxos):
stx = deserialize(tx)
our_inputs = {}
for index, ins in enumerate(stx['ins']):
utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
script = wallet.addr_to_script(utxos[utxo]['address'])
script = wallet_service.addr_to_script(utxos[utxo]['address'])
amount = utxos[utxo]['value']
our_inputs[index] = (script, amount)
return wallet.sign_tx(stx, our_inputs)
return wallet_service.sign_tx(stx, our_inputs)
def get_tumble_log(logsdir):
tumble_log = logging.getLogger('tumbler')
@ -178,7 +177,7 @@ def unconf_update(taker, schedulefile, tumble_log, addtolog=False):
#because addresses are not public until broadcast (whereas for makers,
#they are public *during* negotiation). So updating the cache here
#is sufficient
taker.wallet.update_cache_index()
taker.wallet_service.save_wallet()
#If honest-only was set, and we are going to continue (e.g. Tumbler),
#we switch off the honest-only filter. We also wipe the honest maker
@ -255,9 +254,6 @@ def tumbler_taker_finished_update(taker, schedulefile, tumble_log, options,
waiting_message = "Waiting for: " + str(waittime) + " minutes."
tumble_log.info(waiting_message)
log.info(waiting_message)
txd, txid = txdetails
taker.wallet.remove_old_utxos(txd)
taker.wallet.add_new_utxos(txd, txid)
else:
#a transaction failed, either because insufficient makers
#(acording to minimum_makers) responded in Phase 1, or not all

187
jmclient/jmclient/wallet.py

@ -4,7 +4,6 @@ from builtins import * # noqa: F401
from configparser import NoOptionError
import warnings
import random
import functools
import collections
import numbers
@ -19,6 +18,7 @@ from numbers import Integral
from .configure import jm_single
from .blockchaininterface import INF_HEIGHT
from .support import select_gradual, select_greedy, select_greediest, \
select
from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH,\
@ -26,6 +26,7 @@ from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH,\
from .support import get_random_bytes
from . import mn_encode, mn_decode
import jmbitcoin as btc
from jmbase import JM_WALLET_NAME_PREFIX
"""
@ -67,19 +68,6 @@ class Mnemonic(MnemonicParent):
def detect_language(cls, code):
return "english"
def make_shuffled_tx(ins, outs, deser=True, version=1, locktime=0):
""" Simple utility to ensure transaction
inputs and outputs are randomly ordered.
Can possibly be replaced by BIP69 in future
"""
random.shuffle(ins)
random.shuffle(outs)
tx = btc.mktx(ins, outs, version=version, locktime=locktime)
if deser:
return btc.deserialize(tx)
else:
return tx
def estimate_tx_fee(ins, outs, txtype='p2pkh'):
'''Returns an estimate of the number of satoshis required
for a transaction with the given number of inputs and outputs,
@ -123,7 +111,7 @@ class UTXOManager(object):
def __init__(self, storage, merge_func):
self.storage = storage
self.selector = merge_func
# {mixdexpth: {(txid, index): (path, value)}}
# {mixdexpth: {(txid, index): (path, value, height)}}
self._utxo = None
# metadata kept as a separate key in the database
# for backwards compat; value as dict for forward-compat.
@ -197,16 +185,20 @@ class UTXOManager(object):
return self._utxo[mixdepth].pop((txid, index))
def add_utxo(self, txid, index, path, value, mixdepth):
def add_utxo(self, txid, index, path, value, mixdepth, height=None):
# Assumed: that we add a utxo only if we want it enabled,
# so metadata is not currently added.
# The height (blockheight) field will be "infinity" for unconfirmed.
assert isinstance(txid, bytes)
assert len(txid) == self.TXID_LEN
assert isinstance(index, numbers.Integral)
assert isinstance(value, numbers.Integral)
assert isinstance(mixdepth, numbers.Integral)
if height is None:
height = INF_HEIGHT
assert isinstance(height, numbers.Integral)
self._utxo[mixdepth][(txid, index)] = (path, value)
self._utxo[mixdepth][(txid, index)] = (path, value, height)
def is_disabled(self, txid, index):
if not self._utxo_meta:
@ -231,28 +223,37 @@ class UTXOManager(object):
def enable_utxo(self, txid, index):
self.disable_utxo(txid, index, disable=False)
def select_utxos(self, mixdepth, amount, utxo_filter=(), select_fn=None):
def select_utxos(self, mixdepth, amount, utxo_filter=(), select_fn=None,
maxheight=None):
assert isinstance(mixdepth, numbers.Integral)
utxos = self._utxo[mixdepth]
# do not select anything in the filter
available = [{'utxo': utxo, 'value': val}
for utxo, (addr, val) in utxos.items() if utxo not in utxo_filter]
for utxo, (addr, val, height) in utxos.items() if utxo not in utxo_filter]
# do not select anything with insufficient confirmations:
if maxheight is not None:
available = [{'utxo': utxo, 'value': val}
for utxo, (addr, val, height) in utxos.items(
) if height <= maxheight]
# do not select anything disabled
available = [u for u in available if not self.is_disabled(*u['utxo'])]
selector = select_fn or self.selector
selected = selector(available, amount)
# note that we do not return height; for selection, we expect
# the caller will not want this (after applying the height filter)
return {s['utxo']: {'path': utxos[s['utxo']][0],
'value': utxos[s['utxo']][1]}
for s in selected}
def get_balance_by_mixdepth(self, max_mixdepth=float('Inf'),
include_disabled=True):
include_disabled=True, maxheight=None):
""" By default this returns a dict of aggregated bitcoin
balance per mixdepth: {0: N sats, 1: M sats, ...} for all
currently available mixdepths.
If max_mixdepth is set it will return balances only up
to that mixdepth.
To get only enabled balance, set include_disabled=False.
To get balances only with a certain number of confs, use maxheight.
"""
balance_dict = collections.defaultdict(int)
for mixdepth, utxomap in self._utxo.items():
@ -261,6 +262,9 @@ class UTXOManager(object):
if not include_disabled:
utxomap = {k: v for k, v in utxomap.items(
) if not self.is_disabled(*k)}
if maxheight is not None:
utxomap = {k: v for k, v in utxomap.items(
) if v[2] <= maxheight}
value = sum(x[1] for x in utxomap.values())
balance_dict[mixdepth] = value
return balance_dict
@ -345,6 +349,14 @@ class BaseWallet(object):
self.network = self._storage.data[b'network'].decode('ascii')
self._utxos = UTXOManager(self._storage, self.merge_algorithm)
def get_storage_location(self):
""" Return the location of the
persistent storage, if it exists, or None.
"""
if not self._storage:
return None
return self._storage.get_location()
def save(self):
"""
Write data to associated storage object and trigger persistent update.
@ -426,36 +438,25 @@ class BaseWallet(object):
privkey = self._get_priv_from_path(path)[0]
return hexlify(privkey).decode('ascii')
def _get_addr_int_ext(self, get_script_func, mixdepth, bci=None):
script = get_script_func(mixdepth)
addr = self.script_to_addr(script)
if bci is not None and hasattr(bci, 'import_addresses'):
assert hasattr(bci, 'get_wallet_name')
bci.import_addresses([addr], bci.get_wallet_name(self))
return addr
def _get_addr_int_ext(self, internal, mixdepth):
script = self.get_internal_script(mixdepth) if internal else \
self.get_external_script(mixdepth)
return self.script_to_addr(script)
def get_external_addr(self, mixdepth, bci=None):
def get_external_addr(self, mixdepth):
"""
Return an address suitable for external distribution, including funding
the wallet from other sources, or receiving payments or donations.
JoinMarket will never generate these addresses for internal use.
If the argument bci is non-null, we attempt to import the new
address into this blockchaininterface instance
(based on Bitcoin Core's model).
"""
return self._get_addr_int_ext(self.get_external_script, mixdepth,
bci=bci)
return self._get_addr_int_ext(False, mixdepth)
def get_internal_addr(self, mixdepth, bci=None):
def get_internal_addr(self, mixdepth):
"""
Return an address for internal usage, as change addresses and when
participating in transactions initiated by other parties.
If the argument bci is non-null, we attempt to import the new
address into this blockchaininterface instance
(based on Bitcoin Core's model).
"""
return self._get_addr_int_ext(self.get_internal_script, mixdepth,
bci=bci)
return self._get_addr_int_ext(True, mixdepth)
def get_external_script(self, mixdepth):
return self.get_new_script(mixdepth, False)
@ -575,7 +576,7 @@ class BaseWallet(object):
args:
tx: transaction dict
returns:
{(txid, index): {'script': bytes, 'value': int} for all removed utxos
{(txid, index): {'script': bytes, 'path': str, 'value': int} for all removed utxos
"""
removed_utxos = {}
for inp in tx['ins']:
@ -583,7 +584,7 @@ class BaseWallet(object):
md = self._utxos.have_utxo(txid, index)
if md is False:
continue
path, value = self._utxos.remove_utxo(txid, index, md)
path, value, height = self._utxos.remove_utxo(txid, index, md)
script = self.get_script_path(path)
removed_utxos[(txid, index)] = {'script': script,
'path': path,
@ -591,12 +592,12 @@ class BaseWallet(object):
return removed_utxos
@deprecated
def add_new_utxos(self, tx, txid):
def add_new_utxos(self, tx, txid, height=None):
tx = deepcopy(tx)
for out in tx['outs']:
out['script'] = unhexlify(out['script'])
ret = self.add_new_utxos_(tx, unhexlify(txid))
ret = self.add_new_utxos_(tx, unhexlify(txid), height=height)
added_utxos = {}
for (txid_bin, index), val in ret.items():
@ -605,12 +606,14 @@ class BaseWallet(object):
added_utxos[txid + ':' + str(index)] = val
return added_utxos
def add_new_utxos_(self, tx, txid):
def add_new_utxos_(self, tx, txid, height=None):
"""
Add all outputs of tx for this wallet to internal utxo list.
args:
tx: transaction dict
height: blockheight in which tx was included, or None
if unconfirmed.
returns:
{(txid, index): {'script': bytes, 'path': tuple, 'value': int}
for all added utxos
@ -619,7 +622,8 @@ class BaseWallet(object):
added_utxos = {}
for index, outs in enumerate(tx['outs']):
try:
self.add_utxo(txid, index, outs['script'], outs['value'])
self.add_utxo(txid, index, outs['script'], outs['value'],
height=height)
except WalletError:
continue
@ -629,7 +633,7 @@ class BaseWallet(object):
'value': outs['value']}
return added_utxos
def add_utxo(self, txid, index, script, value):
def add_utxo(self, txid, index, script, value, height=None):
assert isinstance(txid, bytes)
assert isinstance(index, Integral)
assert isinstance(script, bytes)
@ -640,15 +644,30 @@ class BaseWallet(object):
path = self.script_to_path(script)
mixdepth = self._get_mixdepth_from_path(path)
self._utxos.add_utxo(txid, index, path, value, mixdepth)
self._utxos.add_utxo(txid, index, path, value, mixdepth, height=height)
def process_new_tx(self, txd, txid, height=None):
""" Given a newly seen transaction, deserialized as txd and
with transaction id, process its inputs and outputs and update
the utxo contents of this wallet accordingly.
NOTE: this should correctly handle transactions that are not
actually related to the wallet; it will not add (or remove,
obviously) utxos that were not related since the underlying
functions check this condition.
"""
removed_utxos = self.remove_old_utxos(txd)
added_utxos = self.add_new_utxos(txd, txid, height=height)
return (removed_utxos, added_utxos)
@deprecated
def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None):
def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None,
maxheight=None):
utxo_filter_new = None
if utxo_filter:
utxo_filter_new = [(unhexlify(utxo[:64]), int(utxo[65:]))
for utxo in utxo_filter]
ret = self.select_utxos_(mixdepth, amount, utxo_filter_new, select_fn)
ret = self.select_utxos_(mixdepth, amount, utxo_filter_new, select_fn,
maxheight=maxheight)
ret_conv = {}
for utxo, data in ret.items():
addr = self.get_addr_path(data['path'])
@ -657,7 +676,7 @@ class BaseWallet(object):
return ret_conv
def select_utxos_(self, mixdepth, amount, utxo_filter=None,
select_fn=None):
select_fn=None, maxheight=None):
"""
Select a subset of available UTXOS for a given mixdepth whose value is
greater or equal to amount.
@ -667,6 +686,7 @@ class BaseWallet(object):
equal to wallet.max_mixdepth
amount: int, total minimum amount of all selected utxos
utxo_filter: list of (txid, index), utxos not to select
maxheight: only select utxos with blockheight <= this.
returns:
{(txid, index): {'script': bytes, 'path': tuple, 'value': int}}
@ -681,7 +701,7 @@ class BaseWallet(object):
assert isinstance(i[0], bytes)
assert isinstance(i[1], numbers.Integral)
ret = self._utxos.select_utxos(
mixdepth, amount, utxo_filter, select_fn)
mixdepth, amount, utxo_filter, select_fn, maxheight=maxheight)
for data in ret.values():
data['script'] = self.get_script_path(data['path'])
@ -701,20 +721,24 @@ class BaseWallet(object):
self._utxos.reset()
def get_balance_by_mixdepth(self, verbose=True,
include_disabled=False):
include_disabled=False,
maxheight=None):
"""
Get available funds in each active mixdepth.
By default ignores disabled utxos in calculation.
By default returns unconfirmed transactions, to filter
confirmations, set maxheight to max acceptable blockheight.
returns: {mixdepth: value}
"""
# TODO: verbose
return self._utxos.get_balance_by_mixdepth(max_mixdepth=self.mixdepth,
include_disabled=include_disabled)
include_disabled=include_disabled,
maxheight=maxheight)
@deprecated
def get_utxos_by_mixdepth(self, verbose=True):
def get_utxos_by_mixdepth(self, verbose=True, includeheight=False):
# TODO: verbose
ret = self.get_utxos_by_mixdepth_()
ret = self.get_utxos_by_mixdepth_(includeheight=includeheight)
utxos_conv = collections.defaultdict(dict)
for md, utxos in ret.items():
@ -725,13 +749,14 @@ class BaseWallet(object):
utxos_conv[md][utxo_str] = data
return utxos_conv
def get_utxos_by_mixdepth_(self, include_disabled=False):
def get_utxos_by_mixdepth_(self, include_disabled=False, includeheight=False):
"""
Get all UTXOs for active mixdepths.
returns:
{mixdepth: {(txid, index):
{'script': bytes, 'path': tuple, 'value': int}}}
(if `includeheight` is True, adds key 'height': int)
"""
mix_utxos = self._utxos.get_utxos_by_mixdepth()
@ -739,13 +764,15 @@ class BaseWallet(object):
for md, data in mix_utxos.items():
if md > self.mixdepth:
continue
for utxo, (path, value) in data.items():
for utxo, (path, value, height) in data.items():
if not include_disabled and self._utxos.is_disabled(*utxo):
continue
script = self.get_script_path(path)
script_utxos[md][utxo] = {'script': script,
'path': path,
'value': value}
if includeheight:
script_utxos[md][utxo]['height'] = height
return script_utxos
@classmethod
@ -841,6 +868,12 @@ class BaseWallet(object):
priv, engine = self._get_priv_from_path(path)
return engine.sign_message(priv, message)
def get_wallet_name(self):
""" Returns the name used as a label for this
specific Joinmarket wallet in Bitcoin Core.
"""
return JM_WALLET_NAME_PREFIX + self.get_wallet_id()
def get_wallet_id(self):
"""
Get a human-readable identifier for the wallet.
@ -926,6 +959,46 @@ class BaseWallet(object):
"""
raise NotImplementedError()
def rewind_wallet_indices(self, used_indices, saved_indices):
for md in used_indices:
for int_type in (0, 1):
index = max(used_indices[md][int_type],
saved_indices[md][int_type])
self.set_next_index(md, int_type, index, force=True)
def get_used_indices(self, addr_gen):
""" Returns a dict of max used indices for each branch in
the wallet, from the given addresses addr_gen, assuming
that they are known to the wallet.
"""
indices = {x: [0, 0] for x in range(self.max_mixdepth + 1)}
for addr in addr_gen:
if not self.is_known_addr(addr):
continue
md, internal, index = self.get_details(
self.addr_to_path(addr))
if internal not in (0, 1):
assert internal == 'imported'
continue
indices[md][internal] = max(indices[md][internal], index + 1)
return indices
def check_gap_indices(self, used_indices):
""" Return False if any of the provided indices (which should be
those seen from listtransactions as having been used, for
this wallet/label) are higher than the ones recorded in the index
cache."""
for md in used_indices:
for internal in (0, 1):
if used_indices[md][internal] >\
max(self.get_next_unused_index(md, internal), 0):
return False
return True
def close(self):
self._storage.close()

661
jmclient/jmclient/wallet_service.py

@ -0,0 +1,661 @@
#! /usr/bin/env python
from __future__ import (absolute_import, division,
print_function, unicode_literals)
import collections
import time
import ast
import binascii
from decimal import Decimal
from copy import deepcopy
from twisted.internet import reactor
from twisted.internet import task
from twisted.application.service import Service
from numbers import Integral
from jmclient.configure import jm_single, get_log
from jmclient.output import fmt_tx_data
from jmclient.jsonrpc import JsonRpcError
from jmclient.blockchaininterface import INF_HEIGHT
"""Wallet service
The purpose of this independent service is to allow
running applications to keep an up to date, asynchronous
view of the current state of its wallet, deferring any
polling mechanisms needed against the backend blockchain
interface here.
"""
jlog = get_log()
class WalletService(Service):
EXTERNAL_WALLET_LABEL = "joinmarket-notify"
def __init__(self, wallet):
# The two principal member variables
# are the blockchaininterface instance,
# which is currently global in JM but
# could be more flexible in future, and
# the JM wallet object.
self.bci = jm_single().bc_interface
# keep track of the quasi-real-time blockheight
# (updated in main monitor loop)
self.update_blockheight()
self.wallet = wallet
self.synced = False
# Dicts of registered callbacks, by type
# and then by txinfo, for events
# on transactions.
self.callbacks = {}
self.callbacks["all"] = []
self.callbacks["unconfirmed"] = {}
self.callbacks["confirmed"] = {}
self.restart_callback = None
# transactions we are actively monitoring,
# i.e. they are not new but we want to track:
self.active_txids = []
# to ensure transactions are only logged once:
self.logged_txids = []
def update_blockheight(self):
""" Can be called manually (on startup, or for tests)
but will be called as part of main monitoring
loop to ensure new transactions are added at
the right height.
"""
try:
self.current_blockheight = self.bci.rpc("getblockcount", [])
assert isinstance(self.current_blockheight, Integral)
except Exception as e:
jlog.error("Failure to get blockheight from Bitcoin Core:")
jlog.error(repr(e))
return
def startService(self):
""" Encapsulates start up actions.
Here wallet sync.
"""
super(WalletService, self).startService()
self.request_sync_wallet()
def stopService(self):
""" Encapsulates shut down actions.
Here shut down main tx monitoring loop.
"""
self.monitor_loop.stop()
super(WalletService, self).stopService()
def isRunning(self):
if self.running == 1:
return True
return False
def add_restart_callback(self, callback):
""" Sets the function that will be
called in the event that the wallet
sync completes with a restart warning.
The only argument is a message string,
which the calling function displays to
the user before quitting gracefully.
"""
self.restart_callback = callback
def request_sync_wallet(self):
""" Ensures wallet sync is complete
before the main event loop starts.
"""
d = task.deferLater(reactor, 0.0, self.sync_wallet)
d.addCallback(self.start_wallet_monitoring)
def register_callbacks(self, callbacks, txinfo, cb_type="all"):
""" Register callbacks that will be called by the
transaction monitor loop, on transactions stored under
our wallet label (see WalletService.get_wallet_name()).
Callback arguments are currently (txd, txid) and return
is boolean, except "confirmed" callbacks which have
arguments (txd, txid, confirmations).
Note that callbacks MUST correctly return True if they
recognized the transaction and processed it, and False
if not. The True return value will be used to remove
the callback from the list.
Arguments:
`callbacks` - a list of functions with signature as above
and return type boolean.
`txinfo` - either a txid expected for the transaction, if
known, or a tuple of the ordered output set, of the form
(('script': script), ('value': value), ..). This can be
constructed from jmbitcoin.deserialize output, key "outs",
using tuple(). See WalletService.transaction_monitor().
`cb_type` - must be one of "all", "unconfirmed", "confirmed";
the first type will be called back once for every new
transaction, the second only once when the number of
confirmations is 0, and the third only once when the number
of confirmations is > 0.
"""
if cb_type == "all":
# note that in this case, txid is ignored.
self.callbacks["all"].extend(callbacks)
elif cb_type in ["unconfirmed", "confirmed"]:
if txinfo not in self.callbacks[cb_type]:
self.callbacks[cb_type][txinfo] = []
self.callbacks[cb_type][txinfo].extend(callbacks)
else:
assert False, "Invalid argument: " + cb_type
def start_wallet_monitoring(self, syncresult):
""" Once the initialization of the service
(currently, means: wallet sync) is complete,
we start the main monitoring jobs of the
wallet service (currently, means: monitoring
all new transactions on the blockchain that
are recognised as belonging to the Bitcoin
Core wallet with the JM wallet's label).
"""
if not syncresult:
jlog.error("Failed to sync the bitcoin wallet. Shutting down.")
reactor.stop()
return
jlog.info("Starting transaction monitor in walletservice")
self.monitor_loop = task.LoopingCall(
self.transaction_monitor)
self.monitor_loop.start(5.0)
def import_non_wallet_address(self, address):
""" Used for keeping track of transactions which
have no in-wallet destinations. External wallet
label is used to avoid breaking fast sync (which
assumes label => wallet)
"""
if not self.bci.is_address_imported(address):
self.bci.import_addresses([address], self.EXTERNAL_WALLET_LABEL,
restart_cb=self.restart_callback)
def transaction_monitor(self):
"""Keeps track of any changes in the wallet (new transactions).
Intended to be run as a twisted task.LoopingCall so that this
Service is constantly in near-realtime sync with the blockchain.
"""
self.update_blockheight()
txlist = self.bci.list_transactions(100)
new_txs = []
for x in txlist:
# process either (a) a completely new tx or
# (b) a tx that reached unconf status but we are still
# waiting for conf (active_txids)
if x['txid'] in self.active_txids or x['txid'] not in self.old_txs:
new_txs.append(x)
# reset for next polling event:
self.old_txs = [x['txid'] for x in txlist]
for tx in new_txs:
txid = tx["txid"]
res = self.bci.get_transaction(txid)
if not res:
continue
confs = res["confirmations"]
if not isinstance(confs, Integral):
jlog.warning("Malformed gettx result: " + str(res))
continue
if confs < 0:
jlog.info(
"Transaction: " + txid + " has a conflict, abandoning.")
continue
if confs == 0:
height = None
else:
height = self.current_blockheight - confs + 1
txd = self.bci.get_deser_from_gettransaction(res)
if txd is None:
continue
removed_utxos, added_utxos = self.wallet.process_new_tx(txd, txid, height)
# TODO note that this log message will be missed if confirmation
# is absurdly fast, this is considered acceptable compared with
# additional complexity.
if txid not in self.logged_txids:
self.log_new_tx(removed_utxos, added_utxos, txid)
self.logged_txids.append(txid)
# first fire 'all' type callbacks, irrespective of if the
# transaction pertains to anything known (but must
# have correct label per above); filter on this Joinmarket wallet label,
# or the external monitoring label:
if "label" in tx and tx["label"] in [
self.EXTERNAL_WALLET_LABEL, self.get_wallet_name()]:
for f in self.callbacks["all"]:
# note we need no return value as we will never
# remove these from the list
f(txd, txid)
# The tuple given as the second possible key for the dict
# is such because dict keys must be hashable types, so a simple
# replication of the entries in the list tx["outs"], where tx
# was generated via jmbitcoin.deserialize, is unacceptable to
# Python, since they are dicts. However their keyset is deterministic
# so it is sufficient to convert these dicts to tuples with fixed
# ordering, thus it can be used as a key into the self.callbacks
# dicts. (This is needed because txid is not always available
# at the time of callback registration).
possible_keys = [txid, tuple(
(x["script"], x["value"]) for x in txd["outs"])]
# note that len(added_utxos) > 0 is not a sufficient condition for
# the tx being new, since wallet.add_new_utxos will happily re-add
# a utxo that already exists; but this does not cause re-firing
# of callbacks since we in these cases delete the callback after being
# called once.
# Note also that it's entirely possible that there are only removals,
# not additions, to the utxo set, specifically in sweeps to external
# addresses. In this case, since removal can by definition only
# happen once, we must allow entries in self.active_txids through the
# filter.
if len(added_utxos) > 0 or len(removed_utxos) > 0 \
or txid in self.active_txids:
if confs == 0:
for k in possible_keys:
if k in self.callbacks["unconfirmed"]:
for f in self.callbacks["unconfirmed"][k]:
# True implies success, implies removal:
if f(txd, txid):
self.callbacks["unconfirmed"][k].remove(f)
# keep monitoring for conf > 0:
self.active_txids.append(txid)
elif confs > 0:
for k in possible_keys:
if k in self.callbacks["confirmed"]:
for f in self.callbacks["confirmed"][k]:
if f(txd, txid, confs):
self.callbacks["confirmed"][k].remove(f)
if txid in self.active_txids:
self.active_txids.remove(txid)
def check_callback_called(self, txinfo, callback, cbtype, msg):
""" Intended to be a deferred Task to be scheduled some
set time after the callback was registered. "all" type
callbacks do not expire and are not included.
"""
assert cbtype in ["unconfirmed", "confirmed"]
if txinfo in self.callbacks[cbtype]:
if callback in self.callbacks[cbtype][txinfo]:
# the callback was not called, drop it and warn
self.callbacks[cbtype][txinfo].remove(callback)
# TODO - dangling txids in self.active_txids will
# be caused by this, but could also happen for
# other reasons; possibly add logic to ensure that
# this never occurs, although their presence should
# not cause a functional error.
jlog.info("Timed out: " + msg)
# if callback is not in the list, it was already
# processed and so do nothing.
def log_new_tx(self, removed_utxos, added_utxos, txid):
""" Changes to the wallet are logged at INFO level by
the WalletService.
"""
def report_changed(x, utxos):
if len(utxos.keys()) > 0:
jlog.info(x + ' utxos=\n{}'.format('\n'.join(
'{} - {}'.format(u, fmt_tx_data(tx_data, self))
for u, tx_data in utxos.items())))
report_changed("Removed", removed_utxos)
report_changed("Added", added_utxos)
""" Wallet syncing code
"""
def sync_wallet(self, fast=True):
""" Syncs wallet; note that if slow sync
requires multiple rounds this must be called
until self.synced is True.
Before starting the event loop, we cache
the current most recent transactions as
reported by the blockchain interface, since
we are interested in deltas.
"""
# If this is called when syncing already complete:
if self.synced:
return True
if fast:
self.sync_wallet_fast()
else:
self.sync_addresses()
self.sync_unspent()
# Don't attempt updates on transactions that existed
# before startup
self.old_txs = [x['txid'] for x in self.bci.list_transactions(100)]
return self.synced
def resync_wallet(self, fast=True):
""" The self.synced state is generally
updated to True, once, at the start of
a run of a particular program. Here we
can manually force re-sync.
"""
self.synced = False
self.sync_wallet(fast=fast)
def sync_wallet_fast(self):
"""Exploits the fact that given an index_cache,
all addresses necessary should be imported, so we
can just list all used addresses to find the right
index values.
"""
self.get_address_usages()
self.sync_unspent()
def get_address_usages(self):
"""Use rpc `listaddressgroupings` to locate all used
addresses in the account (whether spent or unspent outputs).
This will not result in a full sync if working with a new
Bitcoin Core instance, in which case "fast" should have been
specifically disabled by the user.
"""
wallet_name = self.get_wallet_name()
agd = self.bci.rpc('listaddressgroupings', [])
# flatten all groups into a single list; then, remove duplicates
fagd = (tuple(item) for sublist in agd for item in sublist)
# "deduplicated flattened address grouping data" = dfagd
dfagd = set(fagd)
used_addresses = set()
for addr_info in dfagd:
if len(addr_info) < 3 or addr_info[2] != wallet_name:
continue
used_addresses.add(addr_info[0])
# for a first run, import first chunk
if not used_addresses:
jlog.info("Detected new wallet, performing initial import")
# delegate inital address import to sync_addresses
# this should be fast because "getaddressesbyaccount" should return
# an empty list in this case
self.sync_addresses()
self.synced = True
return
# Wallet has been used; scan forwards.
jlog.debug("Fast sync in progress. Got this many used addresses: " + str(
len(used_addresses)))
# Need to have wallet.index point to the last used address
# Algo:
# 1. Scan batch 1 of each branch, record matched wallet addresses.
# 2. Check if all addresses in 'used addresses' have been matched, if
# so, break.
# 3. Repeat the above for batch 2, 3.. up to max 20 batches.
# 4. If after all 20 batches not all used addresses were matched,
# quit with error.
# 5. Calculate used indices.
# 6. If all used addresses were matched, set wallet index to highest
# found index in each branch and mark wallet sync complete.
# Rationale for this algo:
# Retrieving addresses is a non-zero computational load, so batching
# and then caching allows a small sync to complete *reasonably*
# quickly while a larger one is not really negatively affected.
# The downside is another free variable, batch size, but this need
# not be exposed to the user; it is not the same as gap limit, in fact,
# the concept of gap limit does not apply to this kind of sync, which
# *assumes* that the most recent usage of addresses is indeed recorded.
remaining_used_addresses = used_addresses.copy()
addresses, saved_indices = self.collect_addresses_init()
for addr in addresses:
remaining_used_addresses.discard(addr)
BATCH_SIZE = 100
MAX_ITERATIONS = 20
current_indices = deepcopy(saved_indices)
for j in range(MAX_ITERATIONS):
if not remaining_used_addresses:
break
for addr in \
self.collect_addresses_gap(gap_limit=BATCH_SIZE):
remaining_used_addresses.discard(addr)
# increase wallet indices for next iteration
for md in current_indices:
current_indices[md][0] += BATCH_SIZE
current_indices[md][1] += BATCH_SIZE
self.rewind_wallet_indices(current_indices, current_indices)
else:
self.rewind_wallet_indices(saved_indices, saved_indices)
raise Exception("Failed to sync in fast mode after 20 batches; "
"please re-try wallet sync with --recoversync flag.")
# creating used_indices on-the-fly would be more efficient, but the
# overall performance gain is probably negligible
used_indices = self.get_used_indices(used_addresses)
self.rewind_wallet_indices(used_indices, saved_indices)
self.synced = True
def sync_addresses(self):
""" Triggered by use of --recoversync option in scripts,
attempts a full scan of the blockchain without assuming
anything about past usages of addresses (does not use
wallet.index_cache as hint).
"""
jlog.debug("requesting detailed wallet history")
wallet_name = self.get_wallet_name()
addresses, saved_indices = self.collect_addresses_init()
try:
imported_addresses = set(self.bci.rpc('getaddressesbyaccount',
[wallet_name]))
except JsonRpcError:
if wallet_name in self.bci.rpc('listlabels', []):
imported_addresses = set(self.bci.rpc('getaddressesbylabel',
[wallet_name]).keys())
else:
imported_addresses = set()
if not addresses.issubset(imported_addresses):
self.bci.add_watchonly_addresses(addresses - imported_addresses,
wallet_name, self.restart_callback)
return
used_addresses_gen = (tx['address']
for tx in self.bci._yield_transactions(wallet_name)
if tx['category'] == 'receive')
used_indices = self.get_used_indices(used_addresses_gen)
jlog.debug("got used indices: {}".format(used_indices))
gap_limit_used = not self.check_gap_indices(used_indices)
self.rewind_wallet_indices(used_indices, saved_indices)
new_addresses = self.collect_addresses_gap()
if not new_addresses.issubset(imported_addresses):
jlog.debug("Syncing iteration finished, additional step required")
self.bci.add_watchonly_addresses(new_addresses - imported_addresses,
wallet_name, self.restart_callback)
self.synced = False
elif gap_limit_used:
jlog.debug("Syncing iteration finished, additional step required")
self.synced = False
else:
jlog.debug("Wallet successfully synced")
self.rewind_wallet_indices(used_indices, saved_indices)
self.synced = True
def sync_unspent(self):
st = time.time()
# block height needs to be real time for addition to our utxos:
current_blockheight = self.bci.rpc("getblockcount", [])
wallet_name = self.get_wallet_name()
self.reset_utxos()
listunspent_args = []
if 'listunspent_args' in jm_single().config.options('POLICY'):
listunspent_args = ast.literal_eval(jm_single().config.get(
'POLICY', 'listunspent_args'))
unspent_list = self.bci.rpc('listunspent', listunspent_args)
unspent_list = [x for x in unspent_list if "label" in x]
# filter on label, but note (a) in certain circumstances (in-
# wallet transfer) it is possible for the utxo to be labeled
# with the external label, and (b) the wallet will know if it
# belongs or not anyway (is_known_addr):
our_unspent_list = [x for x in unspent_list if x["label"] in [
wallet_name, self.EXTERNAL_WALLET_LABEL]]
for u in our_unspent_list:
if not self.is_known_addr(u['address']):
continue
self._add_unspent_utxo(u, current_blockheight)
et = time.time()
jlog.debug('bitcoind sync_unspent took ' + str((et - st)) + 'sec')
def _add_unspent_utxo(self, utxo, current_blockheight):
"""
Add a UTXO as returned by rpc's listunspent call to the wallet.
params:
utxo: single utxo dict as returned by listunspent
current_blockheight: blockheight as integer, used to
set the block in which a confirmed utxo is included.
"""
txid = binascii.unhexlify(utxo['txid'])
script = binascii.unhexlify(utxo['scriptPubKey'])
value = int(Decimal(str(utxo['amount'])) * Decimal('1e8'))
confs = int(utxo['confirmations'])
# wallet's utxo database needs to store an absolute rather
# than relative height measure:
height = None
if confs < 0:
jlog.warning("Utxo not added, has a conflict: " + str(utxo))
return
if confs >=1 :
height = current_blockheight - confs + 1
self.add_utxo(txid, int(utxo['vout']), script, value, height)
""" The following functions mostly are not pure
pass through to the underlying wallet, so declared
here; the remainder are covered by the __getattr__
fallback.
"""
def save_wallet(self):
self.wallet.save()
def get_utxos_by_mixdepth(self, include_disabled=False,
verbose=False, hexfmt=True, includeconfs=False):
""" Returns utxos by mixdepth in a dict, optionally including
information about how many confirmations each utxo has.
TODO clean up underlying wallet.get_utxos_by_mixdepth (including verbosity
and formatting options) to make this less confusing.
"""
def height_to_confs(x):
# convert height entries to confirmations:
ubym_conv = collections.defaultdict(dict)
for m, i in x.items():
for u, d in i.items():
ubym_conv[m][u] = d
h = ubym_conv[m][u].pop("height")
if h == INF_HEIGHT:
confs = 0
else:
confs = self.current_blockheight - h + 1
ubym_conv[m][u]["confs"] = confs
return ubym_conv
if hexfmt:
ubym = self.wallet.get_utxos_by_mixdepth(verbose=verbose,
includeheight=includeconfs)
if not includeconfs:
return ubym
else:
return height_to_confs(ubym)
else:
ubym = self.wallet.get_utxos_by_mixdepth_(
include_disabled=include_disabled, includeheight=includeconfs)
if not includeconfs:
return ubym
else:
return height_to_confs(ubym)
def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None,
minconfs=None):
""" Request utxos from the wallet in a particular mixdepth to satisfy
a certain total amount, optionally set the selector function (or use
the currently configured function set by the wallet, and optionally
require a minimum of minconfs confirmations (default none means
unconfirmed are allowed).
"""
if minconfs is None:
maxheight = None
else:
maxheight = self.current_blockheight - minconfs + 1
return self.wallet.select_utxos(mixdepth, amount, utxo_filter=utxo_filter,
select_fn=select_fn, maxheight=maxheight)
def get_balance_by_mixdepth(self, verbose=True,
include_disabled=False,
minconfs=None):
if minconfs is None:
maxheight = None
else:
maxheight = self.current_blockheight - minconfs + 1
return self.wallet.get_balance_by_mixdepth(verbose=verbose,
include_disabled=include_disabled,
maxheight=maxheight)
def get_internal_addr(self, mixdepth):
if self.bci is not None and hasattr(self.bci, 'import_addresses'):
addr = self.wallet.get_internal_addr(mixdepth)
self.bci.import_addresses([addr],
self.wallet.get_wallet_name())
return addr
def collect_addresses_init(self):
""" Collects the "current" set of addresses,
as defined by the indices recorded in the wallet's
index cache (persisted in the wallet file usually).
Note that it collects up to the current indices plus
the gap limit.
"""
addresses = set()
saved_indices = dict()
for md in range(self.max_mixdepth + 1):
saved_indices[md] = [0, 0]
for internal in (0, 1):
next_unused = self.get_next_unused_index(md, internal)
for index in range(next_unused):
addresses.add(self.get_addr(md, internal, index))
for index in range(self.gap_limit):
addresses.add(self.get_new_addr(md, internal))
# reset the indices to the value we had before the
# new address calls:
self.set_next_index(md, internal, next_unused)
saved_indices[md][internal] = next_unused
# include any imported addresses
for path in self.yield_imported_paths(md):
addresses.add(self.get_addr_path(path))
return addresses, saved_indices
def collect_addresses_gap(self, gap_limit=None):
gap_limit = gap_limit or self.gap_limit
addresses = set()
for md in range(self.max_mixdepth + 1):
for internal in (True, False):
old_next = self.get_next_unused_index(md, internal)
for index in range(gap_limit):
addresses.add(self.get_new_addr(md, internal))
self.set_next_index(md, internal, old_next)
return addresses
def get_external_addr(self, mixdepth):
if self.bci is not None and hasattr(self.bci, 'import_addresses'):
addr = self.wallet.get_external_addr(mixdepth)
self.bci.import_addresses([addr],
self.wallet.get_wallet_name())
return addr
def __getattr__(self, attr):
# any method not present here is passed
# to the wallet:
return getattr(self.wallet, attr)

99
jmclient/jmclient/wallet_utils.py

@ -13,9 +13,10 @@ from numbers import Integral
from collections import Counter
from itertools import islice
from jmclient import (get_network, WALLET_IMPLEMENTATIONS, Storage, podle,
jm_single, BitcoinCoreInterface, JsonRpcError, sync_wallet, WalletError,
jm_single, BitcoinCoreInterface, JsonRpcError, WalletError,
VolatileStorage, StoragePasswordError, is_segwit_mode, SegwitLegacyWallet,
LegacyWallet, SegwitWallet, is_native_segwit_mode)
from jmclient.wallet_service import WalletService
from jmbase.support import get_password, jmprint
from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH
from .output import fmt_utxo
@ -77,12 +78,12 @@ def get_wallettool_parser():
default=1,
help=('History method verbosity, 0 (least) to 6 (most), '
'<=2 batches earnings, even values also list TXIDs'))
parser.add_option('--fast',
parser.add_option('--recoversync',
action='store_true',
dest='fastsync',
dest='recoversync',
default=False,
help=('choose to do fast wallet sync, only for Core and '
'only for previously synced wallet'))
help=('choose to do detailed wallet sync, '
'used for recovering on new Core instance.'))
parser.add_option('-H',
'--hd',
action='store',
@ -328,22 +329,22 @@ def get_tx_info(txid):
rpctx.get('blocktime', 0), txd
def get_imported_privkey_branch(wallet, m, showprivkey):
def get_imported_privkey_branch(wallet_service, m, showprivkey):
entries = []
for path in wallet.yield_imported_paths(m):
addr = wallet.get_addr_path(path)
script = wallet.get_script_path(path)
for path in wallet_service.yield_imported_paths(m):
addr = wallet_service.get_addr_path(path)
script = wallet_service.get_script_path(path)
balance = 0.0
for data in wallet.get_utxos_by_mixdepth_(
include_disabled=True)[m].values():
for data in wallet_service.get_utxos_by_mixdepth(include_disabled=True,
hexfmt=False)[m].values():
if script == data['script']:
balance += data['value']
used = ('used' if balance > 0.0 else 'empty')
if showprivkey:
wip_privkey = wallet.get_wif_path(path)
wip_privkey = wallet_service.get_wif_path(path)
else:
wip_privkey = ''
entries.append(WalletViewEntry(wallet.get_path_repr(path), m, -1,
entries.append(WalletViewEntry(wallet_service.get_path_repr(path), m, -1,
0, addr, [balance, balance],
used=used, priv=wip_privkey))
@ -377,7 +378,7 @@ def wallet_showutxos(wallet, showprivkey):
return json.dumps(unsp, indent=4)
def wallet_display(wallet, gaplimit, showprivkey, displayall=False,
def wallet_display(wallet_service, gaplimit, showprivkey, displayall=False,
serialized=True, summarized=False):
"""build the walletview object,
then return its serialization directly if serialized,
@ -412,45 +413,45 @@ def wallet_display(wallet, gaplimit, showprivkey, displayall=False,
acctlist = []
# TODO - either optionally not show disabled utxos, or
# mark them differently in display (labels; colors)
utxos = wallet.get_utxos_by_mixdepth_(include_disabled=True)
for m in range(wallet.mixdepth + 1):
utxos = wallet_service.get_utxos_by_mixdepth(include_disabled=True, hexfmt=False)
for m in range(wallet_service.mixdepth + 1):
branchlist = []
for forchange in [0, 1]:
entrylist = []
if forchange == 0:
# users would only want to hand out the xpub for externals
xpub_key = wallet.get_bip32_pub_export(m, forchange)
xpub_key = wallet_service.get_bip32_pub_export(m, forchange)
else:
xpub_key = ""
unused_index = wallet.get_next_unused_index(m, forchange)
unused_index = wallet_service.get_next_unused_index(m, forchange)
for k in range(unused_index + gaplimit):
path = wallet.get_path(m, forchange, k)
addr = wallet.get_addr_path(path)
path = wallet_service.get_path(m, forchange, k)
addr = wallet_service.get_addr_path(path)
balance, used = get_addr_status(
path, utxos[m], k >= unused_index, forchange)
if showprivkey:
privkey = wallet.get_wif_path(path)
privkey = wallet_service.get_wif_path(path)
else:
privkey = ''
if (displayall or balance > 0 or
(used == 'new' and forchange == 0)):
entrylist.append(WalletViewEntry(
wallet.get_path_repr(path), m, forchange, k, addr,
wallet_service.get_path_repr(path), m, forchange, k, addr,
[balance, balance], priv=privkey, used=used))
wallet.set_next_index(m, forchange, unused_index)
path = wallet.get_path_repr(wallet.get_path(m, forchange))
wallet_service.set_next_index(m, forchange, unused_index)
path = wallet_service.get_path_repr(wallet_service.get_path(m, forchange))
branchlist.append(WalletViewBranch(path, m, forchange, entrylist,
xpub=xpub_key))
ipb = get_imported_privkey_branch(wallet, m, showprivkey)
ipb = get_imported_privkey_branch(wallet_service, m, showprivkey)
if ipb:
branchlist.append(ipb)
#get the xpub key of the whole account
xpub_account = wallet.get_bip32_pub_export(mixdepth=m)
path = wallet.get_path_repr(wallet.get_path(m))
xpub_account = wallet_service.get_bip32_pub_export(mixdepth=m)
path = wallet_service.get_path_repr(wallet_service.get_path(m))
acctlist.append(WalletViewAccount(path, m, branchlist,
xpub=xpub_account))
path = wallet.get_path_repr(wallet.get_path())
path = wallet_service.get_path_repr(wallet_service.get_path())
walletview = WalletView(path, acctlist)
if serialized:
return walletview.serialize(summarize=summarized)
@ -598,7 +599,7 @@ def wallet_fetch_history(wallet, options):
tx_db.execute("CREATE TABLE transactions(txid TEXT, "
"blockhash TEXT, blocktime INTEGER, conflicts INTEGER);")
jm_single().debug_silence[0] = True
wallet_name = jm_single().bc_interface.get_wallet_name(wallet)
wallet_name = wallet.get_wallet_name()
buf = range(1000)
t = 0
while len(buf) == 1000:
@ -850,8 +851,8 @@ def wallet_fetch_history(wallet, options):
jmprint(('BUG ERROR: wallet balance (%s) does not match balance from ' +
'history (%s)') % (sat_to_str(total_wallet_balance),
sat_to_str(balance)))
wallet_utxo_count = sum(map(len, wallet.get_utxos_by_mixdepth_(
include_disabled=True).values()))
wallet_utxo_count = sum(map(len, wallet.get_utxos_by_mixdepth(
include_disabled=True, hexfmt=False).values()))
if utxo_count + unconfirmed_utxo_count != wallet_utxo_count:
jmprint(('BUG ERROR: wallet utxo count (%d) does not match utxo count from ' +
'history (%s)') % (wallet_utxo_count, utxo_count))
@ -969,11 +970,11 @@ def display_utxos_for_disable_choice_default(utxos_enabled, utxos_disabled):
disable = False if chosen_idx <= disabled_max else True
return ulist[chosen_idx], disable
def get_utxos_enabled_disabled(wallet, md):
def get_utxos_enabled_disabled(wallet_service, md):
""" Returns dicts for enabled and disabled separately
"""
utxos_enabled = wallet.get_utxos_by_mixdepth_()[md]
utxos_all = wallet.get_utxos_by_mixdepth_(include_disabled=True)[md]
utxos_enabled = wallet_service.get_utxos_by_mixdepth(hexfmt=False)[md]
utxos_all = wallet_service.get_utxos_by_mixdepth(include_disabled=True, hexfmt=False)[md]
utxos_disabled_keyset = set(utxos_all).difference(set(utxos_enabled))
utxos_disabled = {}
for u in utxos_disabled_keyset:
@ -1207,28 +1208,34 @@ def wallet_tool_main(wallet_root_path):
wallet_path, seed, options.mixdepth, read_only=read_only,
gap_limit=options.gaplimit)
# this object is only to respect the layering,
# the service will not be started since this is a synchronous script:
wallet_service = WalletService(wallet)
if method not in noscan_methods:
# if nothing was configured, we override bitcoind's options so that
# unconfirmed balance is included in the wallet display by default
if 'listunspent_args' not in jm_single().config.options('POLICY'):
jm_single().config.set('POLICY','listunspent_args', '[0]')
while not jm_single().bc_interface.wallet_synced:
sync_wallet(wallet, fast=options.fastsync)
while True:
if wallet_service.sync_wallet(fast = not options.recoversync):
break
#Now the wallet/data is prepared, execute the script according to the method
if method == "display":
return wallet_display(wallet, options.gaplimit, options.showprivkey)
return wallet_display(wallet_service, options.gaplimit, options.showprivkey)
elif method == "displayall":
return wallet_display(wallet, options.gaplimit, options.showprivkey,
return wallet_display(wallet_service, options.gaplimit, options.showprivkey,
displayall=True)
elif method == "summary":
return wallet_display(wallet, options.gaplimit, options.showprivkey, summarized=True)
return wallet_display(wallet_service, options.gaplimit, options.showprivkey, summarized=True)
elif method == "history":
if not isinstance(jm_single().bc_interface, BitcoinCoreInterface):
jmprint('showing history only available when using the Bitcoin Core ' +
'blockchain interface', "error")
sys.exit(0)
else:
return wallet_fetch_history(wallet, options)
return wallet_fetch_history(wallet_service, options)
elif method == "generate":
retval = wallet_generate_recover("generate", wallet_root_path,
mixdepth=options.mixdepth)
@ -1238,22 +1245,22 @@ def wallet_tool_main(wallet_root_path):
mixdepth=options.mixdepth)
return "Recovered wallet OK" if retval else "Failed"
elif method == "showutxos":
return wallet_showutxos(wallet, options.showprivkey)
return wallet_showutxos(wallet_service, options.showprivkey)
elif method == "showseed":
return wallet_showseed(wallet)
return wallet_showseed(wallet_service)
elif method == "dumpprivkey":
return wallet_dumpprivkey(wallet, options.hd_path)
return wallet_dumpprivkey(wallet_service, options.hd_path)
elif method == "importprivkey":
#note: must be interactive (security)
if options.mixdepth is None:
parser.error("You need to specify a mixdepth with -m")
wallet_importprivkey(wallet, options.mixdepth,
wallet_importprivkey(wallet_service, options.mixdepth,
map_key_type(options.key_type))
return "Key import completed."
elif method == "signmessage":
return wallet_signmessage(wallet, options.hd_path, args[2])
return wallet_signmessage(wallet_service, options.hd_path, args[2])
elif method == "freeze":
return wallet_freezeutxo(wallet, options.mixdepth)
return wallet_freezeutxo(wallet_service, options.mixdepth)
else:
parser.error("Unknown wallet-tool method: " + method)
sys.exit(0)

53
jmclient/jmclient/yieldgenerator.py

@ -12,11 +12,13 @@ from twisted.python.log import startLogging
from optparse import OptionParser
from jmbase import get_log
from jmclient import Maker, jm_single, load_program_config, \
sync_wallet, JMClientProtocolFactory, start_reactor, calc_cj_fee
JMClientProtocolFactory, start_reactor, \
calc_cj_fee, WalletService
from .wallet_utils import open_test_wallet_maybe, get_wallet_path
jlog = get_log()
MAX_MIX_DEPTH = 5
class YieldGenerator(Maker):
"""A maker for the purposes of generating a yield from held
@ -26,8 +28,8 @@ class YieldGenerator(Maker):
__metaclass__ = abc.ABCMeta
statement_file = os.path.join('logs', 'yigen-statement.csv')
def __init__(self, wallet):
Maker.__init__(self, wallet)
def __init__(self, wallet_service):
Maker.__init__(self, wallet_service)
self.tx_unconfirm_timestamp = {}
if not os.path.isfile(self.statement_file):
self.log_statement(
@ -44,7 +46,7 @@ class YieldGenerator(Maker):
self.income_statement.write(','.join(data) + '\n')
self.income_statement.close()
def on_tx_unconfirmed(self, offer, txid, removed_utxos):
def on_tx_unconfirmed(self, offer, txid):
self.tx_unconfirm_timestamp[offer["cjaddr"]] = int(time.time())
newoffers = self.create_my_orders()
@ -70,10 +72,10 @@ class YieldGeneratorBasic(YieldGenerator):
It will often (but not always) reannounce orders after transactions,
thus is somewhat suboptimal in giving more information to spies.
"""
def __init__(self, wallet, offerconfig):
def __init__(self, wallet_service, offerconfig):
self.txfee, self.cjfee_a, self.cjfee_r, self.ordertype, self.minsize \
= offerconfig
super(YieldGeneratorBasic,self).__init__(wallet)
super(YieldGeneratorBasic,self).__init__(wallet_service)
def create_my_orders(self):
mix_balance = self.get_available_mixdepths()
@ -129,10 +131,9 @@ class YieldGeneratorBasic(YieldGenerator):
return None, None, None
jlog.info('sending output to address=' + str(cj_addr))
change_addr = self.wallet.get_internal_addr(mixdepth,
jm_single().bc_interface)
change_addr = self.wallet_service.get_internal_addr(mixdepth)
utxos = self.wallet.select_utxos(mixdepth, total_amount)
utxos = self.wallet_service.select_utxos(mixdepth, total_amount, minconfs=1)
my_total_in = sum([va['value'] for va in utxos.values()])
real_cjfee = calc_cj_fee(offer["ordertype"], offer["cjfee"], amount)
change_value = my_total_in - amount - offer["txfee"] + real_cjfee
@ -140,8 +141,8 @@ class YieldGeneratorBasic(YieldGenerator):
jlog.debug(('change value={} below dust threshold, '
'finding new utxos').format(change_value))
try:
utxos = self.wallet.select_utxos(
mixdepth, total_amount + jm_single().DUST_THRESHOLD)
utxos = self.wallet_service.select_utxos(mixdepth,
total_amount + jm_single().DUST_THRESHOLD, minconfs=1)
except Exception:
jlog.info('dont have the required UTXOs to make a '
'output above the dust threshold, quitting')
@ -149,7 +150,7 @@ class YieldGeneratorBasic(YieldGenerator):
return utxos, cj_addr, change_addr
def on_tx_confirmed(self, offer, confirmations, txid):
def on_tx_confirmed(self, offer, txid, confirmations):
if offer["cjaddr"] in self.tx_unconfirm_timestamp:
confirm_time = int(time.time()) - self.tx_unconfirm_timestamp[
offer["cjaddr"]]
@ -162,12 +163,13 @@ class YieldGeneratorBasic(YieldGenerator):
offer["utxos"]), sum([av['value'] for av in offer["utxos"].values(
)]), real_cjfee, real_cjfee - offer["offer"]["txfee"], round(
confirm_time / 60.0, 2), ''])
return self.on_tx_unconfirmed(offer, txid, None)
return self.on_tx_unconfirmed(offer, txid)
def get_available_mixdepths(self):
"""Returns the mixdepth/balance dict from the wallet that contains
all available inputs for offers."""
return self.wallet.get_balance_by_mixdepth(verbose=False)
return self.wallet_service.get_balance_by_mixdepth(verbose=False,
minconfs=1)
def select_input_mixdepth(self, available, offer, amount):
"""Returns the mixdepth from which the given order should spend the
@ -182,8 +184,8 @@ class YieldGeneratorBasic(YieldGenerator):
an order spending from the given input mixdepth. Can return None if
there is no suitable output, in which case the order is
aborted."""
cjoutmix = (input_mixdepth + 1) % (self.wallet.mixdepth + 1)
return self.wallet.get_internal_addr(cjoutmix, jm_single().bc_interface)
cjoutmix = (input_mixdepth + 1) % (self.wallet_service.mixdepth + 1)
return self.wallet_service.get_internal_addr(cjoutmix)
def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffer',
@ -209,12 +211,12 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffe
parser.add_option('-g', '--gap-limit', action='store', type="int",
dest='gaplimit', default=gaplimit,
help='gap limit for wallet, default='+str(gaplimit))
parser.add_option('--fast',
parser.add_option('--recoversync',
action='store_true',
dest='fastsync',
dest='recoversync',
default=False,
help=('choose to do fast wallet sync, only for Core and '
'only for previously synced wallet'))
help=('choose to do detailed wallet sync, '
'used for recovering on new Core instance.'))
parser.add_option('-m', '--mixdepth', action='store', type='int',
dest='mixdepth', default=None,
help="highest mixdepth to use")
@ -248,13 +250,12 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffe
wallet_path, wallet_name, options.mixdepth,
gap_limit=options.gaplimit)
if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == "electrum-server":
jm_single().bc_interface.synctype = "with-script"
wallet_service = WalletService(wallet)
while not wallet_service.synced:
wallet_service.sync_wallet(fast=not options.recoversync)
wallet_service.startService()
while not jm_single().bc_interface.wallet_synced:
sync_wallet(wallet, fast=options.fastsync)
maker = ygclass(wallet, [options.txfee, cjfee_a, cjfee_r,
maker = ygclass(wallet_service, [options.txfee, cjfee_a, cjfee_r,
options.ordertype, options.minsize])
jlog.info('starting yield generator')
clientfactory = JMClientProtocolFactory(maker, proto_type="MAKER")

46
jmclient/test/commontest.py

@ -12,7 +12,8 @@ from decimal import Decimal
from jmbase import get_log
from jmclient import (
jm_single, open_test_wallet_maybe, estimate_tx_fee,
BlockchainInterface, get_p2sh_vbyte, BIP32Wallet, SegwitLegacyWallet)
BlockchainInterface, get_p2sh_vbyte, BIP32Wallet,
SegwitLegacyWallet, WalletService)
from jmbase.support import chunks
import jmbitcoin as btc
@ -30,24 +31,15 @@ class DummyBlockchainInterface(BlockchainInterface):
self.fake_query_results = None
self.qusfail = False
def rpc(self, a, b):
return None
def sync_addresses(self, wallet):
pass
def sync_unspent(self, wallet):
pass
def import_addresses(self, addr_list, wallet_name):
def import_addresses(self, addr_list, wallet_name, restart_cb=None):
pass
def outputs_watcher(self, wallet_name, notifyaddr,
tx_output_set, uf, cf, tf):
pass
def tx_watcher(self, txd, ucf, cf, sf, c, n):
pass
def add_tx_notify(self,
txd,
unconfirmfun,
confirmfun,
notifyaddr,
timeoutfun=None,
vb=None):
def is_address_imported(self, addr):
pass
def pushtx(self, txhex):
@ -127,7 +119,7 @@ def binarize_tx(tx):
def make_sign_and_push(ins_full,
wallet,
wallet_service,
amount,
output_addr=None,
change_addr=None,
@ -136,11 +128,12 @@ def make_sign_and_push(ins_full,
"""Utility function for easily building transactions
from wallets
"""
assert isinstance(wallet_service, WalletService)
total = sum(x['value'] for x in ins_full.values())
ins = list(ins_full.keys())
#random output address and change addr
output_addr = wallet.get_new_addr(1, 1) if not output_addr else output_addr
change_addr = wallet.get_new_addr(1, 0) if not change_addr else change_addr
output_addr = wallet_service.get_new_addr(1, 1) if not output_addr else output_addr
change_addr = wallet_service.get_new_addr(0, 1) if not change_addr else change_addr
fee_est = estimate_tx_fee(len(ins), 2) if estimate_fee else 10000
outs = [{'value': amount,
'address': output_addr}, {'value': total - amount - fee_est,
@ -150,15 +143,18 @@ def make_sign_and_push(ins_full,
scripts = {}
for index, ins in enumerate(de_tx['ins']):
utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
script = wallet.addr_to_script(ins_full[utxo]['address'])
script = wallet_service.addr_to_script(ins_full[utxo]['address'])
scripts[index] = (script, ins_full[utxo]['value'])
binarize_tx(de_tx)
de_tx = wallet.sign_tx(de_tx, scripts, hashcode=hashcode)
de_tx = wallet_service.sign_tx(de_tx, scripts, hashcode=hashcode)
#pushtx returns False on any error
push_succeed = jm_single().bc_interface.pushtx(btc.serialize(de_tx))
if push_succeed:
removed = wallet.remove_old_utxos(de_tx)
return btc.txhash(btc.serialize(de_tx))
txid = btc.txhash(btc.serialize(de_tx))
# in normal operation this happens automatically
# but in some tests there is no monitoring loop:
wallet_service.process_new_tx(de_tx, txid)
return txid
else:
return False
@ -194,17 +190,17 @@ def make_wallets(n,
w = open_test_wallet_maybe(seeds[i], seeds[i], mixdepths - 1,
test_wallet_cls=wallet_cls)
wallet_service = WalletService(w)
wallets[i + start_index] = {'seed': seeds[i],
'wallet': w}
'wallet': wallet_service}
for j in range(mixdepths):
for k in range(wallet_structures[i][j]):
deviation = sdev_amt * random.random()
amt = mean_amt - sdev_amt / 2.0 + deviation
if amt < 0: amt = 0.001
amt = float(Decimal(amt).quantize(Decimal(10)**-8))
jm_single().bc_interface.grab_coins(
w.get_new_addr(j, populate_internal), amt)
jm_single().bc_interface.grab_coins(wallet_service.get_new_addr(
j, populate_internal), amt)
return wallets

113
jmclient/test/test_blockchaininterface.py

@ -9,16 +9,16 @@ from commontest import create_wallet_for_sync
import pytest
from jmbase import get_log
from jmclient import load_program_config, jm_single, sync_wallet
from jmclient import load_program_config, jm_single
log = get_log()
def sync_test_wallet(fast, wallet):
def sync_test_wallet(fast, wallet_service):
sync_count = 0
jm_single().bc_interface.wallet_synced = False
while not jm_single().bc_interface.wallet_synced:
sync_wallet(wallet, fast=fast)
wallet_service.synced = False
while not wallet_service.synced:
wallet_service.sync_wallet(fast=fast)
sync_count += 1
# avoid infinite loop
assert sync_count < 10
@ -27,15 +27,15 @@ def sync_test_wallet(fast, wallet):
@pytest.mark.parametrize('fast', (False, True))
def test_empty_wallet_sync(setup_wallets, fast):
wallet = create_wallet_for_sync([0, 0, 0, 0, 0], ['test_empty_wallet_sync'])
wallet_service = create_wallet_for_sync([0, 0, 0, 0, 0], ['test_empty_wallet_sync'])
sync_test_wallet(fast, wallet)
sync_test_wallet(fast, wallet_service)
broken = True
for md in range(wallet.max_mixdepth + 1):
for md in range(wallet_service.max_mixdepth + 1):
for internal in (True, False):
broken = False
assert 0 == wallet.get_next_unused_index(md, internal)
assert 0 == wallet_service.get_next_unused_index(md, internal)
assert not broken
@ -44,106 +44,115 @@ def test_empty_wallet_sync(setup_wallets, fast):
(True, False), (True, True)))
def test_sequentially_used_wallet_sync(setup_wallets, fast, internal):
used_count = [1, 3, 6, 2, 23]
wallet = create_wallet_for_sync(
wallet_service = create_wallet_for_sync(
used_count, ['test_sequentially_used_wallet_sync'],
populate_internal=internal)
sync_test_wallet(fast, wallet)
sync_test_wallet(fast, wallet_service)
broken = True
for md in range(len(used_count)):
broken = False
assert used_count[md] == wallet.get_next_unused_index(md, internal)
assert used_count[md] == wallet_service.get_next_unused_index(md, internal)
assert not broken
@pytest.mark.parametrize('fast', (False, True))
@pytest.mark.parametrize('fast', (False,))
def test_gap_used_wallet_sync(setup_wallets, fast):
""" After careful examination this test now only includes the Recovery sync.
Note: pre-Aug 2019, because of a bug, this code was not in fact testing both
Fast and Recovery sync, but only Recovery (twice). Also, the scenario set
out in this test (where coins are funded to a wallet which has no index-cache,
and initially no imports) is only appropriate for recovery-mode sync, not for
fast-mode (the now default).
"""
used_count = [1, 3, 6, 2, 23]
wallet = create_wallet_for_sync(used_count, ['test_gap_used_wallet_sync'])
wallet.gap_limit = 20
wallet_service = create_wallet_for_sync(used_count, ['test_gap_used_wallet_sync'])
wallet_service.gap_limit = 20
for md in range(len(used_count)):
x = -1
for x in range(md):
assert x <= wallet.gap_limit, "test broken"
assert x <= wallet_service.gap_limit, "test broken"
# create some unused addresses
wallet.get_new_script(md, True)
wallet.get_new_script(md, False)
wallet_service.get_new_script(md, True)
wallet_service.get_new_script(md, False)
used_count[md] += x + 2
jm_single().bc_interface.grab_coins(wallet.get_new_addr(md, True), 1)
jm_single().bc_interface.grab_coins(wallet.get_new_addr(md, False), 1)
jm_single().bc_interface.grab_coins(wallet_service.get_new_addr(md, True), 1)
jm_single().bc_interface.grab_coins(wallet_service.get_new_addr(md, False), 1)
# reset indices to simulate completely unsynced wallet
for md in range(wallet.max_mixdepth + 1):
wallet.set_next_index(md, True, 0)
wallet.set_next_index(md, False, 0)
sync_test_wallet(fast, wallet)
for md in range(wallet_service.max_mixdepth + 1):
wallet_service.set_next_index(md, True, 0)
wallet_service.set_next_index(md, False, 0)
sync_test_wallet(fast, wallet_service)
broken = True
for md in range(len(used_count)):
broken = False
assert md + 1 == wallet.get_next_unused_index(md, True)
assert used_count[md] == wallet.get_next_unused_index(md, False)
assert md + 1 == wallet_service.get_next_unused_index(md, True)
assert used_count[md] == wallet_service.get_next_unused_index(md, False)
assert not broken
@pytest.mark.parametrize('fast', (False, True))
@pytest.mark.parametrize('fast', (False,))
def test_multigap_used_wallet_sync(setup_wallets, fast):
""" See docstring for test_gap_used_wallet_sync; exactly the
same applies here.
"""
start_index = 5
used_count = [start_index, 0, 0, 0, 0]
wallet = create_wallet_for_sync(used_count, ['test_multigap_used_wallet_sync'])
wallet.gap_limit = 5
wallet_service = create_wallet_for_sync(used_count, ['test_multigap_used_wallet_sync'])
wallet_service.gap_limit = 5
mixdepth = 0
for w in range(5):
for x in range(int(wallet.gap_limit * 0.6)):
assert x <= wallet.gap_limit, "test broken"
for x in range(int(wallet_service.gap_limit * 0.6)):
assert x <= wallet_service.gap_limit, "test broken"
# create some unused addresses
wallet.get_new_script(mixdepth, True)
wallet.get_new_script(mixdepth, False)
wallet_service.get_new_script(mixdepth, True)
wallet_service.get_new_script(mixdepth, False)
used_count[mixdepth] += x + 2
jm_single().bc_interface.grab_coins(wallet.get_new_addr(mixdepth, True), 1)
jm_single().bc_interface.grab_coins(wallet.get_new_addr(mixdepth, False), 1)
jm_single().bc_interface.grab_coins(wallet_service.get_new_addr(mixdepth, True), 1)
jm_single().bc_interface.grab_coins(wallet_service.get_new_addr(mixdepth, False), 1)
# reset indices to simulate completely unsynced wallet
for md in range(wallet.max_mixdepth + 1):
wallet.set_next_index(md, True, 0)
wallet.set_next_index(md, False, 0)
for md in range(wallet_service.max_mixdepth + 1):
wallet_service.set_next_index(md, True, 0)
wallet_service.set_next_index(md, False, 0)
sync_test_wallet(fast, wallet)
sync_test_wallet(fast, wallet_service)
assert used_count[mixdepth] - start_index == wallet.get_next_unused_index(mixdepth, True)
assert used_count[mixdepth] == wallet.get_next_unused_index(mixdepth, False)
assert used_count[mixdepth] - start_index == wallet_service.get_next_unused_index(mixdepth, True)
assert used_count[mixdepth] == wallet_service.get_next_unused_index(mixdepth, False)
@pytest.mark.parametrize('fast', (False, True))
def test_retain_unused_indices_wallet_sync(setup_wallets, fast):
used_count = [0, 0, 0, 0, 0]
wallet = create_wallet_for_sync(used_count, ['test_retain_unused_indices_wallet_sync'])
wallet_service = create_wallet_for_sync(used_count, ['test_retain_unused_indices_wallet_sync'])
for x in range(9):
wallet.get_new_script(0, 1)
wallet_service.get_new_script(0, 1)
sync_test_wallet(fast, wallet)
sync_test_wallet(fast, wallet_service)
assert wallet.get_next_unused_index(0, 1) == 9
assert wallet_service.get_next_unused_index(0, 1) == 9
@pytest.mark.parametrize('fast', (False, True))
def test_imported_wallet_sync(setup_wallets, fast):
used_count = [0, 0, 0, 0, 0]
wallet = create_wallet_for_sync(used_count, ['test_imported_wallet_sync'])
source_wallet = create_wallet_for_sync(used_count, ['test_imported_wallet_sync_origin'])
wallet_service = create_wallet_for_sync(used_count, ['test_imported_wallet_sync'])
source_wallet_service = create_wallet_for_sync(used_count, ['test_imported_wallet_sync_origin'])
address = source_wallet.get_new_addr(0, 1)
wallet.import_private_key(0, source_wallet.get_wif(0, 1, 0))
address = source_wallet_service.get_internal_addr(0)
wallet_service.import_private_key(0, source_wallet_service.get_wif(0, 1, 0))
txid = binascii.unhexlify(jm_single().bc_interface.grab_coins(address, 1))
sync_test_wallet(fast, wallet)
sync_test_wallet(fast, wallet_service)
assert wallet._utxos.have_utxo(txid, 0) == 0
assert wallet_service._utxos.have_utxo(txid, 0) == 0
@pytest.fixture(scope='module')

14
jmclient/test/test_client_protocol.py

@ -6,7 +6,7 @@ from builtins import *
from jmbase import get_log
from jmclient import load_program_config, Taker,\
JMClientProtocolFactory, jm_single, Maker
JMClientProtocolFactory, jm_single, Maker, WalletService
from jmclient.client_protocol import JMTakerClientProtocol
from twisted.python.log import msg as tmsg
from twisted.internet import protocol, reactor, task
@ -83,11 +83,15 @@ class DummyWallet(object):
def get_wallet_id(self):
return 'aaaa'
#class DummyWalletService(object):
# wallet = DummyWallet()
# def register_callbacks(self, callbacks, unconfirmed=True):
# pass
class DummyMaker(Maker):
def __init__(self):
self.aborted = False
self.wallet = DummyWallet()
self.wallet_service = WalletService(DummyWallet())
self.offerlist = self.create_my_orders()
def try_to_create_my_orders(self):
@ -125,10 +129,10 @@ class DummyMaker(Maker):
# utxos, cj_addr, change_addr
return [], '', ''
def on_tx_unconfirmed(self, cjorder, txid, removed_utxos):
def on_tx_unconfirmed(self, cjorder, txid):
return [], []
def on_tx_confirmed(self, cjorder, confirmations, txid):
def on_tx_confirmed(self, cjorder, txid, confirmations):
return [], []
@ -278,7 +282,7 @@ class TrialTestJMClientProto(unittest.TestCase):
self.addCleanup(self.client.transport.loseConnection)
clientfactories = []
takers = [DummyTaker(
None, ["a", "b"], callbacks=(
WalletService(DummyWallet()), ["a", "b"], callbacks=(
None, None, dummy_taker_finished)) for _ in range(len(params))]
for i, p in enumerate(params):
takers[i].set_fail_init(p[0])

95
jmclient/test/test_coinjoin.py

@ -12,8 +12,8 @@ import pytest
from twisted.internet import reactor
from jmbase import get_log
from jmclient import load_program_config, jm_single, \
YieldGeneratorBasic, Taker, sync_wallet, LegacyWallet, SegwitLegacyWallet
from jmclient import load_program_config, jm_single,\
YieldGeneratorBasic, Taker, LegacyWallet, SegwitLegacyWallet
from jmclient.podle import set_commitment_file
from commontest import make_wallets, binarize_tx
from test_taker import dummy_filter_orderbook
@ -30,18 +30,21 @@ def make_wallets_to_list(make_wallets_data):
assert all(wallets)
return wallets
def sync_wallets(wallets):
for w in wallets:
w.gap_limit = 0
jm_single().bc_interface.wallet_synced = False
def sync_wallets(wallet_services, fast=True):
for wallet_service in wallet_services:
wallet_service.synced = False
wallet_service.gap_limit = 0
for x in range(20):
if jm_single().bc_interface.wallet_synced:
if wallet_service.synced:
break
sync_wallet(w)
wallet_service.sync_wallet(fast=fast)
else:
assert False, "Failed to sync wallet"
# because we don't run the monitoring loops for the
# wallet services, we need to update them on the latest
# block manually:
for wallet_service in wallet_services:
wallet_service.update_blockheight()
def create_orderbook(makers):
orderbook = []
@ -119,15 +122,17 @@ def test_simple_coinjoin(monkeypatch, tmpdir, setup_cj, wallet_cls):
set_commitment_file(str(tmpdir.join('commitments.json')))
MAKER_NUM = 3
wallets = make_wallets_to_list(make_wallets(
wallet_services = make_wallets_to_list(make_wallets(
MAKER_NUM + 1, wallet_structures=[[4, 0, 0, 0, 0]] * (MAKER_NUM + 1),
mean_amt=1, wallet_cls=wallet_cls))
jm_single().bc_interface.tickchain()
sync_wallets(wallets)
jm_single().bc_interface.tickchain()
sync_wallets(wallet_services)
makers = [YieldGeneratorBasic(
wallets[i],
wallet_services[i],
[0, 2000, 0, 'swabsoffer', 10**7]) for i in range(MAKER_NUM)]
orderbook = create_orderbook(makers)
@ -136,7 +141,7 @@ def test_simple_coinjoin(monkeypatch, tmpdir, setup_cj, wallet_cls):
cj_amount = int(1.1 * 10**8)
# mixdepth, amount, counterparties, dest_addr, waittime
schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0)]
taker = create_taker(wallets[-1], schedule, monkeypatch)
taker = create_taker(wallet_services[-1], schedule, monkeypatch)
active_orders, maker_data = init_coinjoin(taker, makers,
orderbook, cj_amount)
@ -156,21 +161,22 @@ def test_coinjoin_mixdepth_wrap_taker(monkeypatch, tmpdir, setup_cj):
set_commitment_file(str(tmpdir.join('commitments.json')))
MAKER_NUM = 3
wallets = make_wallets_to_list(make_wallets(
wallet_services = make_wallets_to_list(make_wallets(
MAKER_NUM + 1,
wallet_structures=[[4, 0, 0, 0, 0]] * MAKER_NUM + [[0, 0, 0, 0, 3]],
mean_amt=1))
for w in wallets:
assert w.max_mixdepth == 4
for wallet_service in wallet_services:
assert wallet_service.max_mixdepth == 4
jm_single().bc_interface.tickchain()
jm_single().bc_interface.tickchain()
sync_wallets(wallets)
sync_wallets(wallet_services)
cj_fee = 2000
makers = [YieldGeneratorBasic(
wallets[i],
wallet_services[i],
[0, cj_fee, 0, 'swabsoffer', 10**7]) for i in range(MAKER_NUM)]
orderbook = create_orderbook(makers)
@ -179,7 +185,7 @@ def test_coinjoin_mixdepth_wrap_taker(monkeypatch, tmpdir, setup_cj):
cj_amount = int(1.1 * 10**8)
# mixdepth, amount, counterparties, dest_addr, waittime
schedule = [(4, cj_amount, MAKER_NUM, 'INTERNAL', 0)]
taker = create_taker(wallets[-1], schedule, monkeypatch)
taker = create_taker(wallet_services[-1], schedule, monkeypatch)
active_orders, maker_data = init_coinjoin(taker, makers,
orderbook, cj_amount)
@ -193,11 +199,12 @@ def test_coinjoin_mixdepth_wrap_taker(monkeypatch, tmpdir, setup_cj):
tx = btc.deserialize(txdata[2])
binarize_tx(tx)
w = wallets[-1]
w.remove_old_utxos_(tx)
w.add_new_utxos_(tx, b'\x00' * 32) # fake txid
wallet_service = wallet_services[-1]
# TODO change for new tx monitoring:
wallet_service.remove_old_utxos_(tx)
wallet_service.add_new_utxos_(tx, b'\x00' * 32) # fake txid
balances = w.get_balance_by_mixdepth()
balances = wallet_service.get_balance_by_mixdepth()
assert balances[0] == cj_amount
# <= because of tx fee
assert balances[4] <= 3 * 10**8 - cj_amount - (cj_fee * MAKER_NUM)
@ -210,21 +217,22 @@ def test_coinjoin_mixdepth_wrap_maker(monkeypatch, tmpdir, setup_cj):
set_commitment_file(str(tmpdir.join('commitments.json')))
MAKER_NUM = 2
wallets = make_wallets_to_list(make_wallets(
wallet_services = make_wallets_to_list(make_wallets(
MAKER_NUM + 1,
wallet_structures=[[0, 0, 0, 0, 4]] * MAKER_NUM + [[3, 0, 0, 0, 0]],
mean_amt=1))
for w in wallets:
assert w.max_mixdepth == 4
for wallet_service in wallet_services:
assert wallet_service.max_mixdepth == 4
jm_single().bc_interface.tickchain()
jm_single().bc_interface.tickchain()
sync_wallets(wallets)
sync_wallets(wallet_services)
cj_fee = 2000
makers = [YieldGeneratorBasic(
wallets[i],
wallet_services[i],
[0, cj_fee, 0, 'swabsoffer', 10**7]) for i in range(MAKER_NUM)]
orderbook = create_orderbook(makers)
@ -233,7 +241,7 @@ def test_coinjoin_mixdepth_wrap_maker(monkeypatch, tmpdir, setup_cj):
cj_amount = int(1.1 * 10**8)
# mixdepth, amount, counterparties, dest_addr, waittime
schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0)]
taker = create_taker(wallets[-1], schedule, monkeypatch)
taker = create_taker(wallet_services[-1], schedule, monkeypatch)
active_orders, maker_data = init_coinjoin(taker, makers,
orderbook, cj_amount)
@ -248,43 +256,46 @@ def test_coinjoin_mixdepth_wrap_maker(monkeypatch, tmpdir, setup_cj):
binarize_tx(tx)
for i in range(MAKER_NUM):
w = wallets[i]
w.remove_old_utxos_(tx)
w.add_new_utxos_(tx, b'\x00' * 32) # fake txid
wallet_service = wallet_services[i]
# TODO as above re: monitoring
wallet_service.remove_old_utxos_(tx)
wallet_service.add_new_utxos_(tx, b'\x00' * 32) # fake txid
balances = w.get_balance_by_mixdepth()
balances = wallet_service.get_balance_by_mixdepth()
assert balances[0] == cj_amount
assert balances[4] == 4 * 10**8 - cj_amount + cj_fee
@pytest.mark.parametrize('wallet_cls,wallet_cls_sec', (
(SegwitLegacyWallet, LegacyWallet),
(LegacyWallet, SegwitLegacyWallet)
#(LegacyWallet, SegwitLegacyWallet)
))
def test_coinjoin_mixed_maker_addresses(monkeypatch, tmpdir, setup_cj,
wallet_cls, wallet_cls_sec):
set_commitment_file(str(tmpdir.join('commitments.json')))
MAKER_NUM = 2
wallets = make_wallets_to_list(make_wallets(
wallet_services = make_wallets_to_list(make_wallets(
MAKER_NUM + 1,
wallet_structures=[[1, 0, 0, 0, 0]] * MAKER_NUM + [[3, 0, 0, 0, 0]],
mean_amt=1, wallet_cls=wallet_cls))
wallets_sec = make_wallets_to_list(make_wallets(
wallet_services_sec = make_wallets_to_list(make_wallets(
MAKER_NUM,
wallet_structures=[[1, 0, 0, 0, 0]] * MAKER_NUM,
mean_amt=1, wallet_cls=wallet_cls_sec))
for i in range(MAKER_NUM):
wif = wallets_sec[i].get_wif(0, False, 0)
wallets[i].import_private_key(0, wif, key_type=wallets_sec[i].TYPE)
wif = wallet_services_sec[i].get_wif(0, False, 0)
wallet_services[i].wallet.import_private_key(0, wif,
key_type=wallet_services_sec[i].wallet.TYPE)
jm_single().bc_interface.tickchain()
jm_single().bc_interface.tickchain()
sync_wallets(wallets)
sync_wallets(wallet_services, fast=False)
makers = [YieldGeneratorBasic(
wallets[i],
wallet_services[i],
[0, 2000, 0, 'swabsoffer', 10**7]) for i in range(MAKER_NUM)]
orderbook = create_orderbook(makers)
@ -292,7 +303,7 @@ def test_coinjoin_mixed_maker_addresses(monkeypatch, tmpdir, setup_cj,
cj_amount = int(1.1 * 10**8)
# mixdepth, amount, counterparties, dest_addr, waittime
schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0)]
taker = create_taker(wallets[-1], schedule, monkeypatch)
taker = create_taker(wallet_services[-1], schedule, monkeypatch)
active_orders, maker_data = init_coinjoin(taker, makers,
orderbook, cj_amount)

6
jmclient/test/test_maker.py

@ -6,7 +6,7 @@ from builtins import * # noqa: F401
import jmbitcoin as btc
from jmclient import Maker, get_p2sh_vbyte, get_p2pk_vbyte, \
load_program_config, jm_single
load_program_config, jm_single, WalletService
import jmclient
from commontest import DummyBlockchainInterface
from test_taker import DummyWallet
@ -116,7 +116,7 @@ def test_verify_unsigned_tx_sw_valid(setup_env_nodeps):
p2pkh_gen = address_p2pkh_generator()
wallet = DummyWallet()
maker = OfflineMaker(wallet)
maker = OfflineMaker(WalletService(wallet))
cj_addr, cj_script = next(p2sh_gen)
changeaddr, cj_change_script = next(p2sh_gen)
@ -149,7 +149,7 @@ def test_verify_unsigned_tx_nonsw_valid(setup_env_nodeps):
p2pkh_gen = address_p2pkh_generator()
wallet = DummyWallet()
maker = OfflineMaker(wallet)
maker = OfflineMaker(WalletService(wallet))
cj_addr, cj_script = next(p2pkh_gen)
changeaddr, cj_change_script = next(p2pkh_gen)

41
jmclient/test/test_payjoin.py

@ -12,17 +12,17 @@ import pytest
from twisted.internet import reactor
from jmbase import get_log
from jmclient import cryptoengine
from jmclient import (load_program_config, jm_single, sync_wallet,
from jmclient import (load_program_config, jm_single,
P2EPMaker, P2EPTaker,
LegacyWallet, SegwitLegacyWallet, SegwitWallet)
from commontest import make_wallets
from test_coinjoin import make_wallets_to_list, sync_wallets, create_orderbook
from test_coinjoin import make_wallets_to_list, create_orderbook, sync_wallets
testdir = os.path.dirname(os.path.realpath(__file__))
log = get_log()
def create_taker(wallet, schedule, monkeypatch):
taker = P2EPTaker("fakemaker", wallet, schedule,
def create_taker(wallet_service, schedule, monkeypatch):
taker = P2EPTaker("fakemaker", wallet_service, schedule,
callbacks=(None, None, None))
return taker
@ -32,13 +32,13 @@ def dummy_user_check(message):
log.info(message)
return True
def getbals(wallet, mixdepth):
def getbals(wallet_service, mixdepth):
""" Retrieves balances for a mixdepth and the 'next'
"""
bbm = wallet.get_balance_by_mixdepth()
return (bbm[mixdepth], bbm[(mixdepth + 1) % (wallet.mixdepth + 1)])
bbm = wallet_service.get_balance_by_mixdepth()
return (bbm[mixdepth], bbm[(mixdepth + 1) % (wallet_service.mixdepth + 1)])
def final_checks(wallets, amount, txfee, tsb, msb, source_mixdepth=0):
def final_checks(wallet_services, amount, txfee, tsb, msb, source_mixdepth=0):
"""We use this to check that the wallet contents are
as we've expected according to the test case.
amount is the payment amount going from taker to maker.
@ -48,10 +48,9 @@ def final_checks(wallets, amount, txfee, tsb, msb, source_mixdepth=0):
of two entries, source and destination mixdepth respectively.
"""
jm_single().bc_interface.tickchain()
for wallet in wallets:
sync_wallet(wallet)
takerbals = getbals(wallets[1], source_mixdepth)
makerbals = getbals(wallets[0], source_mixdepth)
sync_wallets(wallet_services)
takerbals = getbals(wallet_services[1], source_mixdepth)
makerbals = getbals(wallet_services[0], source_mixdepth)
# is the payment received?
maker_newcoin_amt = makerbals[1] - msb[1]
if not maker_newcoin_amt >= amount:
@ -97,23 +96,23 @@ def test_simple_payjoin(monkeypatch, tmpdir, setup_cj, wallet_cls,
def raise_exit(i):
raise Exception("sys.exit called")
monkeypatch.setattr(sys, 'exit', raise_exit)
wallets = []
wallets.append(make_wallets_to_list(make_wallets(
wallet_services = []
wallet_services.append(make_wallets_to_list(make_wallets(
1, wallet_structures=[wallet_structures[0]],
mean_amt=mean_amt, wallet_cls=wallet_cls[0]))[0])
wallets.append(make_wallets_to_list(make_wallets(
wallet_services.append(make_wallets_to_list(make_wallets(
1, wallet_structures=[wallet_structures[1]],
mean_amt=mean_amt, wallet_cls=wallet_cls[1]))[0])
jm_single().bc_interface.tickchain()
sync_wallets(wallets)
sync_wallets(wallet_services)
# For accounting purposes, record the balances
# at the start.
msb = getbals(wallets[0], 0)
tsb = getbals(wallets[1], 0)
msb = getbals(wallet_services[0], 0)
tsb = getbals(wallet_services[1], 0)
cj_amount = int(1.1 * 10**8)
maker = P2EPMaker(wallets[0], 0, cj_amount)
maker = P2EPMaker(wallet_services[0], 0, cj_amount)
destaddr = maker.destination_addr
monkeypatch.setattr(maker, 'user_check', dummy_user_check)
# TODO use this to sanity check behaviour
@ -123,7 +122,7 @@ def test_simple_payjoin(monkeypatch, tmpdir, setup_cj, wallet_cls,
# mixdepth, amount, counterparties, dest_addr, waittime;
# in payjoin we only pay attention to the first two entries.
schedule = [(0, cj_amount, 1, destaddr, 0)]
taker = create_taker(wallets[-1], schedule, monkeypatch)
taker = create_taker(wallet_services[-1], schedule, monkeypatch)
monkeypatch.setattr(taker, 'user_check', dummy_user_check)
init_data = taker.initialize(orderbook)
# the P2EPTaker.initialize() returns:
@ -147,7 +146,7 @@ def test_simple_payjoin(monkeypatch, tmpdir, setup_cj, wallet_cls,
assert False
# Although the above OK is proof that a transaction went through,
# it doesn't prove it was a good transaction! Here do balance checks:
assert final_checks(wallets, cj_amount, taker.total_txfee, tsb, msb)
assert final_checks(wallet_services, cj_amount, taker.total_txfee, tsb, msb)
@pytest.fixture(scope='module')
def setup_cj():

6
jmclient/test/test_podle.py

@ -196,14 +196,14 @@ def test_podle_error_string(setup_podle):
('fakepriv2', 'fakeutxo2')]
to = ['tooold1', 'tooold2']
ts = ['toosmall1', 'toosmall2']
wallet = make_wallets(1, [[1, 0, 0, 0, 0]])[0]['wallet']
wallet_service = make_wallets(1, [[1, 0, 0, 0, 0]])[0]['wallet']
cjamt = 100
tua = "3"
tuamtper = "20"
errmgsheader, errmsg = generate_podle_error_string(priv_utxo_pairs,
to,
ts,
wallet,
wallet_service,
cjamt,
tua,
tuamtper)
@ -212,7 +212,7 @@ def test_podle_error_string(setup_podle):
y = [x[1] for x in priv_utxo_pairs]
assert all([errmsg.find(x) != -1 for x in to + ts + y])
#ensure OK with nothing
errmgsheader, errmsg = generate_podle_error_string([], [], [], wallet,
errmgsheader, errmsg = generate_podle_error_string([], [], [], wallet_service,
cjamt, tua, tuamtper)
@pytest.fixture(scope="module")

1
jmclient/test/test_storage.py

@ -84,7 +84,6 @@ def test_storage_invalid():
MockStorage(b'garbagefile', __file__, b'password')
pytest.fail("Non-wallet file, encrypted")
def test_storage_readonly():
s = MockStorage(None, 'nonexistant', b'password', create=True)
s = MockStorage(s.file_data, __file__, b'password', read_only=True)

46
jmclient/test/test_taker.py

@ -15,7 +15,7 @@ import struct
from base64 import b64encode
from jmclient import load_program_config, jm_single, set_commitment_file,\
get_commitment_file, SegwitLegacyWallet, Taker, VolatileStorage,\
get_network
get_network, WalletService
from taker_test_data import t_utxos_by_mixdepth, t_orderbook,\
t_maker_response, t_chosen_orders, t_dummy_ext
@ -35,14 +35,15 @@ class DummyWallet(SegwitLegacyWallet):
txid, index = txid.split(':')
path = (b'dummy', md, i)
self._utxos.add_utxo(binascii.unhexlify(txid), int(index),
path, data['value'], md)
path, data['value'], md, 1)
script = self._ENGINE.address_to_script(data['address'])
self._script_map[script] = path
def get_utxos_by_mixdepth(self, verbose=True):
def get_utxos_by_mixdepth(self, verbose=True, includeheight=False):
return t_utxos_by_mixdepth
def get_utxos_by_mixdepth_(self, verbose=True):
def get_utxos_by_mixdepth_(self, verbose=True, include_disabled=False,
includeheight=False):
utxos = self.get_utxos_by_mixdepth(verbose)
utxos_conv = {}
@ -59,7 +60,8 @@ class DummyWallet(SegwitLegacyWallet):
return utxos_conv
def select_utxos(self, mixdepth, amount):
def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None,
maxheight=None):
if amount > self.get_balance_by_mixdepth()[mixdepth]:
raise Exception("Not enough funds")
return t_utxos_by_mixdepth[mixdepth]
@ -126,10 +128,12 @@ def get_taker(schedule=None, schedule_len=0, on_finished=None,
print("Using schedule: " + str(schedule))
on_finished_callback = on_finished if on_finished else taker_finished
filter_orders_callback = filter_orders if filter_orders else dummy_filter_orderbook
return Taker(DummyWallet(), schedule,
taker = Taker(WalletService(DummyWallet()), schedule,
callbacks=[filter_orders_callback, None, on_finished_callback])
taker.wallet_service.current_blockheight = 10**6
return taker
def test_filter_rejection(createcmtdata):
def test_filter_rejection(setup_taker):
def filter_orders_reject(orders_feesl, cjamount):
print("calling filter orders rejection")
return False
@ -149,7 +153,7 @@ def test_filter_rejection(createcmtdata):
(True, False),
(False, True),
])
def test_make_commitment(createcmtdata, failquery, external):
def test_make_commitment(setup_taker, failquery, external):
def clean_up():
jm_single().config.set("POLICY", "taker_utxo_age", old_taker_utxo_age)
jm_single().config.set("POLICY", "taker_utxo_amtpercent", old_taker_utxo_amtpercent)
@ -175,7 +179,7 @@ def test_make_commitment(createcmtdata, failquery, external):
taker.make_commitment()
clean_up()
def test_not_found_maker_utxos(createcmtdata):
def test_not_found_maker_utxos(setup_taker):
taker = get_taker([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)])
orderbook = copy.deepcopy(t_orderbook)
res = taker.initialize(orderbook)
@ -187,7 +191,7 @@ def test_not_found_maker_utxos(createcmtdata):
assert res[1] == "Not enough counterparties responded to fill, giving up"
jm_single().bc_interface.setQUSFail(False)
def test_auth_pub_not_found(createcmtdata):
def test_auth_pub_not_found(setup_taker):
taker = get_taker([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)])
orderbook = copy.deepcopy(t_orderbook)
res = taker.initialize(orderbook)
@ -239,7 +243,7 @@ def test_auth_pub_not_found(createcmtdata):
([(0, 0, 5, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)], False, False,
2, False, ["J659UPUSLLjHJpaB", "J65z23xdjxJjC7er", 0], None), #test inadequate for sweep
])
def test_taker_init(createcmtdata, schedule, highfee, toomuchcoins, minmakers,
def test_taker_init(setup_taker, schedule, highfee, toomuchcoins, minmakers,
notauthed, ignored, nocommit):
#these tests do not trigger utxo_retries
oldtakerutxoretries = jm_single().config.get("POLICY", "taker_utxo_retries")
@ -349,7 +353,7 @@ def test_taker_init(createcmtdata, schedule, highfee, toomuchcoins, minmakers,
taker.prepare_my_bitcoin_data()
with pytest.raises(NotImplementedError) as e_info:
a = taker.coinjoin_address()
taker.wallet.inject_addr_get_failure = True
taker.wallet_service.wallet.inject_addr_get_failure = True
taker.my_cj_addr = "dummy"
assert not taker.prepare_my_bitcoin_data()
#clean up
@ -365,6 +369,8 @@ def test_unconfirm_confirm(schedule_len):
and merely update schedule index for confirm (useful for schedules/tumbles).
This tests that the on_finished callback correctly reports the fromtx
variable as "False" once the schedule is complete.
The exception to the above is that the txd passed in must match
self.latest_tx, so we use a dummy value here for that.
"""
test_unconfirm_confirm.txflag = True
def finished_for_confirms(res, fromtx=False, waittime=0, txdetails=None):
@ -372,13 +378,14 @@ def test_unconfirm_confirm(schedule_len):
test_unconfirm_confirm.txflag = fromtx
taker = get_taker(schedule_len=schedule_len, on_finished=finished_for_confirms)
taker.unconfirm_callback("a", "b")
taker.latest_tx = {"outs": "blah"}
taker.unconfirm_callback({"ins": "foo", "outs": "blah"}, "b")
for i in range(schedule_len-1):
taker.schedule_index += 1
fromtx = taker.confirm_callback("a", "b", 1)
fromtx = taker.confirm_callback({"ins": "foo", "outs": "blah"}, "b", 1)
assert test_unconfirm_confirm.txflag
taker.schedule_index += 1
fromtx = taker.confirm_callback("a", "b", 1)
fromtx = taker.confirm_callback({"ins": "foo", "outs": "blah"}, "b", 1)
assert not test_unconfirm_confirm.txflag
@pytest.mark.parametrize(
@ -387,7 +394,7 @@ def test_unconfirm_confirm(schedule_len):
("mrcNu71ztWjAQA6ww9kHiW3zBWSQidHXTQ",
[(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)])
])
def test_on_sig(createcmtdata, dummyaddr, schedule):
def test_on_sig(setup_taker, dummyaddr, schedule):
#plan: create a new transaction with known inputs and dummy outputs;
#then, create a signature with various inputs, pass in in b64 to on_sig.
#in order for it to verify, the DummyBlockchainInterface will have to
@ -472,7 +479,12 @@ def test_auth_counterparty(schedule):
assert not taker.auth_counterparty(sig_tweaked, auth_pub, maker_pub)
@pytest.fixture(scope="module")
def createcmtdata(request):
def setup_taker(request):
def clean():
from twisted.internet import reactor
for dc in reactor.getDelayedCalls():
dc.cancel()
request.addfinalizer(clean)
def cmtdatateardown():
shutil.rmtree("cmtdata")
request.addfinalizer(cmtdatateardown)

78
jmclient/test/test_tx_creation.py

@ -13,7 +13,7 @@ from commontest import make_wallets, make_sign_and_push
import jmbitcoin as bitcoin
import pytest
from jmbase import get_log
from jmclient import load_program_config, jm_single, sync_wallet,\
from jmclient import load_program_config, jm_single,\
get_p2pk_vbyte
log = get_log()
@ -36,14 +36,14 @@ def test_create_p2sh_output_tx(setup_tx_creation, nw, wallet_structures,
mean_amt, sdev_amt, amount, pubs, k):
wallets = make_wallets(nw, wallet_structures, mean_amt, sdev_amt)
for w in wallets.values():
sync_wallet(w['wallet'], fast=True)
w['wallet'].sync_wallet(fast=True)
for k, w in enumerate(wallets.values()):
wallet = w['wallet']
ins_full = wallet.select_utxos(0, amount)
wallet_service = w['wallet']
ins_full = wallet_service.select_utxos(0, amount)
script = bitcoin.mk_multisig_script(pubs, k)
output_addr = bitcoin.script_to_address(script, vbyte=196)
txid = make_sign_and_push(ins_full,
wallet,
wallet_service,
amount,
output_addr=output_addr)
assert txid
@ -81,16 +81,16 @@ def test_all_same_priv(setup_tx_creation):
#recipient
priv = "aa"*32 + "01"
addr = bitcoin.privkey_to_address(priv, magicbyte=get_p2pk_vbyte())
wallet = make_wallets(1, [[1,0,0,0,0]], 1)[0]['wallet']
wallet_service = make_wallets(1, [[1,0,0,0,0]], 1)[0]['wallet']
#make another utxo on the same address
addrinwallet = wallet.get_addr(0,0,0)
addrinwallet = wallet_service.get_addr(0,0,0)
jm_single().bc_interface.grab_coins(addrinwallet, 1)
sync_wallet(wallet, fast=True)
insfull = wallet.select_utxos(0, 110000000)
wallet_service.sync_wallet(fast=True)
insfull = wallet_service.select_utxos(0, 110000000)
outs = [{"address": addr, "value": 1000000}]
ins = list(insfull.keys())
tx = bitcoin.mktx(ins, outs)
tx = bitcoin.signall(tx, wallet.get_key_from_addr(addrinwallet))
tx = bitcoin.signall(tx, wallet_service.get_key_from_addr(addrinwallet))
@pytest.mark.parametrize(
"signall",
@ -101,9 +101,9 @@ def test_all_same_priv(setup_tx_creation):
def test_verify_tx_input(setup_tx_creation, signall):
priv = "aa"*32 + "01"
addr = bitcoin.privkey_to_address(priv, magicbyte=get_p2pk_vbyte())
wallet = make_wallets(1, [[2,0,0,0,0]], 1)[0]['wallet']
sync_wallet(wallet, fast=True)
insfull = wallet.select_utxos(0, 110000000)
wallet_service = make_wallets(1, [[2,0,0,0,0]], 1)[0]['wallet']
wallet_service.sync_wallet(fast=True)
insfull = wallet_service.select_utxos(0, 110000000)
print(insfull)
outs = [{"address": addr, "value": 1000000}]
ins = list(insfull.keys())
@ -115,14 +115,14 @@ def test_verify_tx_input(setup_tx_creation, signall):
for index, ins in enumerate(desertx['ins']):
utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
ad = insfull[utxo]['address']
priv = wallet.get_key_from_addr(ad)
priv = wallet_service.get_key_from_addr(ad)
privdict[utxo] = priv
tx = bitcoin.signall(tx, privdict)
else:
for index, ins in enumerate(desertx['ins']):
utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
ad = insfull[utxo]['address']
priv = wallet.get_key_from_addr(ad)
priv = wallet_service.get_key_from_addr(ad)
tx = bitcoin.sign(tx, index, priv)
desertx2 = bitcoin.deserialize(tx)
print(desertx2)
@ -143,28 +143,28 @@ def test_absurd_fees(setup_tx_creation):
"""
jm_single().bc_interface.absurd_fees = True
#pay into it
wallet = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet']
sync_wallet(wallet, fast=True)
wallet_service = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet']
wallet_service.sync_wallet(fast=True)
amount = 350000000
ins_full = wallet.select_utxos(0, amount)
ins_full = wallet_service.select_utxos(0, amount)
with pytest.raises(ValueError) as e_info:
txid = make_sign_and_push(ins_full, wallet, amount, estimate_fee=True)
txid = make_sign_and_push(ins_full, wallet_service, amount, estimate_fee=True)
def test_create_sighash_txs(setup_tx_creation):
#non-standard hash codes:
for sighash in [bitcoin.SIGHASH_ANYONECANPAY + bitcoin.SIGHASH_SINGLE,
bitcoin.SIGHASH_NONE, bitcoin.SIGHASH_SINGLE]:
wallet = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet']
sync_wallet(wallet, fast=True)
wallet_service = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet']
wallet_service.sync_wallet(fast=True)
amount = 350000000
ins_full = wallet.select_utxos(0, amount)
ins_full = wallet_service.select_utxos(0, amount)
print("using hashcode: " + str(sighash))
txid = make_sign_and_push(ins_full, wallet, amount, hashcode=sighash)
txid = make_sign_and_push(ins_full, wallet_service, amount, hashcode=sighash)
assert txid
#trigger insufficient funds
with pytest.raises(Exception) as e_info:
fake_utxos = wallet.select_utxos(4, 1000000000)
fake_utxos = wallet_service.select_utxos(4, 1000000000)
def test_spend_p2sh_utxos(setup_tx_creation):
@ -174,11 +174,11 @@ def test_spend_p2sh_utxos(setup_tx_creation):
script = bitcoin.mk_multisig_script(pubs, 2)
msig_addr = bitcoin.p2sh_scriptaddr(script, magicbyte=196)
#pay into it
wallet = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet']
sync_wallet(wallet, fast=True)
wallet_service = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet']
wallet_service.sync_wallet(fast=True)
amount = 350000000
ins_full = wallet.select_utxos(0, amount)
txid = make_sign_and_push(ins_full, wallet, amount, output_addr=msig_addr)
ins_full = wallet_service.select_utxos(0, amount)
txid = make_sign_and_push(ins_full, wallet_service, amount, output_addr=msig_addr)
assert txid
#wait for mining
time.sleep(1)
@ -186,7 +186,7 @@ def test_spend_p2sh_utxos(setup_tx_creation):
msig_in = txid + ":0"
ins = [msig_in]
#random output address and change addr
output_addr = wallet.get_new_addr(1, 1)
output_addr = wallet_service.get_internal_addr(1)
amount2 = amount - 50000
outs = [{'value': amount2, 'address': output_addr}]
tx = bitcoin.mktx(ins, outs)
@ -205,19 +205,19 @@ def test_spend_p2wpkh(setup_tx_creation):
scriptPubKeys = [bitcoin.pubkey_to_p2wpkh_script(pub) for pub in pubs]
addresses = [bitcoin.pubkey_to_p2wpkh_address(pub) for pub in pubs]
#pay into it
wallet = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet']
sync_wallet(wallet, fast=True)
wallet_service = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet']
wallet_service.sync_wallet(fast=True)
amount = 35000000
p2wpkh_ins = []
for addr in addresses:
ins_full = wallet.select_utxos(0, amount)
txid = make_sign_and_push(ins_full, wallet, amount, output_addr=addr)
ins_full = wallet_service.select_utxos(0, amount)
txid = make_sign_and_push(ins_full, wallet_service, amount, output_addr=addr)
assert txid
p2wpkh_ins.append(txid + ":0")
#wait for mining
time.sleep(1)
#random output address
output_addr = wallet.get_new_addr(1, 1)
output_addr = wallet_service.get_internal_addr(1)
amount2 = amount*3 - 50000
outs = [{'value': amount2, 'address': output_addr}]
tx = bitcoin.mktx(p2wpkh_ins, outs)
@ -249,19 +249,19 @@ def test_spend_p2wsh(setup_tx_creation):
scriptPubKeys = [bitcoin.pubkeys_to_p2wsh_script(pubs[i:i+2]) for i in [0, 2]]
addresses = [bitcoin.pubkeys_to_p2wsh_address(pubs[i:i+2]) for i in [0, 2]]
#pay into it
wallet = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet']
sync_wallet(wallet, fast=True)
wallet_service = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet']
wallet_service.sync_wallet(fast=True)
amount = 35000000
p2wsh_ins = []
for addr in addresses:
ins_full = wallet.select_utxos(0, amount)
txid = make_sign_and_push(ins_full, wallet, amount, output_addr=addr)
ins_full = wallet_service.select_utxos(0, amount)
txid = make_sign_and_push(ins_full, wallet_service, amount, output_addr=addr)
assert txid
p2wsh_ins.append(txid + ":0")
#wait for mining
time.sleep(1)
#random output address and change addr
output_addr = wallet.get_new_addr(1, 1)
output_addr = wallet_service.get_internal_addr(1)
amount2 = amount*2 - 50000
outs = [{'value': amount2, 'address': output_addr}]
tx = bitcoin.mktx(p2wsh_ins, outs)

16
jmclient/test/test_utxomanager.py

@ -32,11 +32,11 @@ def test_utxomanager_persist(setup_env_nodeps):
mixdepth = 0
value = 500
um.add_utxo(txid, index, path, value, mixdepth)
um.add_utxo(txid, index+1, path, value, mixdepth+1)
um.add_utxo(txid, index, path, value, mixdepth, 1)
um.add_utxo(txid, index+1, path, value, mixdepth+1, 2)
# the third utxo will be disabled and we'll check if
# the disablement persists in the storage across UM instances
um.add_utxo(txid, index+2, path, value, mixdepth+1)
um.add_utxo(txid, index+2, path, value, mixdepth+1, 3)
um.disable_utxo(txid, index+2)
um.save()
@ -103,20 +103,24 @@ def test_utxomanager_select(setup_env_nodeps):
mixdepth = 0
value = 500
um.add_utxo(txid, index, path, value, mixdepth)
um.add_utxo(txid, index, path, value, mixdepth, 100)
assert len(um.select_utxos(mixdepth, value)) == 1
assert len(um.select_utxos(mixdepth+1, value)) == 0
um.add_utxo(txid, index+1, path, value, mixdepth)
um.add_utxo(txid, index+1, path, value, mixdepth, None)
assert len(um.select_utxos(mixdepth, value)) == 2
# ensure that added utxos that are disabled do not
# get used by the selector
um.add_utxo(txid, index+2, path, value, mixdepth)
um.add_utxo(txid, index+2, path, value, mixdepth, 101)
um.disable_utxo(txid, index+2)
assert len(um.select_utxos(mixdepth, value)) == 2
# ensure that unconfirmed coins are not selected if
# dis-requested:
assert len(um.select_utxos(mixdepth, value, maxheight=105)) == 1
@pytest.fixture
def setup_env_nodeps(monkeypatch):

12
jmclient/test/test_wallet.py

@ -14,7 +14,7 @@ from jmbase import get_log
from jmclient import load_program_config, jm_single, \
SegwitLegacyWallet,BIP32Wallet, BIP49Wallet, LegacyWallet,\
VolatileStorage, get_network, cryptoengine, WalletError,\
SegwitWallet
SegwitWallet, WalletService
from test_blockchaininterface import sync_test_wallet
testdir = os.path.dirname(os.path.realpath(__file__))
@ -52,7 +52,7 @@ def fund_wallet_addr(wallet, addr, value_btc=1):
txin_id = jm_single().bc_interface.grab_coins(addr, value_btc)
txinfo = jm_single().bc_interface.rpc('gettransaction', [txin_id])
txin = btc.deserialize(unhexlify(txinfo['hex']))
utxo_in = wallet.add_new_utxos_(txin, unhexlify(txin_id))
utxo_in = wallet.add_new_utxos_(txin, unhexlify(txin_id), 1)
assert len(utxo_in) == 1
return list(utxo_in.keys())[0]
@ -407,7 +407,7 @@ def test_add_new_utxos(setup_wallet):
for s in tx_scripts]))
binarize_tx(tx)
txid = b'\x01' * 32
added = wallet.add_new_utxos_(tx, txid)
added = wallet.add_new_utxos_(tx, txid, 1)
assert len(added) == len(scripts)
added_scripts = {x['script'] for x in added.values()}
@ -429,7 +429,7 @@ def test_remove_old_utxos(setup_wallet):
for i in range(3):
txin = jm_single().bc_interface.grab_coins(
wallet.get_internal_addr(1), 1)
wallet.add_utxo(unhexlify(txin), 0, wallet.get_script(1, 1, i), 10**8)
wallet.add_utxo(unhexlify(txin), 0, wallet.get_script(1, 1, i), 10**8, 1)
inputs = wallet.select_utxos_(0, 10**8)
inputs.update(wallet.select_utxos_(1, 2 * 10**8))
@ -466,7 +466,7 @@ def test_initialize_twice(setup_wallet):
def test_is_known(setup_wallet):
wallet = get_populated_wallet(num=0)
script = wallet.get_new_script(1, True)
addr = wallet.get_new_addr(2, False)
addr = wallet.get_external_addr(2)
assert wallet.is_known_script(script)
assert wallet.is_known_addr(addr)
@ -651,7 +651,7 @@ def test_wallet_mixdepth_decrease(setup_wallet):
VolatileStorage(data=storage_data), mixdepth=new_mixdepth)
assert new_wallet.max_mixdepth == max_mixdepth
assert new_wallet.mixdepth == new_mixdepth
sync_test_wallet(True, new_wallet)
sync_test_wallet(True, WalletService(new_wallet))
assert max_mixdepth not in new_wallet.get_balance_by_mixdepth()
assert max_mixdepth not in new_wallet.get_utxos_by_mixdepth()

24
jmclient/test/test_wallets.py

@ -13,38 +13,38 @@ import json
import pytest
from jmbase import get_log
from jmclient import (
load_program_config, jm_single, sync_wallet,
load_program_config, jm_single,
estimate_tx_fee, BitcoinCoreInterface, Mnemonic)
from taker_test_data import t_raw_signed_tx
testdir = os.path.dirname(os.path.realpath(__file__))
log = get_log()
def do_tx(wallet, amount):
ins_full = wallet.select_utxos(0, amount)
cj_addr = wallet.get_internal_addr(1)
change_addr = wallet.get_internal_addr(0)
wallet.update_cache_index()
def do_tx(wallet_service, amount):
ins_full = wallet_service.select_utxos(0, amount)
cj_addr = wallet_service.get_internal_addr(1)
change_addr = wallet_service.get_internal_addr(0)
wallet_service.save_wallet()
txid = make_sign_and_push(ins_full,
wallet,
wallet_service,
amount,
output_addr=cj_addr,
change_addr=change_addr,
estimate_fee=True)
assert txid
time.sleep(2) #blocks
jm_single().bc_interface.sync_unspent(wallet)
wallet_service.sync_unspent()
return txid
def test_query_utxo_set(setup_wallets):
load_program_config()
jm_single().bc_interface.tick_forward_chain_interval = 1
wallet = create_wallet_for_sync([2, 3, 0, 0, 0],
wallet_service = create_wallet_for_sync([2, 3, 0, 0, 0],
["wallet4utxo.json", "4utxo", [2, 3]])
sync_wallet(wallet, fast=True)
txid = do_tx(wallet, 90000000)
txid2 = do_tx(wallet, 20000000)
wallet_service.sync_wallet(fast=True)
txid = do_tx(wallet_service, 90000000)
txid2 = do_tx(wallet_service, 20000000)
print("Got txs: ", txid, txid2)
res1 = jm_single().bc_interface.query_utxo_set(txid + ":0", includeunconf=True)
res2 = jm_single().bc_interface.query_utxo_set(

23
jmclient/test/test_yieldgenerator.py

@ -5,7 +5,8 @@ from builtins import * # noqa: F401
import unittest
from jmclient import load_program_config, jm_single,\
SegwitLegacyWallet, VolatileStorage, YieldGeneratorBasic, get_network
SegwitLegacyWallet, VolatileStorage, YieldGeneratorBasic, \
get_network, WalletService
class CustomUtxoWallet(SegwitLegacyWallet):
@ -37,7 +38,7 @@ class CustomUtxoWallet(SegwitLegacyWallet):
# script, and make it fit the required length (32 bytes).
txid = tx['outs'][0]['script'] + b'x' * 32
txid = txid[:32]
self.add_new_utxos_(tx, txid)
self.add_new_utxos_(tx, txid, 1)
def assert_utxos_from_mixdepth(self, utxos, expected):
"""Asserts that the list of UTXOs (as returned from UTXO selection
@ -56,7 +57,7 @@ def create_yg_basic(balances, txfee=0, cjfee_a=0, cjfee_r=0,
wallet = CustomUtxoWallet(balances)
offerconfig = (txfee, cjfee_a, cjfee_r, ordertype, minsize)
yg = YieldGeneratorBasic(wallet, offerconfig)
yg = YieldGeneratorBasic(WalletService(wallet), offerconfig)
# We don't need any of the logic from Maker, including the waiting
# loop. Just stop it, so that it does not linger around and create
@ -140,9 +141,9 @@ class OidToOrderTests(unittest.TestCase):
yg = create_yg_basic([10, 1000, 2000])
utxos, cj_addr, change_addr = self.call_oid_to_order(yg, 500)
self.assertEqual(len(utxos), 1)
yg.wallet.assert_utxos_from_mixdepth(utxos, 1)
self.assertEqual(yg.wallet.get_addr_mixdepth(cj_addr), 2)
self.assertEqual(yg.wallet.get_addr_mixdepth(change_addr), 1)
yg.wallet_service.wallet.assert_utxos_from_mixdepth(utxos, 1)
self.assertEqual(yg.wallet_service.wallet.get_addr_mixdepth(cj_addr), 2)
self.assertEqual(yg.wallet_service.wallet.get_addr_mixdepth(change_addr), 1)
def test_not_enough_balance_with_dust_threshold(self):
# 410 is exactly the size of the change output. So it will be
@ -158,12 +159,12 @@ class OidToOrderTests(unittest.TestCase):
# over the threshold.
jm_single().DUST_THRESHOLD = 410
yg = create_yg_basic([10, 1000, 10], txfee=100, cjfee_a=10)
yg.wallet.add_utxo_at_mixdepth(1, 500)
yg.wallet_service.wallet.add_utxo_at_mixdepth(1, 500)
utxos, cj_addr, change_addr = self.call_oid_to_order(yg, 500)
self.assertEqual(len(utxos), 2)
yg.wallet.assert_utxos_from_mixdepth(utxos, 1)
self.assertEqual(yg.wallet.get_addr_mixdepth(cj_addr), 2)
self.assertEqual(yg.wallet.get_addr_mixdepth(change_addr), 1)
yg.wallet_service.wallet.assert_utxos_from_mixdepth(utxos, 1)
self.assertEqual(yg.wallet_service.wallet.get_addr_mixdepth(cj_addr), 2)
self.assertEqual(yg.wallet_service.wallet.get_addr_mixdepth(change_addr), 1)
class OfferReannouncementTests(unittest.TestCase):
@ -171,7 +172,7 @@ class OfferReannouncementTests(unittest.TestCase):
def call_on_tx_unconfirmed(self, yg):
"""Calls yg.on_tx_unconfirmed with fake arguments."""
return yg.on_tx_unconfirmed({'cjaddr': 'addr'}, 'txid', [])
return yg.on_tx_unconfirmed({'cjaddr': 'addr'}, 'txid')
def create_yg_and_offer(self, maxsize):
"""Constructs a fake yg instance that has an offer with the given

20
scripts/add-utxo.py

@ -19,7 +19,7 @@ from optparse import OptionParser
from jmbase import jmprint
import jmbitcoin as btc
from jmclient import load_program_config, jm_single, get_p2pk_vbyte,\
open_wallet, sync_wallet, add_external_commitments, update_commitments,\
open_wallet, WalletService, add_external_commitments, update_commitments,\
PoDLE, get_podle_commitments, get_utxo_info, validate_utxo_data, quit,\
get_wallet_path
@ -146,12 +146,12 @@ def main():
help='only validate the provided utxos (file or command line), not add',
default=False
)
parser.add_option('--fast',
parser.add_option('--recoversync',
action='store_true',
dest='fastsync',
dest='recoversync',
default=False,
help=('choose to do fast wallet sync, only for Core and '
'only for previously synced wallet'))
help=('choose to do detailed wallet sync, '
'used for recovering on new Core instance.'))
(options, args) = parser.parse_args()
load_program_config()
#TODO; sort out "commit file location" global so this script can
@ -179,16 +179,18 @@ def main():
if options.loadwallet:
wallet_path = get_wallet_path(options.loadwallet, None)
wallet = open_wallet(wallet_path, gap_limit=options.gaplimit)
while not jm_single().bc_interface.wallet_synced:
sync_wallet(wallet, fast=options.fastsync)
wallet_service = WalletService(wallet)
while True:
if wallet_service.sync_wallet(fast=not options.recoversync):
break
# minor note: adding a utxo from an external wallet for commitments, we
# default to not allowing disabled utxos to avoid a privacy leak, so the
# user would have to explicitly enable.
for md, utxos in wallet.get_utxos_by_mixdepth_().items():
for md, utxos in wallet_service.get_utxos_by_mixdepth(hexfmt=False).items():
for (txid, index), utxo in utxos.items():
txhex = binascii.hexlify(txid).decode('ascii') + ':' + str(index)
wif = wallet.get_wif_path(utxo['path'])
wif = wallet_service.get_wif_path(utxo['path'])
utxo_data.append((txhex, wif))
elif options.in_file:

20
scripts/cli_options.py

@ -33,12 +33,12 @@ def add_common_options(parser):
'for the total transaction fee, default=dynamically estimated, note that this is adjusted '
'based on the estimated fee calculated after tx construction, based on '
'policy set in joinmarket.cfg.')
parser.add_option('--fast',
parser.add_option('--recoversync',
action='store_true',
dest='fastsync',
dest='recoversync',
default=False,
help=('choose to do fast wallet sync, only for Core and '
'only for previously synced wallet'))
help=('choose to do detailed wallet sync, '
'used for recovering on new Core instance.'))
parser.add_option(
'-x',
'--max-cj-fee-abs',
@ -358,6 +358,18 @@ def get_tumbler_parser():
default=9,
help=
'maximum amount of times to re-create a transaction before giving up, default 9')
# note that this is used slightly differently in tumbler from sendpayment,
# hence duplicated:
parser.add_option('-A',
'--amtmixdepths',
action='store',
type='int',
dest='amtmixdepths',
help='number of mixdepths ever used in wallet, '
'only to be used if mixdepths higher than '
'mixdepthsrc + number of mixdepths to tumble '
'have been used.',
default=-1)
add_common_options(parser)
return parser

162
scripts/joinmarket-qt.py

@ -72,7 +72,7 @@ from jmclient import load_program_config, get_network,\
open_test_wallet_maybe, get_wallet_path, get_p2sh_vbyte, get_p2pk_vbyte,\
jm_single, validate_address, weighted_order_choose, Taker,\
JMClientProtocolFactory, start_reactor, get_schedule, schedule_to_text,\
get_blockchain_interface_instance, direct_send,\
get_blockchain_interface_instance, direct_send, WalletService,\
RegtestBitcoinCoreInterface, tumbler_taker_finished_update,\
get_tumble_log, restart_wait, tumbler_filter_orders_callback,\
wallet_generate_recover_bip39, wallet_display, get_utxos_enabled_disabled
@ -621,12 +621,23 @@ class SpendTab(QWidget):
makercount = int(self.widgets[1][1].text())
mixdepth = int(self.widgets[2][1].text())
if makercount == 0:
txid = direct_send(w.wallet, amount, mixdepth,
txid = direct_send(w.wallet_service, amount, mixdepth,
destaddr, accept_callback=self.checkDirectSend,
info_callback=self.infoDirectSend)
if not txid:
self.giveUp()
else:
# since direct_send() assumes a one-shot processing, it does
# not add a callback for confirmation, so that event could
# get lost; we do that here to ensure that the confirmation
# event is noticed:
def qt_directsend_callback(rtxd, rtxid, confs):
if rtxid == txid:
return True
return False
w.wallet_service.active_txids.append(txid)
w.wallet_service.register_callbacks([qt_directsend_callback],
txid, cb_type="confirmed")
self.persistTxToHistory(destaddr, self.direct_send_amount, txid)
self.cleanUp()
return
@ -640,7 +651,7 @@ class SpendTab(QWidget):
self.startJoin()
def startJoin(self):
if not w.wallet:
if not w.wallet_service:
JMQtMessageBox(self, "Cannot start without a loaded wallet.",
mbtype="crit", title="Error")
return
@ -654,7 +665,7 @@ class SpendTab(QWidget):
check_offers_callback = None
destaddrs = self.tumbler_destaddrs if self.tumbler_options else []
self.taker = Taker(w.wallet,
self.taker = Taker(w.wallet_service,
self.spendstate.loaded_schedule,
order_chooser=weighted_order_choose,
callbacks=[check_offers_callback,
@ -797,7 +808,7 @@ class SpendTab(QWidget):
#TODO prob best to completely fold multiple and tumble to reduce
#complexity/duplication
if self.spendstate.typestate == 'multiple' and not self.tumbler_options:
self.taker.wallet.update_cache_index()
self.taker.wallet_service.save_wallet()
return
if fromtx:
if res:
@ -811,12 +822,6 @@ class SpendTab(QWidget):
self.nextTxTimer.start(int(waittime*60*1000))
#QtCore.QTimer.singleShot(int(self.taker_finished_waittime*60*1000),
# self.startNextTransaction)
#see note above re multiple/tumble duplication
if self.spendstate.typestate == 'multiple' and \
not self.tumbler_options:
txd, txid = txdetails
self.taker.wallet.remove_old_utxos(txd)
self.taker.wallet.add_new_utxos(txd, txid)
else:
if self.tumbler_options:
w.statusBar().showMessage("Transaction failed, trying again...")
@ -915,7 +920,7 @@ class SpendTab(QWidget):
if len(self.widgets[i][1].text()) == 0:
JMQtMessageBox(self, errs[i - 1], mbtype='warn', title="Error")
return False
if not w.wallet:
if not w.wallet_service:
JMQtMessageBox(self,
"There is no wallet loaded.",
mbtype='warn',
@ -1033,13 +1038,13 @@ class CoinsTab(QWidget):
self.cTW.addChild(m_item)
self.cTW.show()
if not w.wallet:
if not w.wallet_service:
show_blank()
return
utxos_enabled = {}
utxos_disabled = {}
for i in range(jm_single().config.getint("GUI", "max_mix_depth")):
utxos_e, utxos_d = get_utxos_enabled_disabled(w.wallet, i)
utxos_e, utxos_d = get_utxos_enabled_disabled(w.wallet_service, i)
if utxos_e != {}:
utxos_enabled[i] = utxos_e
if utxos_d != {}:
@ -1066,7 +1071,7 @@ class CoinsTab(QWidget):
# txid:index, btc, address
t = btc.safe_hexlify(k[0])+":"+str(k[1])
s = "{0:.08f}".format(v['value']/1e8)
a = w.wallet.script_to_addr(v["script"])
a = w.wallet_service.script_to_addr(v["script"])
item = QTreeWidgetItem([t, s, a])
item.setFont(0, QFont(MONOSPACE_FONT))
#if rows[i][forchange][j][3] != 'new':
@ -1076,7 +1081,7 @@ class CoinsTab(QWidget):
def toggle_utxo_disable(self, txid, idx):
txid_bytes = btc.safe_from_hex(txid)
w.wallet.toggle_disable_utxo(txid_bytes, idx)
w.wallet_service.toggle_disable_utxo(txid_bytes, idx)
self.updateUtxos()
def create_menu(self, position):
@ -1175,30 +1180,42 @@ class JMWalletTab(QWidget):
if xpub_exists:
menu.addAction("Copy extended pubkey to clipboard",
lambda: app.clipboard().setText(xpub))
menu.addAction("Resync wallet from blockchain",
lambda: w.resyncWallet())
#TODO add more items to context menu
menu.exec_(self.walletTree.viewport().mapToGlobal(position))
if address_valid or xpub_exists:
menu.exec_(self.walletTree.viewport().mapToGlobal(position))
def openQRCodePopup(self, address):
popup = BitcoinQRCodePopup(self, address)
popup.show()
def updateWalletInfo(self, walletinfo=None):
nm = jm_single().config.getint("GUI", "max_mix_depth")
l = self.walletTree
# before deleting, note whether items were expanded
esrs = []
for i in range(l.topLevelItemCount()):
tli = l.invisibleRootItem().child(i)
# must check top and also the two subitems (branches):
expandedness = tuple(
x.isExpanded() for x in [tli, tli.child(0), tli.child(1)])
esrs.append(expandedness)
l.clear()
if walletinfo:
self.mainwindow = self.parent().parent().parent()
rows, mbalances, xpubs, total_bal = walletinfo
if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == "regtest":
self.wallet_name = self.mainwindow.wallet.seed
self.wallet_name = self.mainwindow.testwalletname
else:
self.wallet_name = os.path.basename(self.mainwindow.wallet._storage.path)
self.wallet_name = os.path.basename(
self.mainwindow.wallet_service.get_storage_location())
if total_bal is None:
total_bal = " (syncing..)"
self.label1.setText("CURRENT WALLET: " + self.wallet_name +
', total balance: ' + total_bal)
self.walletTree.show()
l.show()
for i in range(jm_single().config.getint("GUI", "max_mix_depth")):
for i in range(nm):
if walletinfo:
mdbalance = mbalances[i]
else:
@ -1206,6 +1223,10 @@ class JMWalletTab(QWidget):
m_item = QTreeWidgetItem(["Mixdepth " + str(i) + " , balance: " +
mdbalance, '', '', '', ''])
l.addChild(m_item)
# if expansion states existed, reinstate them:
if len(esrs) == nm:
m_item.setExpanded(esrs[i][0])
for forchange in [0, 1]:
heading = "EXTERNAL" if forchange == 0 else "INTERNAL"
if walletinfo and heading == "EXTERNAL":
@ -1213,8 +1234,11 @@ class JMWalletTab(QWidget):
heading += heading_end
seq_item = QTreeWidgetItem([heading, '', '', '', ''])
m_item.addChild(seq_item)
# by default, external is expanded, but remember user choice:
if not forchange:
seq_item.setExpanded(True)
if len(esrs) == nm:
seq_item.setExpanded(esrs[i][forchange+1])
if not walletinfo:
item = QTreeWidgetItem(['None', '', '', ''])
seq_item.addChild(item)
@ -1234,7 +1258,14 @@ class JMMainWindow(QMainWindow):
def __init__(self, reactor):
super(JMMainWindow, self).__init__()
self.wallet = None
# the wallet service that encapsulates
# the wallet we will interact with
self.wallet_service = None
# the monitoring loop that queries
# the walletservice to update the GUI
self.walletRefresh = None
self.reactor = reactor
self.initUI()
@ -1290,7 +1321,7 @@ class JMMainWindow(QMainWindow):
msgbox.setWindowTitle(appWindowTitle)
label1 = QLabel()
label1.setText(
"<a href=" + "'https://github.com/joinmarket-org/joinmarket/wiki'>"
"<a href=" + "'https://github.com/joinmarket-org/joinmarket-clientserver/'>"
+ "Read more about Joinmarket</a><p>" + "<p>".join(
["Joinmarket core software version: " + JM_CORE_VERSION,
"JoinmarketQt version: " + JM_GUI_VERSION,
@ -1317,7 +1348,7 @@ class JMMainWindow(QMainWindow):
msgbox.exec_()
def exportPrivkeysJson(self):
if not self.wallet:
if not self.wallet_service:
JMQtMessageBox(self,
"No wallet loaded.",
mbtype='crit',
@ -1347,7 +1378,7 @@ class JMMainWindow(QMainWindow):
#option for anyone with gaplimit troubles, although
#that is a complete mess for a user, mostly changing
#the gaplimit in the Settings tab should address it.
rows = get_wallet_printout(self.wallet)
rows = get_wallet_printout(self.wallet_service)
addresses = []
for forchange in rows[0]:
for mixdepth in forchange:
@ -1361,7 +1392,7 @@ class JMMainWindow(QMainWindow):
time.sleep(0.1)
if done:
break
priv = self.wallet.get_key_from_addr(addr)
priv = self.wallet_service.get_key_from_addr(addr)
private_keys[addr] = btc.wif_compressed_privkey(
priv,
vbyte=get_p2pk_vbyte())
@ -1515,7 +1546,7 @@ class JMMainWindow(QMainWindow):
if firstarg:
wallet_path = get_wallet_path(str(firstarg), None)
try:
self.wallet = open_test_wallet_maybe(wallet_path, str(firstarg),
wallet = open_test_wallet_maybe(wallet_path, str(firstarg),
None, ask_for_password=False, password=pwd.encode('utf-8') if pwd else None,
gap_limit=jm_single().config.getint("GUI", "gaplimit"))
except Exception as e:
@ -1524,58 +1555,40 @@ class JMMainWindow(QMainWindow):
mbtype='warn',
title="Error")
return False
self.wallet.seed = str(firstarg)
# only used for GUI display on regtest:
self.testwalletname = wallet.seed = str(firstarg)
if 'listunspent_args' not in jm_single().config.options('POLICY'):
jm_single().config.set('POLICY', 'listunspent_args', '[0]')
assert self.wallet, "No wallet loaded"
reactor.callLater(0, self.syncWalletUpdate, True, restart_cb)
assert wallet, "No wallet loaded"
# shut down any existing wallet service
# monitoring loops
if self.wallet_service is not None:
if self.wallet_service.isRunning():
self.wallet_service.stopService()
if self.walletRefresh is not None:
self.walletRefresh.stop()
self.wallet_service = WalletService(wallet)
self.wallet_service.add_restart_callback(restart_cb)
self.wallet_service.startService()
self.walletRefresh = task.LoopingCall(self.updateWalletInfo)
self.walletRefresh.start(5.0)
self.statusBar().showMessage("Reading wallet from blockchain ...")
return True
def syncWalletUpdate(self, fast, restart_cb=None):
if restart_cb:
fast=False
#Special syncing condition for Electrum
iselectrum = jm_single().config.get("BLOCKCHAIN",
"blockchain_source") == "electrum-server"
if iselectrum:
jm_single().bc_interface.synctype = "with-script"
jm_single().bc_interface.sync_wallet(self.wallet, fast=fast,
restart_cb=restart_cb)
if iselectrum:
#sync_wallet only initialises, we must manually call its entry
#point here (because we can't use connectionMade as a trigger)
jm_single().bc_interface.sync_addresses(self.wallet)
self.wait_for_sync_loop = task.LoopingCall(self.updateWalletInfo)
self.wait_for_sync_loop.start(0.2)
else:
self.updateWalletInfo()
def updateWalletInfo(self):
if jm_single().config.get("BLOCKCHAIN",
"blockchain_source") == "electrum-server":
if not jm_single().bc_interface.wallet_synced:
return
self.wait_for_sync_loop.stop()
t = self.centralWidget().widget(0)
if not self.wallet: #failure to sync in constructor means object is not created
if not self.wallet_service: #failure to sync in constructor means object is not created
newstmsg = "Unable to sync wallet - see error in console."
elif not self.wallet_service.synced:
return
else:
t.updateWalletInfo(get_wallet_printout(self.wallet))
t.updateWalletInfo(get_wallet_printout(self.wallet_service))
newstmsg = "Wallet synced successfully."
self.statusBar().showMessage(newstmsg)
def resyncWallet(self):
if not self.wallet:
JMQtMessageBox(self,
"No wallet loaded",
mbtype='warn',
title="Error")
return
self.loadWalletFromBlockchain()
def generateWallet(self):
log.debug('generating wallet')
if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == "regtest":
@ -1685,8 +1698,8 @@ class JMMainWindow(QMainWindow):
self.loadWalletFromBlockchain(self.walletname, pwd=self.textpassword,
restart_cb=restart_cb)
def get_wallet_printout(wallet):
"""Given a joinmarket wallet, retrieve the list of
def get_wallet_printout(wallet_service):
"""Given a WalletService object, retrieve the list of
addresses and corresponding balances to be displayed.
We retrieve a WalletView abstraction, and iterate over
sub-objects to arrange the per-mixdepth and per-address lists.
@ -1697,7 +1710,7 @@ def get_wallet_printout(wallet):
xpubs: [[xpubext, xpubint], ...]
Bitcoin amounts returned are in btc, not satoshis
"""
walletview = wallet_display(wallet, jm_single().config.getint("GUI",
walletview = wallet_display(wallet_service, jm_single().config.getint("GUI",
"gaplimit"), False, serialized=False)
rows = []
mbalances = []
@ -1714,7 +1727,10 @@ def get_wallet_printout(wallet):
entry.serialize_wallet_position(),
entry.serialize_amounts(),
entry.serialize_extra_data()])
return (rows, mbalances, xpubs, walletview.get_fmt_balance())
# in case the wallet is not yet synced, don't return an incorrect
# 0 balance, but signal incompleteness:
total_bal = walletview.get_fmt_balance() if wallet_service.synced else None
return (rows, mbalances, xpubs, total_bal)
################################
config_load_error = False

10
scripts/receive-payjoin.py

@ -20,12 +20,12 @@ def receive_payjoin_main(makerclass):
parser.add_option('-g', '--gap-limit', action='store', type="int",
dest='gaplimit', default=6,
help='gap limit for wallet, default=6')
parser.add_option('--fast',
parser.add_option('--recoversync',
action='store_true',
dest='fastsync',
dest='recoversync',
default=False,
help=('choose to do fast wallet sync, only for Core and '
'only for previously synced wallet'))
help=('choose to do detailed wallet sync, '
'used for recovering on new Core instance.'))
parser.add_option('-m', '--mixdepth', action='store', type='int',
dest='mixdepth', default=0,
help="mixdepth to source coins from")
@ -68,7 +68,7 @@ def receive_payjoin_main(makerclass):
jm_single().bc_interface.synctype = "with-script"
while not jm_single().bc_interface.wallet_synced:
sync_wallet(wallet, fast=options.fastsync)
sync_wallet(wallet, fast=not options.recoversync)
maker = makerclass(wallet, options.mixdepth, receiving_amount)

25
scripts/sendpayment.py

@ -16,7 +16,7 @@ import pprint
from jmclient import Taker, P2EPTaker, load_program_config, get_schedule,\
JMClientProtocolFactory, start_reactor, validate_address, jm_single,\
sync_wallet, estimate_tx_fee, direct_send,\
estimate_tx_fee, direct_send, WalletService,\
open_test_wallet_maybe, get_wallet_path
from twisted.python.log import startLogging
from jmbase.support import get_log, set_logging_level, jmprint
@ -128,19 +128,20 @@ def main():
wallet_path = get_wallet_path(wallet_name, None)
wallet = open_test_wallet_maybe(
wallet_path, wallet_name, max_mix_depth, gap_limit=options.gaplimit)
wallet_service = WalletService(wallet)
# in this script, we need the wallet synced before
# logic processing for some paths, so do it now:
while not wallet_service.synced:
wallet_service.sync_wallet(fast=not options.recoversync)
# the sync call here will now be a no-op:
wallet_service.startService()
if jm_single().config.get("BLOCKCHAIN",
"blockchain_source") == "electrum-server" and options.makercount != 0:
jm_single().bc_interface.synctype = "with-script"
#wallet sync will now only occur on reactor start if we're joining.
while not jm_single().bc_interface.wallet_synced:
sync_wallet(wallet, fast=options.fastsync)
# From the estimated tx fees, check if the expected amount is a
# significant value compared the the cj amount
total_cj_amount = amount
if total_cj_amount == 0:
total_cj_amount = wallet.get_balance_by_mixdepth()[options.mixdepth]
total_cj_amount = wallet_service.get_balance_by_mixdepth()[options.mixdepth]
if total_cj_amount == 0:
raise ValueError("No confirmed coins in the selected mixdepth. Quitting")
exp_tx_fees_ratio = ((1 + options.makercount) * options.txfee) / total_cj_amount
@ -155,7 +156,7 @@ def main():
.format(exp_tx_fees_ratio))
if options.makercount == 0 and not options.p2ep:
direct_send(wallet, amount, mixdepth, destaddr, options.answeryes)
direct_send(wallet_service, amount, mixdepth, destaddr, options.answeryes)
return
if wallet.get_txtype() == 'p2pkh':
@ -193,8 +194,6 @@ def main():
if fromtx:
if res:
txd, txid = txdetails
taker.wallet.remove_old_utxos(txd)
taker.wallet.add_new_utxos(txd, txid)
reactor.callLater(waittime*60,
clientfactory.getClient().clientStart)
else:
@ -259,10 +258,10 @@ def main():
txdetails=None):
log.error("PayJoin payment was NOT made, timed out.")
reactor.stop()
taker = P2EPTaker(options.p2ep, wallet, schedule,
taker = P2EPTaker(options.p2ep, wallet_service, schedule,
callbacks=(None, None, p2ep_on_finished_callback))
else:
taker = Taker(wallet,
taker = Taker(wallet_service,
schedule,
order_chooser=chooseOrdersFunc,
max_cj_fee=maxcjfee,

22
scripts/tumbler.py

@ -10,8 +10,8 @@ import pprint
from twisted.python.log import startLogging
from jmclient import Taker, load_program_config, get_schedule,\
JMClientProtocolFactory, start_reactor, jm_single, get_wallet_path,\
open_test_wallet_maybe, sync_wallet, get_tumble_schedule,\
schedule_to_text, estimate_tx_fee, restart_waiter,\
open_test_wallet_maybe, get_tumble_schedule,\
schedule_to_text, estimate_tx_fee, restart_waiter, WalletService,\
get_tumble_log, tumbler_taker_finished_update,\
tumbler_filter_orders_callback, validate_address
from jmbase.support import get_log, jmprint
@ -38,13 +38,17 @@ def main():
#Load the wallet
wallet_name = args[0]
max_mix_depth = options['mixdepthsrc'] + options['mixdepthcount']
if options['amtmixdepths'] > max_mix_depth:
max_mix_depth = options['amtmixdepths']
wallet_path = get_wallet_path(wallet_name, None)
wallet = open_test_wallet_maybe(wallet_path, wallet_name, max_mix_depth)
if jm_single().config.get("BLOCKCHAIN",
"blockchain_source") == "electrum-server":
jm_single().bc_interface.synctype = "with-script"
while not jm_single().bc_interface.wallet_synced:
sync_wallet(wallet, fast=options['fastsync'])
wallet_service = WalletService(wallet)
# in this script, we need the wallet synced before
# logic processing for some paths, so do it now:
while not wallet_service.synced:
wallet_service.sync_wallet(fast=not options['recoversync'])
# the sync call here will now be a no-op:
wallet_service.startService()
maxcjfee = get_max_cj_fee_values(jm_single().config, options_org)
log.info("Using maximum coinjoin fee limits per maker of {:.4%}, {} sat"
@ -128,7 +132,7 @@ def main():
max_mix_to_tumble = min(options['mixdepthsrc']+options['mixdepthcount'], \
max_mix_depth)
for i in range(options['mixdepthsrc'], max_mix_to_tumble):
total_tumble_amount += wallet.get_balance_by_mixdepth()[i]
total_tumble_amount += wallet_service.get_balance_by_mixdepth()[i]
if total_tumble_amount == 0:
raise ValueError("No confirmed coins in the selected mixdepth(s). Quitting")
exp_tx_fees_ratio = (involved_parties * options['txfee']) \
@ -164,7 +168,7 @@ def main():
reactor.callLater(waittime*60, clientfactory.getClient().clientStart)
#instantiate Taker with given schedule and run
taker = Taker(wallet,
taker = Taker(wallet_service,
schedule,
order_chooser=options['order_choose_fn'],
max_cj_fee=maxcjfee,

4
scripts/yg-privacyenhanced.py

@ -34,8 +34,8 @@ jlog = get_log()
class YieldGeneratorPrivacyEnhanced(YieldGeneratorBasic):
def __init__(self, wallet, offerconfig):
super(YieldGeneratorPrivacyEnhanced, self).__init__(wallet, offerconfig)
def __init__(self, wallet_service, offerconfig):
super(YieldGeneratorPrivacyEnhanced, self).__init__(wallet_service, offerconfig)
def create_my_orders(self):
mix_balance = self.get_available_mixdepths()

18
test/common.py

@ -15,27 +15,28 @@ sys.path.insert(0, os.path.join(data_dir))
from jmbase import get_log
from jmclient import open_test_wallet_maybe, BIP32Wallet, SegwitLegacyWallet, \
estimate_tx_fee, jm_single
estimate_tx_fee, jm_single, WalletService
import jmbitcoin as btc
from jmbase import chunks
log = get_log()
def make_sign_and_push(ins_full,
wallet,
wallet_service,
amount,
output_addr=None,
change_addr=None,
hashcode=btc.SIGHASH_ALL,
estimate_fee = False):
"""Utility function for easily building transactions
from wallets
from wallets.
"""
assert isinstance(wallet_service, WalletService)
total = sum(x['value'] for x in ins_full.values())
ins = ins_full.keys()
#random output address and change addr
output_addr = wallet.get_new_addr(1, 1) if not output_addr else output_addr
change_addr = wallet.get_new_addr(1, 0) if not change_addr else change_addr
output_addr = wallet_service.get_new_addr(1, 1) if not output_addr else output_addr
change_addr = wallet_service.get_new_addr(0, 1) if not change_addr else change_addr
fee_est = estimate_tx_fee(len(ins), 2) if estimate_fee else 10000
outs = [{'value': amount,
'address': output_addr}, {'value': total - amount - fee_est,
@ -46,7 +47,7 @@ def make_sign_and_push(ins_full,
for index, ins in enumerate(de_tx['ins']):
utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
addr = ins_full[utxo]['address']
priv = wallet.get_key_from_addr(addr)
priv = wallet_service.get_key_from_addr(addr)
if index % 2:
priv = binascii.unhexlify(priv)
tx = btc.sign(tx, index, priv, hashcode=hashcode)
@ -99,15 +100,16 @@ def make_wallets(n,
w = open_test_wallet_maybe(seeds[i], seeds[i], mixdepths - 1,
test_wallet_cls=walletclass)
wallet_service = WalletService(w)
wallets[i + start_index] = {'seed': seeds[i].decode('ascii'),
'wallet': w}
'wallet': wallet_service}
for j in range(mixdepths):
for k in range(wallet_structures[i][j]):
deviation = sdev_amt * random.random()
amt = mean_amt - sdev_amt / 2.0 + deviation
if amt < 0: amt = 0.001
amt = float(Decimal(amt).quantize(Decimal(10)**-8))
jm_single().bc_interface.grab_coins(w.get_external_addr(j), amt)
jm_single().bc_interface.grab_coins(wallet_service.get_new_addr(j, 1), amt)
return wallets

2
test/test_full_coinjoin.py

@ -57,7 +57,7 @@ def test_cj(setup_full_coinjoin, num_ygs, wallet_structures, mean_amt,
wallet = wallets[num_ygs]['wallet']
sync_wallet(wallet, fast=True)
# grab a dest addr from the wallet
destaddr = wallet.get_new_addr(4, 0)
destaddr = wallet.get_external_addr(4)
coinjoin_amt = 20000000
schedule = [[1, coinjoin_amt, 2, destaddr,
0.0, False]]

29
test/test_segwit.py

@ -60,14 +60,14 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount,
MIXDEPTH = 0
# set up wallets and inputs
nsw_wallet = make_wallets(1, wallet_structure, in_amt,
nsw_wallet_service = make_wallets(1, wallet_structure, in_amt,
walletclass=LegacyWallet)[0]['wallet']
jm_single().bc_interface.sync_wallet(nsw_wallet, fast=True)
sw_wallet = make_wallets(1, [[len(segwit_ins), 0, 0, 0, 0]], segwit_amt)[0]['wallet']
jm_single().bc_interface.sync_wallet(sw_wallet, fast=True)
nsw_wallet_service.sync_wallet(fast=True)
sw_wallet_service = make_wallets(1, [[len(segwit_ins), 0, 0, 0, 0]], segwit_amt)[0]['wallet']
sw_wallet_service.sync_wallet(fast=True)
nsw_utxos = nsw_wallet.get_utxos_by_mixdepth_()[MIXDEPTH]
sw_utxos = sw_wallet.get_utxos_by_mixdepth_()[MIXDEPTH]
nsw_utxos = nsw_wallet_service.get_utxos_by_mixdepth(hexfmt=False)[MIXDEPTH]
sw_utxos = sw_wallet_service.get_utxos_by_mixdepth(hexfmt=False)[MIXDEPTH]
assert len(o_ins) <= len(nsw_utxos), "sync failed"
assert len(segwit_ins) <= len(sw_utxos), "sync failed"
@ -102,8 +102,8 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount,
FEE = 50000
assert FEE < total_amt_in_sat - amount, "test broken, not enough funds"
cj_script = nsw_wallet.get_new_script(MIXDEPTH + 1, True)
change_script = nsw_wallet.get_new_script(MIXDEPTH, True)
cj_script = nsw_wallet_service.get_new_script(MIXDEPTH + 1, True)
change_script = nsw_wallet_service.get_new_script(MIXDEPTH, True)
change_amt = total_amt_in_sat - amount - FEE
tx_outs = [
@ -115,22 +115,21 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount,
# import new addresses to bitcoind
jm_single().bc_interface.import_addresses(
[nsw_wallet.script_to_addr(x)
for x in [cj_script, change_script]],
jm_single().bc_interface.get_wallet_name(nsw_wallet))
[nsw_wallet_service.script_to_addr(x)
for x in [cj_script, change_script]], nsw_wallet_service.get_wallet_name())
# sign tx
scripts = {}
for nsw_in_index in o_ins:
inp = nsw_ins[nsw_in_index][1]
scripts[nsw_in_index] = (inp['script'], inp['value'])
tx = nsw_wallet.sign_tx(tx, scripts)
tx = nsw_wallet_service.sign_tx(tx, scripts)
scripts = {}
for sw_in_index in segwit_ins:
inp = sw_ins[sw_in_index][1]
scripts[sw_in_index] = (inp['script'], inp['value'])
tx = sw_wallet.sign_tx(tx, scripts)
tx = sw_wallet_service.sign_tx(tx, scripts)
print(tx)
@ -139,8 +138,8 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount,
assert txid
balances = jm_single().bc_interface.get_received_by_addr(
[nsw_wallet.script_to_addr(cj_script),
nsw_wallet.script_to_addr(change_script)], None)['data']
[nsw_wallet_service.script_to_addr(cj_script),
nsw_wallet_service.script_to_addr(change_script)], None)['data']
assert balances[0]['balance'] == amount
assert balances[1]['balance'] == change_amt

17
test/ygrunner.py

@ -19,7 +19,7 @@ import pytest
import random
from jmbase import jmprint
from jmclient import YieldGeneratorBasic, load_program_config, jm_single,\
sync_wallet, JMClientProtocolFactory, start_reactor, SegwitWallet,\
JMClientProtocolFactory, start_reactor, SegwitWallet,\
SegwitLegacyWallet, cryptoengine
@ -109,21 +109,21 @@ def test_start_ygs(setup_ygrunner, num_ygs, wallet_structures, mean_amt,
# TODO add Legacy
walletclass = SegwitLegacyWallet
wallets = make_wallets(num_ygs + 1,
wallet_services = make_wallets(num_ygs + 1,
wallet_structures=wallet_structures,
mean_amt=mean_amt,
walletclass=walletclass)
#the sendpayment bot uses the last wallet in the list
wallet = wallets[num_ygs]['wallet']
jmprint("\n\nTaker wallet seed : " + wallets[num_ygs]['seed'])
wallet_service = wallet_services[num_ygs]['wallet']
jmprint("\n\nTaker wallet seed : " + wallet_services[num_ygs]['seed'])
# for manual audit if necessary, show the maker's wallet seeds
# also (note this audit should be automated in future, see
# test_full_coinjoin.py in this directory)
jmprint("\n\nMaker wallet seeds: ")
for i in range(num_ygs):
jmprint("Maker seed: " + wallets[i]['seed'])
jmprint("Maker seed: " + wallet_services[i]['seed'])
jmprint("\n")
sync_wallet(wallet, fast=True)
wallet_service.sync_wallet(fast=True)
txfee = 1000
cjfee_a = 4200
cjfee_r = '0.001'
@ -138,8 +138,9 @@ def test_start_ygs(setup_ygrunner, num_ygs, wallet_structures, mean_amt,
for i in range(num_ygs):
cfg = [txfee, cjfee_a, cjfee_r, ordertype, minsize]
sync_wallet(wallets[i]["wallet"], fast=True)
yg = ygclass(wallets[i]["wallet"], cfg)
wallet_service_yg = wallet_services[i]["wallet"]
wallet_service_yg.startService()
yg = ygclass(wallet_service_yg, cfg)
if malicious:
yg.set_maliciousness(malicious, mtype="tx")
clientfactory = JMClientProtocolFactory(yg, proto_type="MAKER")

Loading…
Cancel
Save