Browse Source

Merge #197: Fix mixdepth behaviour

310fac8 add test for wallet.mixdepth (undeath)
3921882 change wallet mixdepth behaviour (undeath)
4f23647 revert 98bcc86446 (undeath)
master
AdamISZ 7 years ago
parent
commit
99690bd5f7
No known key found for this signature in database
GPG Key ID: B3AE09F1E9A3197A
  1. 2
      jmclient/jmclient/taker.py
  2. 61
      jmclient/jmclient/wallet.py
  3. 72
      jmclient/jmclient/wallet_utils.py
  4. 10
      jmclient/jmclient/yieldgenerator.py
  5. 55
      jmclient/test/test_wallet.py
  6. 8
      scripts/convert_old_wallet.py
  7. 2
      scripts/joinmarket-qt.py
  8. 4
      scripts/sendpayment.py
  9. 7
      scripts/tumbler.py
  10. 2
      test/common.py

2
jmclient/jmclient/taker.py

@ -177,7 +177,7 @@ class Taker(object):
#if destination is flagged "INTERNAL", choose a destination
#from the next mixdepth modulo the maxmixdepth
if self.my_cj_addr == "INTERNAL":
next_mixdepth = (self.mixdepth + 1) % (self.wallet.max_mixdepth + 1)
next_mixdepth = (self.mixdepth + 1) % (self.wallet.mixdepth + 1)
jlog.info("Choosing a destination from mixdepth: " + str(next_mixdepth))
self.my_cj_addr = self.wallet.get_internal_addr(next_mixdepth)
jlog.info("Chose destination address: " + self.my_cj_addr)

61
jmclient/jmclient/wallet.py

@ -169,9 +169,11 @@ class UTXOManager(object):
'value': utxos[s['utxo']][1]}
for s in selected}
def get_balance_by_mixdepth(self):
def get_balance_by_mixdepth(self, max_mixdepth=float('Inf')):
balance_dict = collections.defaultdict(int)
for mixdepth, utxomap in self._utxo.items():
if mixdepth > max_mixdepth:
continue
value = sum(x[1] for x in utxomap.values())
balance_dict[mixdepth] = value
return balance_dict
@ -201,7 +203,8 @@ class BaseWallet(object):
_ENGINE = None
def __init__(self, storage, gap_limit=6, merge_algorithm_name=None):
def __init__(self, storage, gap_limit=6, merge_algorithm_name=None,
mixdepth=None):
# to be defined by inheriting classes
assert self.TYPE is not None
assert self._ENGINE is not None
@ -210,7 +213,10 @@ class BaseWallet(object):
self.gap_limit = gap_limit
self._storage = storage
self._utxos = None
# highest mixdepth ever used in wallet, important for synching
self.max_mixdepth = None
# effective maximum mixdepth to be used by joinmarket
self.mixdepth = None
self.network = None
# {script: path}, should always hold mappings for all "known" keys
@ -223,10 +229,22 @@ class BaseWallet(object):
assert self.max_mixdepth >= 0
assert self.network in ('mainnet', 'testnet')
if mixdepth is not None:
assert mixdepth >= 0
if self._storage.read_only and mixdepth > self.max_mixdepth:
raise Exception("Effective max mixdepth must be at most {}!"
.format(self.max_mixdepth))
self.max_mixdepth = max(self.max_mixdepth, mixdepth)
self.mixdepth = mixdepth
else:
self.mixdepth = self.max_mixdepth
assert self.mixdepth is not None
@property
@deprecated
def max_mix_depth(self):
return self.max_mixdepth
return self.mixdepth
@property
@deprecated
@ -241,14 +259,12 @@ class BaseWallet(object):
raise Exception("Wrong class to initialize wallet of type {}."
.format(self.TYPE))
self.network = self._storage.data[b'network'].decode('ascii')
self.max_mixdepth = self._storage.data[b'max_mixdepth']
self._utxos = UTXOManager(self._storage, self.merge_algorithm)
def save(self):
"""
Write data to associated storage object and trigger persistent update.
"""
self._storage.data[b'max_mixdepth'] = self.max_mixdepth
self._storage.save()
@classmethod
@ -277,7 +293,6 @@ class BaseWallet(object):
timestamp = datetime.now().strftime('%Y/%m/%d %H:%M:%S')
storage.data[b'network'] = network.encode('ascii')
storage.data[b'max_mixdepth'] = max_mixdepth
storage.data[b'created'] = timestamp.encode('ascii')
storage.data[b'wallet_type'] = cls.TYPE
@ -517,8 +532,12 @@ class BaseWallet(object):
def select_utxos_(self, mixdepth, amount, utxo_filter=None):
"""
Select a subset of available UTXOS for a given mixdepth whose value is
greater or equal to amount.
args:
mixdepth: int, mixdepth to select utxos from
mixdepth: int, mixdepth to select utxos from, must be smaller or
equal to wallet.max_mixdepth
amount: int, total minimum amount of all selected utxos
utxo_filter: list of (txid, index), utxos not to select
@ -545,8 +564,13 @@ class BaseWallet(object):
self._utxos.reset()
def get_balance_by_mixdepth(self, verbose=True):
"""
Get available funds in each active mixdepth.
returns: {mixdepth: value}
"""
# TODO: verbose
return self._utxos.get_balance_by_mixdepth()
return self._utxos.get_balance_by_mixdepth(max_mixdepth=self.mixdepth)
@deprecated
def get_utxos_by_mixdepth(self, verbose=True):
@ -564,6 +588,8 @@ class BaseWallet(object):
def get_utxos_by_mixdepth_(self):
"""
Get all UTXOs for active mixdepths.
returns:
{mixdepth: {(txid, index):
{'script': bytes, 'path': tuple, 'value': int}}}
@ -572,6 +598,8 @@ class BaseWallet(object):
script_utxos = collections.defaultdict(dict)
for md, data in mix_utxos.items():
if md > self.mixdepth:
continue
for utxo, (path, value) in data.items():
script = self.get_script_path(path)
script_utxos[md][utxo] = {'script': script,
@ -771,12 +799,11 @@ class ImportWalletMixin(object):
_IMPORTED_STORAGE_KEY = b'imported_keys'
_IMPORTED_ROOT_PATH = b'imported'
def __init__(self, storage, gap_limit=6, merge_algorithm_name=None):
def __init__(self, storage, **kwargs):
# {mixdepth: [(privkey, type)]}
self._imported = None
# path is (_IMPORTED_ROOT_PATH, mixdepth, key_index)
super(ImportWalletMixin, self).__init__(storage, gap_limit,
merge_algorithm_name)
super(ImportWalletMixin, self).__init__(storage, **kwargs)
def _load_storage(self):
super(ImportWalletMixin, self)._load_storage()
@ -1020,15 +1047,14 @@ class BIP32Wallet(BaseWallet):
BIP32_INT_ID = 1
ENTROPY_BYTES = 16
def __init__(self, storage, gap_limit=6, merge_algorithm_name=None):
def __init__(self, storage, **kwargs):
self._entropy = None
# {mixdepth: {type: index}} with type being 0/1 for [non]-internal
self._index_cache = None
# path is a tuple of BIP32 levels,
# m is the master key's fingerprint
# other levels are ints
super(BIP32Wallet, self).__init__(storage, gap_limit,
merge_algorithm_name)
super(BIP32Wallet, self).__init__(storage, **kwargs)
assert self._index_cache is not None
assert self._verify_entropy(self._entropy)
@ -1060,7 +1086,8 @@ class BIP32Wallet(BaseWallet):
entropy = get_random_bytes(cls.ENTROPY_BYTES, True)
storage.data[cls._STORAGE_ENTROPY_KEY] = entropy
storage.data[cls._STORAGE_INDEX_CACHE] = {}
storage.data[cls._STORAGE_INDEX_CACHE] = {
_int_to_bytestr(i): {} for i in range(max_mixdepth + 1)}
if write:
storage.save()
@ -1074,12 +1101,12 @@ class BIP32Wallet(BaseWallet):
for md, data in self._storage.data[self._STORAGE_INDEX_CACHE].items():
md = int(md)
if md > self.max_mixdepth:
continue
md_map = self._index_cache[md]
for t, k in data.items():
md_map[int(t)] = k
self.max_mixdepth = max(0, 0, *self._index_cache.keys())
def _populate_script_map(self):
for md in self._index_cache:
for int_type in (self.BIP32_EXT_ID, self.BIP32_INT_ID):

72
jmclient/jmclient/wallet_utils.py

@ -16,6 +16,10 @@ from cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH
import jmclient.btc as btc
# used for creating new wallets
DEFAULT_MIXDEPTH = 4
def get_wallettool_parser():
description = (
'Use this script to monitor and manage your Joinmarket wallet.\n'
@ -31,10 +35,7 @@ def get_wallettool_parser():
'(importprivkey) Adds privkeys to this wallet, privkeys are spaces or commas separated.\n'
'(dumpprivkey) Export a single private key, specify an hd wallet path\n'
'(signmessage) Sign a message with the private key from an address in \n'
'the wallet. Use with -H and specify an HD wallet path for the address.\n'
'(changemixdepth) Use with -M to change the *maximum* number of mixdepths\n'
'in the wallet; you are advised to only increase, not decrease this \n'
'number from the current value (initially 5).')
'the wallet. Use with -H and specify an HD wallet path for the address.')
parser = OptionParser(usage='usage: %prog [options] [wallet file] [method]',
description=description)
parser.add_option('-p',
@ -43,12 +44,13 @@ def get_wallettool_parser():
dest='showprivkey',
help='print private key along with address, default false')
parser.add_option('-m',
'--maxmixdepth',
'--mixdepth',
action='store',
type='int',
dest='mixdepths',
help='how many mixing depths to initialize in the wallet',
default=5)
dest='mixdepth',
help="Mixdepth(s) to use in the wallet. Default: {}"
.format(DEFAULT_MIXDEPTH),
default=None)
parser.add_option('-g',
'--gap-limit',
type="int",
@ -56,13 +58,6 @@ def get_wallettool_parser():
dest='gaplimit',
help='gap limit for wallet, default=6',
default=6)
parser.add_option('-M',
'--mix-depth',
type="int",
action='store',
dest='mixdepth',
help='mixing depth to import private key into',
default=0)
parser.add_option('--csv',
action='store_true',
dest='csv',
@ -347,7 +342,7 @@ def wallet_display(wallet, gaplimit, showprivkey, displayall=False,
else return the WalletView object.
"""
acctlist = []
for m in xrange(wallet.max_mixdepth + 1):
for m in xrange(wallet.mixdepth + 1):
branchlist = []
for forchange in [0, 1]:
entrylist = []
@ -429,7 +424,7 @@ def cli_get_mnemonic_extension():
def wallet_generate_recover_bip39(method, walletspath, default_wallet_name,
mixdepths=5,
mixdepth=DEFAULT_MIXDEPTH,
callbacks=(cli_display_user_words,
cli_user_mnemonic_entry,
cli_get_wallet_passphrase_check,
@ -469,7 +464,7 @@ def wallet_generate_recover_bip39(method, walletspath, default_wallet_name,
wallet_name = default_wallet_name
wallet_path = os.path.join(walletspath, wallet_name)
wallet = create_wallet(wallet_path, password, mixdepths - 1,
wallet = create_wallet(wallet_path, password, mixdepth,
entropy=entropy,
entropy_extension=mnemonic_extension)
mnemonic, mnext = wallet.get_mnemonic_words()
@ -480,11 +475,11 @@ def wallet_generate_recover_bip39(method, walletspath, default_wallet_name,
def wallet_generate_recover(method, walletspath,
default_wallet_name='wallet.jmdat',
mixdepths=5):
mixdepth=DEFAULT_MIXDEPTH):
if is_segwit_mode():
#Here using default callbacks for scripts (not used in Qt)
return wallet_generate_recover_bip39(
method, walletspath, default_wallet_name, mixdepths=mixdepths)
method, walletspath, default_wallet_name, mixdepth=mixdepth)
entropy = None
if method == 'recover':
@ -507,7 +502,7 @@ def wallet_generate_recover(method, walletspath,
wallet_name = default_wallet_name
wallet_path = os.path.join(walletspath, wallet_name)
wallet = create_wallet(wallet_path, password, mixdepths - 1,
wallet = create_wallet(wallet_path, password, mixdepth,
wallet_cls=LegacyWallet, entropy=entropy)
print("Write down and safely store this wallet recovery seed\n\n{}\n"
.format(wallet.get_mnemonic_words()[0]))
@ -855,11 +850,6 @@ def get_wallet_cls(wtype=None):
return cls
def change_wallet_mixdepth(wallet, max_mixdepth):
wallet.max_mixdepth = max_mixdepth
wallet.save()
return "Maximum mixdepth successfully updated."
def create_wallet(path, password, max_mixdepth, wallet_cls=None, **kwargs):
storage = Storage(path, password, create=True)
wallet_cls = wallet_cls or get_wallet_cls()
@ -878,7 +868,7 @@ def open_test_wallet_maybe(path, seed, max_mixdepth,
params:
path: path to wallet file, ignored for test wallets
seed: hex-encoded test seed
max_mixdepth: see create_wallet(), ignored when calling open_wallet()
max_mixdepth: maximum mixdepth to use
kwargs: see open_wallet()
returns:
@ -890,6 +880,9 @@ def open_test_wallet_maybe(path, seed, max_mixdepth,
except binascii.Error:
pass
else:
if max_mixdepth is None:
max_mixdepth = DEFAULT_MIXDEPTH
storage = VolatileStorage()
test_wallet_cls.initialize(
storage, get_network(), max_mixdepth=max_mixdepth,
@ -901,11 +894,11 @@ def open_test_wallet_maybe(path, seed, max_mixdepth,
del kwargs['ask_for_password']
if 'password' in kwargs:
del kwargs['password']
assert 'ask_for_password' not in kwargs
assert 'read_only' not in kwargs
if 'read_only' in kwargs:
del kwargs['read_only']
return test_wallet_cls(storage, **kwargs)
return open_wallet(path, **kwargs)
return open_wallet(path, mixdepth=max_mixdepth, **kwargs)
def open_wallet(path, ask_for_password=True, password=None, read_only=False,
@ -986,8 +979,7 @@ def wallet_tool_main(wallet_root_path):
methods = ['display', 'displayall', 'summary', 'showseed', 'importprivkey',
'history', 'showutxos']
methods.extend(noseed_methods)
noscan_methods = ['showseed', 'importprivkey', 'dumpprivkey',
'signmessage', 'changemixdepth']
noscan_methods = ['showseed', 'importprivkey', 'dumpprivkey', 'signmessage']
readonly_methods = ['display', 'displayall', 'summary', 'showseed',
'history', 'showutxos', 'dumpprivkey', 'signmessage']
@ -995,12 +987,14 @@ def wallet_tool_main(wallet_root_path):
parser.error('Needs a wallet file or method')
sys.exit(0)
if options.mixdepths < 1:
if options.mixdepth is not None and options.mixdepth < 0:
parser.error("Must have at least one mixdepth.")
sys.exit(0)
if args[0] in noseed_methods:
method = args[0]
if options.mixdepth is None:
options.mixdepth = DEFAULT_MIXDEPTH
else:
seed = args[0]
wallet_path = get_wallet_path(seed, wallet_root_path)
@ -1008,7 +1002,7 @@ def wallet_tool_main(wallet_root_path):
read_only = method in readonly_methods
wallet = open_test_wallet_maybe(
wallet_path, seed, options.mixdepths - 1, read_only=read_only,
wallet_path, seed, options.mixdepth, read_only=read_only,
gap_limit=options.gaplimit)
if method not in noscan_methods:
@ -1035,11 +1029,11 @@ def wallet_tool_main(wallet_root_path):
return wallet_fetch_history(wallet, options)
elif method == "generate":
retval = wallet_generate_recover("generate", wallet_root_path,
mixdepths=options.mixdepths)
mixdepth=options.mixdepth)
return retval if retval else "Failed"
elif method == "recover":
retval = wallet_generate_recover("recover", wallet_root_path,
mixdepths=options.mixdepths)
mixdepth=options.mixdepth)
return retval if retval else "Failed"
elif method == "showutxos":
return wallet_showutxos(wallet, options.showprivkey)
@ -1049,15 +1043,13 @@ def wallet_tool_main(wallet_root_path):
return wallet_dumpprivkey(wallet, options.hd_path)
elif method == "importprivkey":
#note: must be interactive (security)
if options.mixdepth is None:
parser.error("You need to specify a mixdepth with -m")
wallet_importprivkey(wallet, options.mixdepth,
map_key_type(options.key_type))
return "Key import completed."
elif method == "signmessage":
return wallet_signmessage(wallet, options.hd_path, args[2])
elif method == 'changemixdepth':
if options.mixdepth < 1:
return "Number of mixdepths must be at least 1"
return change_wallet_mixdepth(wallet, options.mixdepth-1)
#Testing (can port to test modules, TODO)

10
jmclient/jmclient/yieldgenerator.py

@ -129,7 +129,7 @@ class YieldGeneratorBasic(YieldGenerator):
# mixdepth is the chosen depth we'll be spending from
cj_addr = self.wallet.get_internal_addr(
(mixdepth + 1) % (self.wallet.max_mixdepth + 1))
(mixdepth + 1) % (self.wallet.mixdepth + 1))
change_addr = self.wallet.get_internal_addr(mixdepth)
self.import_new_addresses([cj_addr, change_addr])
@ -207,7 +207,10 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffe
dest='fastsync',
default=False,
help=('choose to do fast wallet sync, only for Core and '
'only for previously synced wallet'))
'only for previously synced wallet'))
parser.add_option('-m', '--mixdepth', action='store', type='int',
dest='mixdepth', default=None,
help="highest mixdepth to use")
(options, args) = parser.parse_args()
if len(args) < 1:
parser.error('Needs a wallet')
@ -235,7 +238,8 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffe
wallet_path = get_wallet_path(wallet_name, 'wallets')
wallet = open_test_wallet_maybe(
wallet_path, wallet_name, 4, gap_limit=options.gaplimit)
wallet_path, wallet_name, options.mixdepth,
gap_limit=options.gaplimit)
if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == "electrum-server":
jm_single().bc_interface.synctype = "with-script"

55
jmclient/test/test_wallet.py

@ -11,6 +11,7 @@ from commontest import binarize_tx
from jmclient import load_program_config, jm_single, get_log,\
SegwitLegacyWallet,BIP32Wallet, BIP49Wallet, LegacyWallet,\
VolatileStorage, get_network, cryptoengine, WalletError
from test_blockchaininterface import sync_test_wallet
testdir = os.path.dirname(os.path.realpath(__file__))
log = get_log()
@ -584,6 +585,60 @@ def test_imported_key_removed(setup_wallet):
wallet.get_script_path(path)
def test_wallet_mixdepth_simple(setup_wallet):
wallet = get_populated_wallet(num=0)
mixdepth = wallet.mixdepth
assert wallet.max_mixdepth == mixdepth
wallet.close()
storage_data = wallet._storage.file_data
new_wallet = type(wallet)(VolatileStorage(data=storage_data))
assert new_wallet.mixdepth == mixdepth
assert new_wallet.max_mixdepth == mixdepth
def test_wallet_mixdepth_increase(setup_wallet):
wallet = get_populated_wallet(num=0)
mixdepth = wallet.mixdepth
wallet.close()
storage_data = wallet._storage.file_data
new_mixdepth = mixdepth + 2
new_wallet = type(wallet)(
VolatileStorage(data=storage_data), mixdepth=new_mixdepth)
assert new_wallet.mixdepth == new_mixdepth
assert new_wallet.max_mixdepth == new_mixdepth
def test_wallet_mixdepth_decrease(setup_wallet):
wallet = get_populated_wallet(num=1)
# setup
max_mixdepth = wallet.max_mixdepth
assert max_mixdepth >= 1, "bad default value for mixdepth for this test"
utxo = fund_wallet_addr(wallet, wallet.get_internal_addr(max_mixdepth), 1)
assert wallet.get_balance_by_mixdepth()[max_mixdepth] == 10**8
wallet.close()
storage_data = wallet._storage.file_data
# actual test
new_mixdepth = max_mixdepth - 1
new_wallet = type(wallet)(
VolatileStorage(data=storage_data), mixdepth=new_mixdepth)
assert new_wallet.max_mixdepth == max_mixdepth
assert new_wallet.mixdepth == new_mixdepth
sync_test_wallet(True, new_wallet)
assert max_mixdepth not in new_wallet.get_balance_by_mixdepth()
assert max_mixdepth not in new_wallet.get_utxos_by_mixdepth()
# wallet.select_utxos will still return utxos from higher mixdepths
# because we explicitly ask for a specific mixdepth
assert utxo in new_wallet.select_utxos_(max_mixdepth, 10**7)
@pytest.fixture(scope='module')
def setup_wallet():
load_program_config()

8
scripts/convert_old_wallet.py

@ -17,11 +17,6 @@ class ConvertException(Exception):
pass
def get_max_mixdepth(data):
return max(1, len(data.get('index_cache', [1])) - 1,
*data.get('imported', {}).keys())
def is_encrypted(wallet_data):
return 'encrypted_seed' in wallet_data or 'encrypted_entropy' in wallet_data
@ -73,8 +68,7 @@ def new_wallet_from_data(data, file_name):
kwdata = {
'entropy': data['entropy'],
'timestamp': data.get('creation_time'),
'max_mixdepth': get_max_mixdepth(data)
'timestamp': data.get('creation_time')
}
if 'entropy_ext' in data:

2
scripts/joinmarket-qt.py

@ -1369,7 +1369,7 @@ class JMMainWindow(QMainWindow):
wallet_path = get_wallet_path(str(firstarg), None)
try:
self.wallet = open_test_wallet_maybe(wallet_path, str(firstarg),
4, ask_for_password=False, password=pwd,
None, ask_for_password=False, password=pwd,
gap_limit=jm_single().config.getint("GUI", "gaplimit"))
except Exception as e:
JMQtMessageBox(self,

4
scripts/sendpayment.py

@ -115,9 +115,7 @@ def main():
log.debug('starting sendpayment')
if not options.userpcwallet:
#maxmixdepth in the wallet is actually the *number* of mixdepths (so misnamed);
#to ensure we have enough, must be at least (requested index+1)
max_mix_depth = max([mixdepth+1, options.amtmixdepths])
max_mix_depth = max([mixdepth, options.amtmixdepths - 1])
wallet_path = get_wallet_path(wallet_name, None)
wallet = open_test_wallet_maybe(

7
scripts/tumbler.py

@ -32,13 +32,6 @@ def main():
max_mix_depth = options['mixdepthsrc'] + options['mixdepthcount']
wallet_path = get_wallet_path(wallet_name, None)
wallet = open_test_wallet_maybe(wallet_path, wallet_name, max_mix_depth)
if wallet.max_mixdepth < max_mix_depth:
print("Your wallet does not contain the required number of mixdepths: ",
max_mix_depth)
print("Increase using this command: `python wallet-tool.py -M ",
max_mix_depth + 1, " (yourwalletname) changemixdepth")
print("Then start the tumbler again with the same settings.")
sys.exit(0)
if jm_single().config.get("BLOCKCHAIN",
"blockchain_source") == "electrum-server":
jm_single().bc_interface.synctype = "with-script"

2
test/common.py

@ -93,7 +93,7 @@ def make_wallets(n,
else:
pwd = None
w = open_test_wallet_maybe(seeds[i], seeds[i], mixdepths,
w = open_test_wallet_maybe(seeds[i], seeds[i], mixdepths - 1,
test_wallet_cls=walletclass)
wallets[i + start_index] = {'seed': seeds[i],

Loading…
Cancel
Save