From 392188209cceac781f0c5da9a9c779f2e82b455f Mon Sep 17 00:00:00 2001 From: undeath Date: Sun, 7 Oct 2018 23:27:38 +0200 Subject: [PATCH] change wallet mixdepth behaviour --- jmclient/jmclient/taker.py | 2 +- jmclient/jmclient/wallet.py | 61 +++++++++++++++++++++-------- jmclient/jmclient/wallet_utils.py | 55 ++++++++++++++------------ jmclient/jmclient/yieldgenerator.py | 10 +++-- scripts/convert_old_wallet.py | 8 +--- scripts/joinmarket-qt.py | 2 +- scripts/sendpayment.py | 4 +- test/common.py | 2 +- 8 files changed, 86 insertions(+), 58 deletions(-) diff --git a/jmclient/jmclient/taker.py b/jmclient/jmclient/taker.py index 8074982..42d526d 100644 --- a/jmclient/jmclient/taker.py +++ b/jmclient/jmclient/taker.py @@ -177,7 +177,7 @@ class Taker(object): #if destination is flagged "INTERNAL", choose a destination #from the next mixdepth modulo the maxmixdepth if self.my_cj_addr == "INTERNAL": - next_mixdepth = (self.mixdepth + 1) % (self.wallet.max_mixdepth + 1) + next_mixdepth = (self.mixdepth + 1) % (self.wallet.mixdepth + 1) jlog.info("Choosing a destination from mixdepth: " + str(next_mixdepth)) self.my_cj_addr = self.wallet.get_internal_addr(next_mixdepth) jlog.info("Chose destination address: " + self.my_cj_addr) diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index 96e8604..7fee692 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -169,9 +169,11 @@ class UTXOManager(object): 'value': utxos[s['utxo']][1]} for s in selected} - def get_balance_by_mixdepth(self): + def get_balance_by_mixdepth(self, max_mixdepth=float('Inf')): balance_dict = collections.defaultdict(int) for mixdepth, utxomap in self._utxo.items(): + if mixdepth > max_mixdepth: + continue value = sum(x[1] for x in utxomap.values()) balance_dict[mixdepth] = value return balance_dict @@ -201,7 +203,8 @@ class BaseWallet(object): _ENGINE = None - def __init__(self, storage, gap_limit=6, merge_algorithm_name=None): + def __init__(self, storage, gap_limit=6, merge_algorithm_name=None, + mixdepth=None): # to be defined by inheriting classes assert self.TYPE is not None assert self._ENGINE is not None @@ -210,7 +213,10 @@ class BaseWallet(object): self.gap_limit = gap_limit self._storage = storage self._utxos = None + # highest mixdepth ever used in wallet, important for synching self.max_mixdepth = None + # effective maximum mixdepth to be used by joinmarket + self.mixdepth = None self.network = None # {script: path}, should always hold mappings for all "known" keys @@ -223,10 +229,22 @@ class BaseWallet(object): assert self.max_mixdepth >= 0 assert self.network in ('mainnet', 'testnet') + if mixdepth is not None: + assert mixdepth >= 0 + if self._storage.read_only and mixdepth > self.max_mixdepth: + raise Exception("Effective max mixdepth must be at most {}!" + .format(self.max_mixdepth)) + self.max_mixdepth = max(self.max_mixdepth, mixdepth) + self.mixdepth = mixdepth + else: + self.mixdepth = self.max_mixdepth + + assert self.mixdepth is not None + @property @deprecated def max_mix_depth(self): - return self.max_mixdepth + return self.mixdepth @property @deprecated @@ -241,14 +259,12 @@ class BaseWallet(object): raise Exception("Wrong class to initialize wallet of type {}." .format(self.TYPE)) self.network = self._storage.data[b'network'].decode('ascii') - self.max_mixdepth = self._storage.data[b'max_mixdepth'] self._utxos = UTXOManager(self._storage, self.merge_algorithm) def save(self): """ Write data to associated storage object and trigger persistent update. """ - self._storage.data[b'max_mixdepth'] = self.max_mixdepth self._storage.save() @classmethod @@ -277,7 +293,6 @@ class BaseWallet(object): timestamp = datetime.now().strftime('%Y/%m/%d %H:%M:%S') storage.data[b'network'] = network.encode('ascii') - storage.data[b'max_mixdepth'] = max_mixdepth storage.data[b'created'] = timestamp.encode('ascii') storage.data[b'wallet_type'] = cls.TYPE @@ -517,8 +532,12 @@ class BaseWallet(object): def select_utxos_(self, mixdepth, amount, utxo_filter=None): """ + Select a subset of available UTXOS for a given mixdepth whose value is + greater or equal to amount. + args: - mixdepth: int, mixdepth to select utxos from + mixdepth: int, mixdepth to select utxos from, must be smaller or + equal to wallet.max_mixdepth amount: int, total minimum amount of all selected utxos utxo_filter: list of (txid, index), utxos not to select @@ -545,8 +564,13 @@ class BaseWallet(object): self._utxos.reset() def get_balance_by_mixdepth(self, verbose=True): + """ + Get available funds in each active mixdepth. + + returns: {mixdepth: value} + """ # TODO: verbose - return self._utxos.get_balance_by_mixdepth() + return self._utxos.get_balance_by_mixdepth(max_mixdepth=self.mixdepth) @deprecated def get_utxos_by_mixdepth(self, verbose=True): @@ -564,6 +588,8 @@ class BaseWallet(object): def get_utxos_by_mixdepth_(self): """ + Get all UTXOs for active mixdepths. + returns: {mixdepth: {(txid, index): {'script': bytes, 'path': tuple, 'value': int}}} @@ -572,6 +598,8 @@ class BaseWallet(object): script_utxos = collections.defaultdict(dict) for md, data in mix_utxos.items(): + if md > self.mixdepth: + continue for utxo, (path, value) in data.items(): script = self.get_script_path(path) script_utxos[md][utxo] = {'script': script, @@ -771,12 +799,11 @@ class ImportWalletMixin(object): _IMPORTED_STORAGE_KEY = b'imported_keys' _IMPORTED_ROOT_PATH = b'imported' - def __init__(self, storage, gap_limit=6, merge_algorithm_name=None): + def __init__(self, storage, **kwargs): # {mixdepth: [(privkey, type)]} self._imported = None # path is (_IMPORTED_ROOT_PATH, mixdepth, key_index) - super(ImportWalletMixin, self).__init__(storage, gap_limit, - merge_algorithm_name) + super(ImportWalletMixin, self).__init__(storage, **kwargs) def _load_storage(self): super(ImportWalletMixin, self)._load_storage() @@ -1020,15 +1047,14 @@ class BIP32Wallet(BaseWallet): BIP32_INT_ID = 1 ENTROPY_BYTES = 16 - def __init__(self, storage, gap_limit=6, merge_algorithm_name=None): + def __init__(self, storage, **kwargs): self._entropy = None # {mixdepth: {type: index}} with type being 0/1 for [non]-internal self._index_cache = None # path is a tuple of BIP32 levels, # m is the master key's fingerprint # other levels are ints - super(BIP32Wallet, self).__init__(storage, gap_limit, - merge_algorithm_name) + super(BIP32Wallet, self).__init__(storage, **kwargs) assert self._index_cache is not None assert self._verify_entropy(self._entropy) @@ -1060,7 +1086,8 @@ class BIP32Wallet(BaseWallet): entropy = get_random_bytes(cls.ENTROPY_BYTES, True) storage.data[cls._STORAGE_ENTROPY_KEY] = entropy - storage.data[cls._STORAGE_INDEX_CACHE] = {} + storage.data[cls._STORAGE_INDEX_CACHE] = { + _int_to_bytestr(i): {} for i in range(max_mixdepth + 1)} if write: storage.save() @@ -1074,12 +1101,12 @@ class BIP32Wallet(BaseWallet): for md, data in self._storage.data[self._STORAGE_INDEX_CACHE].items(): md = int(md) - if md > self.max_mixdepth: - continue md_map = self._index_cache[md] for t, k in data.items(): md_map[int(t)] = k + self.max_mixdepth = max(0, 0, *self._index_cache.keys()) + def _populate_script_map(self): for md in self._index_cache: for int_type in (self.BIP32_EXT_ID, self.BIP32_INT_ID): diff --git a/jmclient/jmclient/wallet_utils.py b/jmclient/jmclient/wallet_utils.py index 70b6d4a..de3c28f 100644 --- a/jmclient/jmclient/wallet_utils.py +++ b/jmclient/jmclient/wallet_utils.py @@ -16,6 +16,10 @@ from cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH import jmclient.btc as btc +# used for creating new wallets +DEFAULT_MIXDEPTH = 4 + + def get_wallettool_parser(): description = ( 'Use this script to monitor and manage your Joinmarket wallet.\n' @@ -40,12 +44,13 @@ def get_wallettool_parser(): dest='showprivkey', help='print private key along with address, default false') parser.add_option('-m', - '--maxmixdepth', + '--mixdepth', action='store', type='int', - dest='mixdepths', - help='how many mixing depths to initialize in the wallet', - default=5) + dest='mixdepth', + help="Mixdepth(s) to use in the wallet. Default: {}" + .format(DEFAULT_MIXDEPTH), + default=None) parser.add_option('-g', '--gap-limit', type="int", @@ -53,13 +58,6 @@ def get_wallettool_parser(): dest='gaplimit', help='gap limit for wallet, default=6', default=6) - parser.add_option('-M', - '--mix-depth', - type="int", - action='store', - dest='mixdepth', - help='mixing depth to import private key into', - default=0) parser.add_option('--csv', action='store_true', dest='csv', @@ -344,7 +342,7 @@ def wallet_display(wallet, gaplimit, showprivkey, displayall=False, else return the WalletView object. """ acctlist = [] - for m in xrange(wallet.max_mixdepth + 1): + for m in xrange(wallet.mixdepth + 1): branchlist = [] for forchange in [0, 1]: entrylist = [] @@ -426,7 +424,7 @@ def cli_get_mnemonic_extension(): def wallet_generate_recover_bip39(method, walletspath, default_wallet_name, - mixdepths=5, + mixdepth=DEFAULT_MIXDEPTH, callbacks=(cli_display_user_words, cli_user_mnemonic_entry, cli_get_wallet_passphrase_check, @@ -466,7 +464,7 @@ def wallet_generate_recover_bip39(method, walletspath, default_wallet_name, wallet_name = default_wallet_name wallet_path = os.path.join(walletspath, wallet_name) - wallet = create_wallet(wallet_path, password, mixdepths - 1, + wallet = create_wallet(wallet_path, password, mixdepth, entropy=entropy, entropy_extension=mnemonic_extension) mnemonic, mnext = wallet.get_mnemonic_words() @@ -477,11 +475,11 @@ def wallet_generate_recover_bip39(method, walletspath, default_wallet_name, def wallet_generate_recover(method, walletspath, default_wallet_name='wallet.jmdat', - mixdepths=5): + mixdepth=DEFAULT_MIXDEPTH): if is_segwit_mode(): #Here using default callbacks for scripts (not used in Qt) return wallet_generate_recover_bip39( - method, walletspath, default_wallet_name, mixdepths=mixdepths) + method, walletspath, default_wallet_name, mixdepth=mixdepth) entropy = None if method == 'recover': @@ -504,7 +502,7 @@ def wallet_generate_recover(method, walletspath, wallet_name = default_wallet_name wallet_path = os.path.join(walletspath, wallet_name) - wallet = create_wallet(wallet_path, password, mixdepths - 1, + wallet = create_wallet(wallet_path, password, mixdepth, wallet_cls=LegacyWallet, entropy=entropy) print("Write down and safely store this wallet recovery seed\n\n{}\n" .format(wallet.get_mnemonic_words()[0])) @@ -870,7 +868,7 @@ def open_test_wallet_maybe(path, seed, max_mixdepth, params: path: path to wallet file, ignored for test wallets seed: hex-encoded test seed - max_mixdepth: see create_wallet(), ignored when calling open_wallet() + max_mixdepth: maximum mixdepth to use kwargs: see open_wallet() returns: @@ -882,6 +880,9 @@ def open_test_wallet_maybe(path, seed, max_mixdepth, except binascii.Error: pass else: + if max_mixdepth is None: + max_mixdepth = DEFAULT_MIXDEPTH + storage = VolatileStorage() test_wallet_cls.initialize( storage, get_network(), max_mixdepth=max_mixdepth, @@ -893,11 +894,11 @@ def open_test_wallet_maybe(path, seed, max_mixdepth, del kwargs['ask_for_password'] if 'password' in kwargs: del kwargs['password'] - assert 'ask_for_password' not in kwargs - assert 'read_only' not in kwargs + if 'read_only' in kwargs: + del kwargs['read_only'] return test_wallet_cls(storage, **kwargs) - return open_wallet(path, **kwargs) + return open_wallet(path, mixdepth=max_mixdepth, **kwargs) def open_wallet(path, ask_for_password=True, password=None, read_only=False, @@ -986,12 +987,14 @@ def wallet_tool_main(wallet_root_path): parser.error('Needs a wallet file or method') sys.exit(0) - if options.mixdepths < 1: + if options.mixdepth is not None and options.mixdepth < 0: parser.error("Must have at least one mixdepth.") sys.exit(0) if args[0] in noseed_methods: method = args[0] + if options.mixdepth is None: + options.mixdepth = DEFAULT_MIXDEPTH else: seed = args[0] wallet_path = get_wallet_path(seed, wallet_root_path) @@ -999,7 +1002,7 @@ def wallet_tool_main(wallet_root_path): read_only = method in readonly_methods wallet = open_test_wallet_maybe( - wallet_path, seed, options.mixdepths - 1, read_only=read_only, + wallet_path, seed, options.mixdepth, read_only=read_only, gap_limit=options.gaplimit) if method not in noscan_methods: @@ -1026,11 +1029,11 @@ def wallet_tool_main(wallet_root_path): return wallet_fetch_history(wallet, options) elif method == "generate": retval = wallet_generate_recover("generate", wallet_root_path, - mixdepths=options.mixdepths) + mixdepth=options.mixdepth) return retval if retval else "Failed" elif method == "recover": retval = wallet_generate_recover("recover", wallet_root_path, - mixdepths=options.mixdepths) + mixdepth=options.mixdepth) return retval if retval else "Failed" elif method == "showutxos": return wallet_showutxos(wallet, options.showprivkey) @@ -1040,6 +1043,8 @@ def wallet_tool_main(wallet_root_path): return wallet_dumpprivkey(wallet, options.hd_path) elif method == "importprivkey": #note: must be interactive (security) + if options.mixdepth is None: + parser.error("You need to specify a mixdepth with -m") wallet_importprivkey(wallet, options.mixdepth, map_key_type(options.key_type)) return "Key import completed." diff --git a/jmclient/jmclient/yieldgenerator.py b/jmclient/jmclient/yieldgenerator.py index b29ec27..d325223 100644 --- a/jmclient/jmclient/yieldgenerator.py +++ b/jmclient/jmclient/yieldgenerator.py @@ -129,7 +129,7 @@ class YieldGeneratorBasic(YieldGenerator): # mixdepth is the chosen depth we'll be spending from cj_addr = self.wallet.get_internal_addr( - (mixdepth + 1) % (self.wallet.max_mixdepth + 1)) + (mixdepth + 1) % (self.wallet.mixdepth + 1)) change_addr = self.wallet.get_internal_addr(mixdepth) self.import_new_addresses([cj_addr, change_addr]) @@ -207,7 +207,10 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffe dest='fastsync', default=False, help=('choose to do fast wallet sync, only for Core and ' - 'only for previously synced wallet')) + 'only for previously synced wallet')) + parser.add_option('-m', '--mixdepth', action='store', type='int', + dest='mixdepth', default=None, + help="highest mixdepth to use") (options, args) = parser.parse_args() if len(args) < 1: parser.error('Needs a wallet') @@ -235,7 +238,8 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffe wallet_path = get_wallet_path(wallet_name, 'wallets') wallet = open_test_wallet_maybe( - wallet_path, wallet_name, 4, gap_limit=options.gaplimit) + wallet_path, wallet_name, options.mixdepth, + gap_limit=options.gaplimit) if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == "electrum-server": jm_single().bc_interface.synctype = "with-script" diff --git a/scripts/convert_old_wallet.py b/scripts/convert_old_wallet.py index 23bc5da..7814aad 100644 --- a/scripts/convert_old_wallet.py +++ b/scripts/convert_old_wallet.py @@ -17,11 +17,6 @@ class ConvertException(Exception): pass -def get_max_mixdepth(data): - return max(1, len(data.get('index_cache', [1])) - 1, - *data.get('imported', {}).keys()) - - def is_encrypted(wallet_data): return 'encrypted_seed' in wallet_data or 'encrypted_entropy' in wallet_data @@ -73,8 +68,7 @@ def new_wallet_from_data(data, file_name): kwdata = { 'entropy': data['entropy'], - 'timestamp': data.get('creation_time'), - 'max_mixdepth': get_max_mixdepth(data) + 'timestamp': data.get('creation_time') } if 'entropy_ext' in data: diff --git a/scripts/joinmarket-qt.py b/scripts/joinmarket-qt.py index 57f2da0..f31b4c2 100644 --- a/scripts/joinmarket-qt.py +++ b/scripts/joinmarket-qt.py @@ -1369,7 +1369,7 @@ class JMMainWindow(QMainWindow): wallet_path = get_wallet_path(str(firstarg), None) try: self.wallet = open_test_wallet_maybe(wallet_path, str(firstarg), - 4, ask_for_password=False, password=pwd, + None, ask_for_password=False, password=pwd, gap_limit=jm_single().config.getint("GUI", "gaplimit")) except Exception as e: JMQtMessageBox(self, diff --git a/scripts/sendpayment.py b/scripts/sendpayment.py index f6c6b8f..b84a0e8 100644 --- a/scripts/sendpayment.py +++ b/scripts/sendpayment.py @@ -115,9 +115,7 @@ def main(): log.debug('starting sendpayment') if not options.userpcwallet: - #maxmixdepth in the wallet is actually the *number* of mixdepths (so misnamed); - #to ensure we have enough, must be at least (requested index+1) - max_mix_depth = max([mixdepth+1, options.amtmixdepths]) + max_mix_depth = max([mixdepth, options.amtmixdepths - 1]) wallet_path = get_wallet_path(wallet_name, None) wallet = open_test_wallet_maybe( diff --git a/test/common.py b/test/common.py index d503023..95a3978 100644 --- a/test/common.py +++ b/test/common.py @@ -93,7 +93,7 @@ def make_wallets(n, else: pwd = None - w = open_test_wallet_maybe(seeds[i], seeds[i], mixdepths, + w = open_test_wallet_maybe(seeds[i], seeds[i], mixdepths - 1, test_wallet_cls=walletclass) wallets[i + start_index] = {'seed': seeds[i],