change wallet mixdepth behaviour

jmclient/jmclient/taker.py

@@ -177,7 +177,7 @@ class Taker(object):
             #if destination is flagged "INTERNAL", choose a destination
             #from the next mixdepth modulo the maxmixdepth
             if self.my_cj_addr == "INTERNAL":
-                next_mixdepth = (self.mixdepth + 1) % (self.wallet.max_mixdepth + 1)
+                next_mixdepth = (self.mixdepth + 1) % (self.wallet.mixdepth + 1)
                 jlog.info("Choosing a destination from mixdepth: " + str(next_mixdepth))
                 self.my_cj_addr = self.wallet.get_internal_addr(next_mixdepth)
                 jlog.info("Chose destination address: " + self.my_cj_addr)

jmclient/jmclient/wallet.py

@@ -169,9 +169,11 @@ class UTXOManager(object):
                             'value': utxos[s['utxo']][1]}
                 for s in selected}
 
-    def get_balance_by_mixdepth(self):
+    def get_balance_by_mixdepth(self, max_mixdepth=float('Inf')):
         balance_dict = collections.defaultdict(int)
         for mixdepth, utxomap in self._utxo.items():
+            if mixdepth > max_mixdepth:
+                continue
             value = sum(x[1] for x in utxomap.values())
             balance_dict[mixdepth] = value
         return balance_dict
@@ -201,7 +203,8 @@ class BaseWallet(object):
 
     _ENGINE = None
 
-    def __init__(self, storage, gap_limit=6, merge_algorithm_name=None):
+    def __init__(self, storage, gap_limit=6, merge_algorithm_name=None,
+                 mixdepth=None):
         # to be defined by inheriting classes
         assert self.TYPE is not None
         assert self._ENGINE is not None
@@ -210,7 +213,10 @@ class BaseWallet(object):
         self.gap_limit = gap_limit
         self._storage = storage
         self._utxos = None
+        # highest mixdepth ever used in wallet, important for synching
         self.max_mixdepth = None
+        # effective maximum mixdepth to be used by joinmarket
+        self.mixdepth = None
         self.network = None
 
         # {script: path}, should always hold mappings for all "known" keys
@@ -223,10 +229,22 @@ class BaseWallet(object):
         assert self.max_mixdepth >= 0
         assert self.network in ('mainnet', 'testnet')
 
+        if mixdepth is not None:
+            assert mixdepth >= 0
+            if self._storage.read_only and mixdepth > self.max_mixdepth:
+                raise Exception("Effective max mixdepth must be at most {}!"
+                                .format(self.max_mixdepth))
+            self.max_mixdepth = max(self.max_mixdepth, mixdepth)
+            self.mixdepth = mixdepth
+        else:
+            self.mixdepth = self.max_mixdepth
+
+        assert self.mixdepth is not None
+
     @property
     @deprecated
     def max_mix_depth(self):
-        return self.max_mixdepth
+        return self.mixdepth
 
     @property
     @deprecated
@@ -241,14 +259,12 @@ class BaseWallet(object):
             raise Exception("Wrong class to initialize wallet of type {}."
                             .format(self.TYPE))
         self.network = self._storage.data[b'network'].decode('ascii')
-        self.max_mixdepth = self._storage.data[b'max_mixdepth']
         self._utxos = UTXOManager(self._storage, self.merge_algorithm)
 
     def save(self):
         """
         Write data to associated storage object and trigger persistent update.
         """
-        self._storage.data[b'max_mixdepth'] = self.max_mixdepth
         self._storage.save()
 
     @classmethod
@@ -277,7 +293,6 @@ class BaseWallet(object):
             timestamp = datetime.now().strftime('%Y/%m/%d %H:%M:%S')
 
         storage.data[b'network'] = network.encode('ascii')
-        storage.data[b'max_mixdepth'] = max_mixdepth
         storage.data[b'created'] = timestamp.encode('ascii')
         storage.data[b'wallet_type'] = cls.TYPE
 
@@ -517,8 +532,12 @@ class BaseWallet(object):
 
     def select_utxos_(self, mixdepth, amount, utxo_filter=None):
         """
+        Select a subset of available UTXOS for a given mixdepth whose value is
+        greater or equal to amount.
+
         args:
-            mixdepth: int, mixdepth to select utxos from
+            mixdepth: int, mixdepth to select utxos from, must be smaller or
+                equal to wallet.max_mixdepth
             amount: int, total minimum amount of all selected utxos
             utxo_filter: list of (txid, index), utxos not to select
 
@@ -545,8 +564,13 @@ class BaseWallet(object):
         self._utxos.reset()
 
     def get_balance_by_mixdepth(self, verbose=True):
+        """
+        Get available funds in each active mixdepth.
+
+        returns: {mixdepth: value}
+        """
         # TODO: verbose
-        return self._utxos.get_balance_by_mixdepth()
+        return self._utxos.get_balance_by_mixdepth(max_mixdepth=self.mixdepth)
 
     @deprecated
     def get_utxos_by_mixdepth(self, verbose=True):
@@ -564,6 +588,8 @@ class BaseWallet(object):
 
     def get_utxos_by_mixdepth_(self):
         """
+        Get all UTXOs for active mixdepths.
+
         returns:
             {mixdepth: {(txid, index):
                 {'script': bytes, 'path': tuple, 'value': int}}}
@@ -572,6 +598,8 @@ class BaseWallet(object):
 
         script_utxos = collections.defaultdict(dict)
         for md, data in mix_utxos.items():
+            if md > self.mixdepth:
+                continue
             for utxo, (path, value) in data.items():
                 script = self.get_script_path(path)
                 script_utxos[md][utxo] = {'script': script,
@@ -771,12 +799,11 @@ class ImportWalletMixin(object):
     _IMPORTED_STORAGE_KEY = b'imported_keys'
     _IMPORTED_ROOT_PATH = b'imported'
 
-    def __init__(self, storage, gap_limit=6, merge_algorithm_name=None):
+    def __init__(self, storage, **kwargs):
         # {mixdepth: [(privkey, type)]}
         self._imported = None
         # path is (_IMPORTED_ROOT_PATH, mixdepth, key_index)
-        super(ImportWalletMixin, self).__init__(storage, gap_limit,
+        super(ImportWalletMixin, self).__init__(storage, **kwargs)
-                                                merge_algorithm_name)
 
     def _load_storage(self):
         super(ImportWalletMixin, self)._load_storage()
@@ -1020,15 +1047,14 @@ class BIP32Wallet(BaseWallet):
     BIP32_INT_ID = 1
     ENTROPY_BYTES = 16
 
-    def __init__(self, storage, gap_limit=6, merge_algorithm_name=None):
+    def __init__(self, storage, **kwargs):
         self._entropy = None
         # {mixdepth: {type: index}} with type being 0/1 for [non]-internal
         self._index_cache = None
         # path is a tuple of BIP32 levels,
         # m is the master key's fingerprint
         # other levels are ints
-        super(BIP32Wallet, self).__init__(storage, gap_limit,
+        super(BIP32Wallet, self).__init__(storage, **kwargs)
-                                          merge_algorithm_name)
         assert self._index_cache is not None
         assert self._verify_entropy(self._entropy)
 
@@ -1060,7 +1086,8 @@ class BIP32Wallet(BaseWallet):
             entropy = get_random_bytes(cls.ENTROPY_BYTES, True)
 
         storage.data[cls._STORAGE_ENTROPY_KEY] = entropy
-        storage.data[cls._STORAGE_INDEX_CACHE] = {}
+        storage.data[cls._STORAGE_INDEX_CACHE] = {
+            _int_to_bytestr(i): {} for i in range(max_mixdepth + 1)}
 
         if write:
             storage.save()
@@ -1074,12 +1101,12 @@ class BIP32Wallet(BaseWallet):
 
         for md, data in self._storage.data[self._STORAGE_INDEX_CACHE].items():
             md = int(md)
-            if md > self.max_mixdepth:
-                continue
             md_map = self._index_cache[md]
             for t, k in data.items():
                 md_map[int(t)] = k
 
+        self.max_mixdepth = max(0, 0, *self._index_cache.keys())
+
     def _populate_script_map(self):
         for md in self._index_cache:
             for int_type in (self.BIP32_EXT_ID, self.BIP32_INT_ID):

jmclient/jmclient/wallet_utils.py

@@ -16,6 +16,10 @@ from cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH
 import jmclient.btc as btc
 
 
+# used for creating new wallets
+DEFAULT_MIXDEPTH = 4
+
+
 def get_wallettool_parser():
     description = (
         'Use this script to monitor and manage your Joinmarket wallet.\n'
@@ -40,12 +44,13 @@ def get_wallettool_parser():
                       dest='showprivkey',
                       help='print private key along with address, default false')
     parser.add_option('-m',
-                      '--maxmixdepth',
+                      '--mixdepth',
                       action='store',
                       type='int',
-                      dest='mixdepths',
+                      dest='mixdepth',
-                      help='how many mixing depths to initialize in the wallet',
+                      help="Mixdepth(s) to use in the wallet. Default: {}"
-                      default=5)
+                           .format(DEFAULT_MIXDEPTH),
+                      default=None)
     parser.add_option('-g',
                       '--gap-limit',
                       type="int",
@@ -53,13 +58,6 @@ def get_wallettool_parser():
                       dest='gaplimit',
                       help='gap limit for wallet, default=6',
                       default=6)
-    parser.add_option('-M',
-                      '--mix-depth',
-                      type="int",
-                      action='store',
-                      dest='mixdepth',
-                      help='mixing depth to import private key into',
-                      default=0)
     parser.add_option('--csv',
                       action='store_true',
                       dest='csv',
@@ -344,7 +342,7 @@ def wallet_display(wallet, gaplimit, showprivkey, displayall=False,
     else return the WalletView object.
     """
     acctlist = []
-    for m in xrange(wallet.max_mixdepth + 1):
+    for m in xrange(wallet.mixdepth + 1):
         branchlist = []
         for forchange in [0, 1]:
             entrylist = []
@@ -426,7 +424,7 @@ def cli_get_mnemonic_extension():
 
 
 def wallet_generate_recover_bip39(method, walletspath, default_wallet_name,
-                                  mixdepths=5,
+                                  mixdepth=DEFAULT_MIXDEPTH,
                                   callbacks=(cli_display_user_words,
                                              cli_user_mnemonic_entry,
                                              cli_get_wallet_passphrase_check,
@@ -466,7 +464,7 @@ def wallet_generate_recover_bip39(method, walletspath, default_wallet_name,
         wallet_name = default_wallet_name
     wallet_path = os.path.join(walletspath, wallet_name)
 
-    wallet = create_wallet(wallet_path, password, mixdepths - 1,
+    wallet = create_wallet(wallet_path, password, mixdepth,
                            entropy=entropy,
                            entropy_extension=mnemonic_extension)
     mnemonic, mnext = wallet.get_mnemonic_words()
@@ -477,11 +475,11 @@ def wallet_generate_recover_bip39(method, walletspath, default_wallet_name,
 
 def wallet_generate_recover(method, walletspath,
                             default_wallet_name='wallet.jmdat',
-                            mixdepths=5):
+                            mixdepth=DEFAULT_MIXDEPTH):
     if is_segwit_mode():
         #Here using default callbacks for scripts (not used in Qt)
         return wallet_generate_recover_bip39(
-            method, walletspath, default_wallet_name, mixdepths=mixdepths)
+            method, walletspath, default_wallet_name, mixdepth=mixdepth)
 
     entropy = None
     if method == 'recover':
@@ -504,7 +502,7 @@ def wallet_generate_recover(method, walletspath,
         wallet_name = default_wallet_name
     wallet_path = os.path.join(walletspath, wallet_name)
 
-    wallet = create_wallet(wallet_path, password, mixdepths - 1,
+    wallet = create_wallet(wallet_path, password, mixdepth,
                            wallet_cls=LegacyWallet, entropy=entropy)
     print("Write down and safely store this wallet recovery seed\n\n{}\n"
           .format(wallet.get_mnemonic_words()[0]))
@@ -870,7 +868,7 @@ def open_test_wallet_maybe(path, seed, max_mixdepth,
     params:
         path: path to wallet file, ignored for test wallets
         seed: hex-encoded test seed
-        max_mixdepth: see create_wallet(), ignored when calling open_wallet()
+        max_mixdepth: maximum mixdepth to use
         kwargs: see open_wallet()
 
     returns:
@@ -882,6 +880,9 @@ def open_test_wallet_maybe(path, seed, max_mixdepth,
         except binascii.Error:
             pass
         else:
+            if max_mixdepth is None:
+                max_mixdepth = DEFAULT_MIXDEPTH
+
             storage = VolatileStorage()
             test_wallet_cls.initialize(
                 storage, get_network(), max_mixdepth=max_mixdepth,
@@ -893,11 +894,11 @@ def open_test_wallet_maybe(path, seed, max_mixdepth,
                 del kwargs['ask_for_password']
             if 'password' in kwargs:
                 del kwargs['password']
-            assert 'ask_for_password' not in kwargs
+            if 'read_only' in kwargs:
-            assert 'read_only' not in kwargs
+                del kwargs['read_only']
             return test_wallet_cls(storage, **kwargs)
 
-    return open_wallet(path, **kwargs)
+    return open_wallet(path, mixdepth=max_mixdepth, **kwargs)
 
 
 def open_wallet(path, ask_for_password=True, password=None, read_only=False,
@@ -986,12 +987,14 @@ def wallet_tool_main(wallet_root_path):
         parser.error('Needs a wallet file or method')
         sys.exit(0)
 
-    if options.mixdepths < 1:
+    if options.mixdepth is not None and options.mixdepth < 0:
         parser.error("Must have at least one mixdepth.")
         sys.exit(0)
 
     if args[0] in noseed_methods:
         method = args[0]
+        if options.mixdepth is None:
+            options.mixdepth = DEFAULT_MIXDEPTH
     else:
         seed = args[0]
         wallet_path = get_wallet_path(seed, wallet_root_path)
@@ -999,7 +1002,7 @@ def wallet_tool_main(wallet_root_path):
         read_only = method in readonly_methods
 
         wallet = open_test_wallet_maybe(
-            wallet_path, seed, options.mixdepths - 1, read_only=read_only,
+            wallet_path, seed, options.mixdepth, read_only=read_only,
             gap_limit=options.gaplimit)
 
         if method not in noscan_methods:
@@ -1026,11 +1029,11 @@ def wallet_tool_main(wallet_root_path):
             return wallet_fetch_history(wallet, options)
     elif method == "generate":
         retval = wallet_generate_recover("generate", wallet_root_path,
-                                         mixdepths=options.mixdepths)
+                                         mixdepth=options.mixdepth)
         return retval if retval else "Failed"
     elif method == "recover":
         retval = wallet_generate_recover("recover", wallet_root_path,
-                                         mixdepths=options.mixdepths)
+                                         mixdepth=options.mixdepth)
         return retval if retval else "Failed"
     elif method == "showutxos":
         return wallet_showutxos(wallet, options.showprivkey)
@@ -1040,6 +1043,8 @@ def wallet_tool_main(wallet_root_path):
         return wallet_dumpprivkey(wallet, options.hd_path)
     elif method == "importprivkey":
         #note: must be interactive (security)
+        if options.mixdepth is None:
+            parser.error("You need to specify a mixdepth with -m")
         wallet_importprivkey(wallet, options.mixdepth,
                              map_key_type(options.key_type))
         return "Key import completed."

jmclient/jmclient/yieldgenerator.py

@@ -129,7 +129,7 @@ class YieldGeneratorBasic(YieldGenerator):
 
         # mixdepth is the chosen depth we'll be spending from
         cj_addr = self.wallet.get_internal_addr(
-            (mixdepth + 1) % (self.wallet.max_mixdepth + 1))
+            (mixdepth + 1) % (self.wallet.mixdepth + 1))
         change_addr = self.wallet.get_internal_addr(mixdepth)
         self.import_new_addresses([cj_addr, change_addr])
 
@@ -207,7 +207,10 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffe
                       dest='fastsync',
                       default=False,
                       help=('choose to do fast wallet sync, only for Core and '
-                      'only for previously synced wallet'))
+                            'only for previously synced wallet'))
+    parser.add_option('-m', '--mixdepth', action='store', type='int',
+                      dest='mixdepth', default=None,
+                      help="highest mixdepth to use")
     (options, args) = parser.parse_args()
     if len(args) < 1:
         parser.error('Needs a wallet')
@@ -235,7 +238,8 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffe
 
     wallet_path = get_wallet_path(wallet_name, 'wallets')
     wallet = open_test_wallet_maybe(
-        wallet_path, wallet_name, 4, gap_limit=options.gaplimit)
+        wallet_path, wallet_name, options.mixdepth,
+        gap_limit=options.gaplimit)
 
     if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == "electrum-server":
         jm_single().bc_interface.synctype = "with-script"

scripts/convert_old_wallet.py

@@ -17,11 +17,6 @@ class ConvertException(Exception):
     pass
 
 
-def get_max_mixdepth(data):
-    return max(1, len(data.get('index_cache', [1])) - 1,
-               *data.get('imported', {}).keys())
-
-
 def is_encrypted(wallet_data):
     return 'encrypted_seed' in wallet_data or 'encrypted_entropy' in wallet_data
 
@@ -73,8 +68,7 @@ def new_wallet_from_data(data, file_name):
 
     kwdata = {
         'entropy': data['entropy'],
-        'timestamp': data.get('creation_time'),
+        'timestamp': data.get('creation_time')
-        'max_mixdepth': get_max_mixdepth(data)
     }
 
     if 'entropy_ext' in data:

scripts/joinmarket-qt.py

@@ -1369,7 +1369,7 @@ class JMMainWindow(QMainWindow):
             wallet_path = get_wallet_path(str(firstarg), None)
             try:
                 self.wallet = open_test_wallet_maybe(wallet_path, str(firstarg),
-                        4, ask_for_password=False, password=pwd,
+                        None, ask_for_password=False, password=pwd,
                         gap_limit=jm_single().config.getint("GUI", "gaplimit"))
             except Exception as e:
                 JMQtMessageBox(self,

scripts/sendpayment.py

@@ -115,9 +115,7 @@ def main():
     log.debug('starting sendpayment')
 
     if not options.userpcwallet:
-        #maxmixdepth in the wallet is actually the *number* of mixdepths (so misnamed);
+        max_mix_depth = max([mixdepth, options.amtmixdepths - 1])
-        #to ensure we have enough, must be at least (requested index+1)
-        max_mix_depth = max([mixdepth+1, options.amtmixdepths])
 
         wallet_path = get_wallet_path(wallet_name, None)
         wallet = open_test_wallet_maybe(

test/common.py

@@ -93,7 +93,7 @@ def make_wallets(n,
         else:
             pwd = None
 
-        w = open_test_wallet_maybe(seeds[i], seeds[i], mixdepths,
+        w = open_test_wallet_maybe(seeds[i], seeds[i], mixdepths - 1,
                                    test_wallet_cls=walletclass)
 
         wallets[i + start_index] = {'seed': seeds[i],