Merge #104: [WIP] re-implement wallet

a0c1d5a add upgrade notes (undeath)
8885e61 revert bad assert fix (undeath)
a929cf3 make log output human-readable again (undeath)
aa2c1d9 fix some bugs in wallet_utils (undeath)
9dd1dc7 fix wallet sync in fast mode (undeath)
98f41f7 make SimpleLruCache an actual LRU cache (undeath)
703ae04 remove wallet.sign() (undeath)
34f8600 fix wallet syncing (undeath)
747c227 fix some max_mixdepth off-by-one errors (undeath)
39e4276 change default wallet name (undeath)
8b9abef add is_segwit_mode() utility function (undeath)
8ca6cfc make sure new addresses always get imported (undeath)
914a40e adopt wallet_utils for new wallet (undeath)
cdbb345 remove uses of internal wallet data from electruminterface. NOTE: changes untested, probably breaks electruminterface somehow (undeath)
1f30967 adopt blockchaininterface for new wallet (undeath)
705d41d remove usages of wallet.unspent (undeath)
89b5cd4 add new wallet classes to existing tests (undeath)
3cf9926 remove references to old wallet classes (undeath)
2a0757c remove BitcoinCoreWallet (undeath)
6aaabb2 change yieldgenerator using new wallet implementation, start porting wallet_utils (undeath)
995c123 replace old wallet implementation with new one (undeath)
474a77d add setup.py dependencies (undeath)
ca57a14 add new wallet implementation (undeath)

docs/release-notes/release-notes-future.md

@@ -0,0 +1,29 @@
+Joinmarket-clientserver future:
+===============================
+
+
+Upgrading 
+=========
+
+To upgrade: run the `install.sh` script as mentioned in the README. When prompted to overwrite the directory `jmvenv`, accept.
+
+A new wallet format has been introduced. Old wallets require conversion. In order to convert your existing wallet to the new format you can use the included conversion tool at `scripts/convert_old_wallet.py`.
+
+usage:
+
+    python convert_old_wallet.py full/path/to/wallets/wallet.json
+
+This will place the newly converted `wallet.jmdat` file in the existing joinmarket `wallets/` directory. The wallet name will be adopted accordingly if it differs from `wallet`.
+
+
+Notable changes
+===============
+
+
+Credits
+=======
+
+Thanks to everyone who directly contributed to this release -
+
+
+And thanks also to those who submitted bug reports, tested and otherwise helped out.

jmbitcoin/jmbitcoin/secp256k1_transaction.py

@@ -178,7 +178,8 @@ SIGHASH_NONE = 2
 SIGHASH_SINGLE = 3
 SIGHASH_ANYONECANPAY = 0x80
 
-def segwit_signature_form(txobj, i, script, amount, hashcode=SIGHASH_ALL):
+def segwit_signature_form(txobj, i, script, amount, hashcode=SIGHASH_ALL,
+                          decoder_func=binascii.unhexlify):
     """Given a deserialized transaction txobj, an input index i,
     which spends from a witness,
     a script for redemption and an amount in satoshis, prepare
@@ -187,7 +188,7 @@ def segwit_signature_form(txobj, i, script, amount, hashcode=SIGHASH_ALL):
     #if isinstance(txobj, string_or_bytes_types):
     #    return serialize(segwit_signature_form(deserialize(txobj), i, script,
     #                                           amount, hashcode))
-    script = binascii.unhexlify(script)
+    script = decoder_func(script)
     nVersion = encode(txobj["version"], 256, 4)[::-1]
     #create hashPrevouts
     if hashcode & SIGHASH_ANYONECANPAY:
@@ -195,7 +196,7 @@ def segwit_signature_form(txobj, i, script, amount, hashcode=SIGHASH_ALL):
     else:
         pi = ""
         for inp in txobj["ins"]:
-            pi += binascii.unhexlify(inp["outpoint"]["hash"])[::-1]
+            pi += decoder_func(inp["outpoint"]["hash"])[::-1]
             pi += encode(inp["outpoint"]["index"], 256, 4)[::-1]
         hashPrevouts = bin_dbl_sha256(pi)
     #create hashSequence
@@ -208,7 +209,7 @@ def segwit_signature_form(txobj, i, script, amount, hashcode=SIGHASH_ALL):
     else:
         hashSequence = "\x00"*32
     #add this input's outpoint
-    thisOut = binascii.unhexlify(txobj["ins"][i]["outpoint"]["hash"])[::-1]
+    thisOut = decoder_func(txobj["ins"][i]["outpoint"]["hash"])[::-1]
     thisOut += encode(txobj["ins"][i]["outpoint"]["index"], 256, 4)[::-1]
     scriptCode = num_to_var_int(len(script)) + script
     amt = encode(amount, 256, 8)[::-1]
@@ -218,13 +219,13 @@ def segwit_signature_form(txobj, i, script, amount, hashcode=SIGHASH_ALL):
         pi = ""
         for out in txobj["outs"]:
             pi += encode(out["value"], 256, 8)[::-1]
-            pi += (num_to_var_int(len(binascii.unhexlify(out["script"]))) + \
-                   binascii.unhexlify(out["script"]))
+            pi += (num_to_var_int(len(decoder_func(out["script"]))) + \
+                   decoder_func(out["script"]))
         hashOutputs = bin_dbl_sha256(pi)
     elif hashcode & 0x1f == SIGHASH_SINGLE and i < len(txobj['outs']):
         pi = encode(txobj["outs"][i]["value"], 256, 8)[::-1]
-        pi += (num_to_var_int(len(binascii.unhexlify(txobj["outs"][i]["script"]))) +
-               binascii.unhexlify(txobj["outs"][i]["script"]))
+        pi += (num_to_var_int(len(decoder_func(txobj["outs"][i]["script"]))) +
+               decoder_func(txobj["outs"][i]["script"]))
         hashOutputs = bin_dbl_sha256(pi)
     else:
         hashOutputs = "\x00"*32

jmclient/jmclient/__init__.py

@@ -11,26 +11,33 @@ from btc import *
 from .support import (calc_cj_fee, choose_sweep_orders, choose_orders,
                       cheapest_order_choose, weighted_order_choose,
                       rand_norm_array, rand_pow_array, rand_exp_array, select,
-                      select_gradual, select_greedy, select_greediest)
+                      select_gradual, select_greedy, select_greediest,
+                      get_random_bytes)
 from .jsonrpc import JsonRpcError, JsonRpcConnectionError, JsonRpc
 from .old_mnemonic import mn_decode, mn_encode
 from .slowaes import decryptData, encryptData
 from .taker import Taker
-from .wallet import (AbstractWallet, BitcoinCoreInterface, Wallet,
-                     BitcoinCoreWallet, estimate_tx_fee, WalletError,
-                     create_wallet_file, SegwitWallet, Bip39Wallet, get_wallet_cls)
-from .configure import (load_program_config, get_p2pk_vbyte,
-    jm_single, get_network, validate_address, get_irc_mchannels,
-    get_blockchain_interface_instance, get_p2sh_vbyte, set_config)
+from .wallet import (estimate_tx_fee, WalletError, BaseWallet, ImportWalletMixin,
+                     BIP39WalletMixin, BIP32Wallet, BIP49Wallet, LegacyWallet,
+                     SegwitLegacyWallet, UTXOManager, WALLET_IMPLEMENTATIONS)
+from .storage import (Argon2Hash, Storage, StorageError,
+                      StoragePasswordError, VolatileStorage)
+from .cryptoengine import BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, EngineError
+from .configure import (
+    load_program_config, get_p2pk_vbyte, jm_single, get_network,
+    validate_address, get_irc_mchannels, get_blockchain_interface_instance,
+    get_p2sh_vbyte, set_config, is_segwit_mode)
 from .blockchaininterface import (BlockchainInterface, sync_wallet,
                                   RegtestBitcoinCoreInterface, BitcoinCoreInterface)
 from .electruminterface import ElectrumInterface
 from .client_protocol import (JMTakerClientProtocol, JMClientProtocolFactory,
                               start_reactor)
 from .podle import (set_commitment_file, get_commitment_file,
-                    generate_podle_error_string, add_external_commitments,
+                    add_external_commitments,
                     PoDLE, generate_podle, get_podle_commitments,
                     update_commitments)
+from .output import generate_podle_error_string, fmt_utxos, fmt_utxo,\
+    fmt_tx_data
 from .schedule import (get_schedule, get_tumble_schedule, schedule_to_text,
                        tweak_tumble_schedule, human_readable_schedule_entry,
                        schedule_to_text)
@@ -38,8 +45,10 @@ from .commitment_utils import get_utxo_info, validate_utxo_data, quit
 from .taker_utils import (tumbler_taker_finished_update, restart_waiter,
                              restart_wait, get_tumble_log, direct_send,
                              tumbler_filter_orders_callback)
-from .wallet_utils import (wallet_tool_main, wallet_generate_recover_bip39,
-                           wallet_display)
+from .wallet_utils import (
+    wallet_tool_main, wallet_generate_recover_bip39, open_wallet,
+    open_test_wallet_maybe, create_wallet, get_wallet_cls, get_wallet_path,
+    wallet_display)
 from .maker import Maker
 from .yieldgenerator import YieldGenerator, YieldGeneratorBasic, ygmain
 # Set default logging handler to avoid "No handler found" warnings.

jmclient/jmclient/blockchaininterface.py

@@ -1,17 +1,12 @@
 from __future__ import print_function
 
-import BaseHTTPServer
 import abc
 import ast
-import json
-import os
-import pprint
 import random
-import re
 import sys
 import time
-import traceback
 import binascii
+from copy import deepcopy
 from decimal import Decimal
 from twisted.internet import reactor, task
 
@@ -19,15 +14,10 @@ import btc
 
 from jmclient.jsonrpc import JsonRpcConnectionError, JsonRpcError
 from jmclient.configure import get_p2pk_vbyte, jm_single
-from jmbase.support import get_log, chunks
+from jmbase.support import get_log
 
 log = get_log()
 
-def is_index_ahead_of_cache(wallet, mix_depth, forchange):
-    if mix_depth >= len(wallet.index_cache):
-        return True
-    return wallet.index[mix_depth][forchange] >= wallet.index_cache[mix_depth][
-        forchange]
 
 def sync_wallet(wallet, fast=False):
     """Wrapper function to choose fast syncing where it's
@@ -57,17 +47,15 @@ class BlockchainInterface(object):
 
     @staticmethod
     def get_wallet_name(wallet):
-        return 'joinmarket-wallet-' + btc.dbl_sha256(wallet.keys[0][0])[:6]
+        return 'joinmarket-wallet-' + wallet.get_wallet_id()
 
     @abc.abstractmethod
     def sync_addresses(self, wallet):
-        """Finds which addresses have been used and sets
-        wallet.index appropriately"""
+        """Finds which addresses have been used"""
 
     @abc.abstractmethod
     def sync_unspent(self, wallet):
-        """Finds the unspent transaction outputs belonging to this wallet,
-        sets wallet.unspent """
+        """Finds the unspent transaction outputs belonging to this wallet"""
 
     def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr,
                       wallet_name=None, timeoutfun=None, spentfun=None, txid_flag=True,
@@ -425,47 +413,40 @@ class BitcoinCoreInterface(BlockchainInterface):
         Bitcoin Core instance, in which case "fast" should have been
         specifically disabled by the user.
         """
-        from jmclient.wallet import BitcoinCoreWallet
-        if isinstance(wallet, BitcoinCoreWallet):
-            return
         wallet_name = self.get_wallet_name(wallet)
         agd = self.rpc('listaddressgroupings', [])
         #flatten all groups into a single list; then, remove duplicates
-        fagd = [tuple(item) for sublist in agd for item in sublist]
+        fagd = (tuple(item) for sublist in agd for item in sublist)
         #"deduplicated flattened address grouping data" = dfagd
-        dfagd = list(set(fagd))
-        #for lookup, want dict of form {"address": amount}
-        used_address_dict = {}
+        dfagd = set(fagd)
+        used_addresses = set()
         for addr_info in dfagd:
             if len(addr_info) < 3 or addr_info[2] != wallet_name:
                 continue
-            used_address_dict[addr_info[0]] = (addr_info[1], addr_info[2])
+            used_addresses.add(addr_info[0])
+
         #for a first run, import first chunk
-        if len(used_address_dict.keys()) == 0:
+        if not used_addresses:
             log.info("Detected new wallet, performing initial import")
-            for i in range(wallet.max_mix_depth):
-                for j in [0, 1]:
-                    addrs_to_import = []
-                    for k in range(wallet.gaplimit + 10): # a few more for safety
-                        addrs_to_import.append(wallet.get_addr(i, j, k))
-                    self.import_addresses(addrs_to_import, wallet_name)
-                    wallet.index[i][j] = 0
+            # delegate inital address import to sync_addresses
+            # this should be fast because "getaddressesbyaccount" should return
+            # an empty list in this case
+            self.sync_addresses(wallet)
             self.wallet_synced = True
             return
+
         #Wallet has been used; scan forwards.
         log.debug("Fast sync in progress. Got this many used addresses: " + str(
-            len(used_address_dict)))
+            len(used_addresses)))
         #Need to have wallet.index point to the last used address
-        #and fill addr_cache.
         #Algo:
-        #    1. Scan batch 1 of each branch, accumulate wallet addresses into dict.
-        #    2. Find matches between that dict and used addresses, add those to
-        #        used_indices dict and add to address cache.
-        #    3. Check if all addresses in 'used addresses' have been matched, if
+        #    1. Scan batch 1 of each branch, record matched wallet addresses.
+        #    2. Check if all addresses in 'used addresses' have been matched, if
         #       so, break.
-        #    4. Repeat the above for batch 2, 3.. up to max 20 batches.
-        #    5. If after all 20 batches not all used addresses were matched,
+        #    3. Repeat the above for batch 2, 3.. up to max 20 batches.
+        #    4. If after all 20 batches not all used addresses were matched,
         #       quit with error.
+        #    5. Calculate used indices.
         #    6. If all used addresses were matched, set wallet index to highest
         #       found index in each branch and mark wallet sync complete.
         #Rationale for this algo:
@@ -476,174 +457,152 @@ class BitcoinCoreInterface(BlockchainInterface):
         #    not be exposed to the user; it is not the same as gap limit, in fact,
         #    the concept of gap limit does not apply to this kind of sync, which
         #    *assumes* that the most recent usage of addresses is indeed recorded.
-        used_indices = {}
-        local_addr_cache = {}
-        found_addresses = []
+        remaining_used_addresses = used_addresses.copy()
+        addresses, saved_indices = self._collect_addresses_init(wallet)
+        for addr in addresses:
+            remaining_used_addresses.discard(addr)
+
         BATCH_SIZE = 100
-        for j in range(20):
-            for md in range(wallet.max_mix_depth):
-                if md not in used_indices:
-                    used_indices[md] = {}
-                for fc in [0, 1]:
-                    if fc not in used_indices[md]:
-                        used_indices[md][fc] = []
-                    for i in range(j*BATCH_SIZE, (j+1)*BATCH_SIZE):
-                        local_addr_cache[(md, fc, i)] = wallet.get_addr(md, fc, i)
-            batch_found_addresses = [x for x in local_addr_cache.iteritems(
-                ) if x[1] in used_address_dict.keys()]
-            for x in batch_found_addresses:
-                md, fc, i = x[0]
-                addr = x[1]
-                used_indices[md][fc].append(i)
-                wallet.addr_cache[addr] = (md, fc, i)
-            found_addresses.extend(batch_found_addresses)
-            if len(found_addresses) == len(used_address_dict.keys()):
+        MAX_ITERATIONS = 20
+        current_indices = deepcopy(saved_indices)
+        for j in range(MAX_ITERATIONS):
+            if not remaining_used_addresses:
                 break
-        if j == 19:
+            for addr in \
+                    self._collect_addresses_gap(wallet, gap_limit=BATCH_SIZE):
+                remaining_used_addresses.discard(addr)
+
+            # increase wallet indices for next iteration
+            for md in current_indices:
+                current_indices[md][0] += BATCH_SIZE
+                current_indices[md][1] += BATCH_SIZE
+            self._rewind_wallet_indices(wallet, current_indices,
+                                        current_indices)
+        else:
+            self._rewind_wallet_indices(wallet, saved_indices, saved_indices)
             raise Exception("Failed to sync in fast mode after 20 batches; "
                             "please re-try wallet sync without --fast flag.")
-        #Find the highest index in each branch and set the wallet index
-        for md in range(wallet.max_mix_depth):
-            for fc in [0, 1]:
-                if len(used_indices[md][fc]):
-                    used_indices[md][fc].sort()
-                    wallet.index[md][fc] = used_indices[md][fc][-1] + 1
-                else:
-                    wallet.index[md][fc] = 0
-                if not is_index_ahead_of_cache(wallet, md, fc):
-                    wallet.index[md][fc] = wallet.index_cache[md][fc]
-        self.wallet_synced = True
 
+        # creating used_indices on-the-fly would be more efficient, but the
+        # overall performance gain is probably negligible
+        used_indices = self._get_used_indices(wallet, used_addresses)
+        self._rewind_wallet_indices(wallet, used_indices, saved_indices)
+        self.wallet_synced = True
 
     def sync_addresses(self, wallet, restart_cb=None):
-        from jmclient.wallet import BitcoinCoreWallet
-
-        if isinstance(wallet, BitcoinCoreWallet):
-            return
-        log.debug('requesting detailed wallet history')
+        log.debug("requesting detailed wallet history")
         wallet_name = self.get_wallet_name(wallet)
-        #TODO It is worth considering making this user configurable:
-        addr_req_count = 20
-        wallet_addr_list = []
-        for mix_depth in range(wallet.max_mix_depth):
-            for forchange in [0, 1]:
-                #If we have an index-cache available, we can use it
-                #to decide how much to import (note that this list
-                #*always* starts from index 0 on each branch).
-                #In cases where the Bitcoin Core instance is fresh,
-                #this will allow the entire import+rescan to occur
-                #in 2 steps only.
-                if wallet.index_cache != [[0, 0]] * wallet.max_mix_depth:
-                    #Need to request N*addr_req_count where N is least s.t.
-                    #N*addr_req_count > index_cache val. This is so that the batching
-                    #process in the main loop *always* has already imported enough
-                    #addresses to complete.
-                    req_count = int(wallet.index_cache[mix_depth][forchange] /
-                                    addr_req_count) + 1
-                    req_count *= addr_req_count
-                else:
-                    #If we have *nothing* - no index_cache, and no info
-                    #in Core wallet (imports), we revert to a batching mode
-                    #with a default size.
-                    #In this scenario it could require several restarts *and*
-                    #rescans; perhaps user should set addr_req_count high
-                    #(see above TODO)
-                    req_count = addr_req_count
-                wallet_addr_list += [wallet.get_new_addr(mix_depth, forchange)
-                                     for _ in range(req_count)]
-                #Indices are reset here so that the next algorithm step starts
-                #from the beginning of each branch
-                wallet.index[mix_depth][forchange] = 0
-        # makes more sense to add these in an account called "joinmarket-imported" but its much
-        # simpler to add to the same account here
-        for privkey_list in wallet.imported_privkeys.values():
-            for privkey in privkey_list:
-                imported_addr = btc.privtoaddr(privkey,
-                                               magicbyte=get_p2pk_vbyte())
-                wallet_addr_list.append(imported_addr)
-        imported_addr_list = self.rpc('getaddressesbyaccount', [wallet_name])
-        if not set(wallet_addr_list).issubset(set(imported_addr_list)):
-            self.add_watchonly_addresses(wallet_addr_list, wallet_name, restart_cb)
-            return
 
-        buf = self.rpc('listtransactions', [wallet_name, 1000, 0, True])
-        txs = buf
-        # If the buffer's full, check for more, until it ain't
-        while len(buf) == 1000:
-            buf = self.rpc('listtransactions', [wallet_name, 1000, len(txs),
-                                                True])
-            txs += buf
-        # TODO check whether used_addr_list can be a set, may be faster (if
-        # its a hashset) and allows using issubset() here and setdiff() for
-        # finding which addresses need importing
-
-        # TODO also check the fastest way to build up python lists, i suspect
-        #  using += is slow
-        used_addr_list = [tx['address']
-                          for tx in txs if tx['category'] == 'receive']
-        too_few_addr_mix_change = []
-        for mix_depth in range(wallet.max_mix_depth):
-            for forchange in [0, 1]:
-                unused_addr_count = 0
-                last_used_addr = ''
-                breakloop = False
-                while not breakloop:
-                    if unused_addr_count >= wallet.gaplimit and \
-                            is_index_ahead_of_cache(wallet, mix_depth,
-                                                    forchange):
-                        break
-                    mix_change_addrs = [
-                        wallet.get_new_addr(mix_depth, forchange)
-                        for _ in range(addr_req_count)
-                    ]
-                    for mc_addr in mix_change_addrs:
-                        if mc_addr not in imported_addr_list:
-                            too_few_addr_mix_change.append((mix_depth, forchange
-                                                           ))
-                            breakloop = True
-                            break
-                        if mc_addr in used_addr_list:
-                            last_used_addr = mc_addr
-                            unused_addr_count = 0
-                        else:
-                            unused_addr_count += 1
-#index setting here depends on whether we broke out of the loop
-#early; if we did, it means we need to prepare the index
-#at the level of the last used address or zero so as to not
-#miss any imports in add_watchonly_addresses.
-#If we didn't, we need to respect the index_cache to avoid
-#potential address reuse.
-                if breakloop:
-                    if last_used_addr == '':
-                        wallet.index[mix_depth][forchange] = 0
-                    else:
-                        wallet.index[mix_depth][forchange] = \
-                            wallet.addr_cache[last_used_addr][2] + 1
-                else:
-                    if last_used_addr == '':
-                        next_avail_idx = max([wallet.index_cache[mix_depth][
-                            forchange], 0])
-                    else:
-                        next_avail_idx = max([wallet.addr_cache[last_used_addr][
-                            2] + 1, wallet.index_cache[mix_depth][forchange]])
-                    wallet.index[mix_depth][forchange] = next_avail_idx
-
-        wallet_addr_list = []
-        if len(too_few_addr_mix_change) > 0:
-            indices = [wallet.index[mc[0]][mc[1]]
-                       for mc in too_few_addr_mix_change]
-            log.debug('too few addresses in ' + str(too_few_addr_mix_change) +
-                      ' at ' + str(indices))
-            for mix_depth, forchange in too_few_addr_mix_change:
-                wallet_addr_list += [
-                    wallet.get_new_addr(mix_depth, forchange)
-                    for _ in range(addr_req_count * 3)
-                ]
-
-            self.add_watchonly_addresses(wallet_addr_list, wallet_name, restart_cb)
+        addresses, saved_indices = self._collect_addresses_init(wallet)
+        imported_addresses = set(self.rpc('getaddressesbyaccount', [wallet_name]))
+
+        if not addresses.issubset(imported_addresses):
+            self.add_watchonly_addresses(addresses - imported_addresses,
+                                         wallet_name, restart_cb)
             return
 
-        self.wallet_synced = True
+        used_addresses_gen = (tx['address']
+                              for tx in self._yield_transactions(wallet_name)
+                              if tx['category'] == 'receive')
+
+        used_indices = self._get_used_indices(wallet, used_addresses_gen)
+        log.debug("got used indices: {}".format(used_indices))
+        gap_limit_used = not self._check_gap_indices(wallet, used_indices)
+        self._rewind_wallet_indices(wallet, used_indices, saved_indices)
+
+        new_addresses = self._collect_addresses_gap(wallet)
+        if not new_addresses.issubset(imported_addresses):
+            log.debug("Syncing iteration finished, additional step required")
+            self.add_watchonly_addresses(new_addresses - imported_addresses,
+                                         wallet_name, restart_cb)
+            self.wallet_synced = False
+        elif gap_limit_used:
+            log.debug("Syncing iteration finished, additional step required")
+            self.wallet_synced = False
+        else:
+            log.debug("Wallet successfully synced")
+            self._rewind_wallet_indices(wallet, used_indices, saved_indices)
+            self.wallet_synced = True
+
+    @staticmethod
+    def _rewind_wallet_indices(wallet, used_indices, saved_indices):
+        for md in used_indices:
+            for int_type in (0, 1):
+                index = max(used_indices[md][int_type],
+                            saved_indices[md][int_type])
+                wallet.set_next_index(md, int_type, index, force=True)
+
+    @staticmethod
+    def _get_used_indices(wallet, addr_gen):
+        indices = {x: [0, 0] for x in range(wallet.max_mixdepth + 1)}
+
+        for addr in addr_gen:
+            if not wallet.is_known_addr(addr):
+                continue
+            md, internal, index = wallet.get_details(
+                wallet.addr_to_path(addr))
+            if internal not in (0, 1):
+                assert internal == 'imported'
+                continue
+            indices[md][internal] = max(indices[md][internal], index + 1)
+
+        return indices
+
+    @staticmethod
+    def _check_gap_indices(wallet, used_indices):
+        for md in used_indices:
+            for internal in (0, 1):
+                if used_indices[md][internal] >\
+                        max(wallet.get_next_unused_index(md, internal), 0):
+                    return False
+        return True
+
+    @staticmethod
+    def _collect_addresses_init(wallet):
+        addresses = set()
+        saved_indices = dict()
+
+        for md in range(wallet.max_mixdepth + 1):
+            saved_indices[md] = [0, 0]
+            for internal in (0, 1):
+                next_unused = wallet.get_next_unused_index(md, internal)
+                for index in range(next_unused):
+                    addresses.add(wallet.get_addr(md, internal, index))
+                for index in range(wallet.gap_limit):
+                    addresses.add(wallet.get_new_addr(md, internal))
+                wallet.set_next_index(md, internal, next_unused)
+                saved_indices[md][internal] = next_unused
+            for path in wallet.yield_imported_paths(md):
+                addresses.add(wallet.get_addr_path(path))
+
+        return addresses, saved_indices
+
+    @staticmethod
+    def _collect_addresses_gap(wallet, gap_limit=None):
+        gap_limit = gap_limit or wallet.gap_limit
+        addresses = set()
+
+        for md in range(wallet.max_mixdepth + 1):
+            for internal in (True, False):
+                old_next = wallet.get_next_unused_index(md, internal)
+                for index in range(gap_limit):
+                    addresses.add(wallet.get_new_addr(md, internal))
+                wallet.set_next_index(md, internal, old_next)
+
+        return addresses
+
+    def _yield_transactions(self, wallet_name):
+        batch_size = 1000
+        iteration = 0
+        while True:
+            new = self.rpc(
+                'listtransactions',
+                [wallet_name, batch_size, iteration * batch_size, True])
+            for tx in new:
+                yield tx
+            if len(new) < batch_size:
+                return
+            iteration += 1
 
     def start_unspent_monitoring(self, wallet):
         self.unspent_monitoring_loop = task.LoopingCall(self.sync_unspent, wallet)
@@ -653,13 +612,9 @@ class BitcoinCoreInterface(BlockchainInterface):
         self.unspent_monitoring_loop.stop()
 
     def sync_unspent(self, wallet):
-        from jmclient.wallet import BitcoinCoreWallet
-
-        if isinstance(wallet, BitcoinCoreWallet):
-            return
         st = time.time()
         wallet_name = self.get_wallet_name(wallet)
-        wallet.unspent = {}
+        wallet.reset_utxos()
 
         listunspent_args = []
         if 'listunspent_args' in jm_single().config.options('POLICY'):
@@ -672,16 +627,28 @@ class BitcoinCoreInterface(BlockchainInterface):
                 continue
             if u['account'] != wallet_name:
                 continue
-            if u['address'] not in wallet.addr_cache:
+            if not wallet.is_known_addr(u['address']):
                 continue
-            wallet.unspent[u['txid'] + ':' + str(u['vout'])] = {
-                'address': u['address'],
-                'value': int(Decimal(str(u['amount'])) * Decimal('1e8'))
-            }
+            self._add_unspent_utxo(wallet, u)
         et = time.time()
         log.debug('bitcoind sync_unspent took ' + str((et - st)) + 'sec')
         self.wallet_synced = True
 
+    @staticmethod
+    def _add_unspent_utxo(wallet, utxo):
+        """
+        Add a UTXO as returned by rpc's listunspent call to the wallet.
+
+        params:
+            wallet: wallet
+            utxo: single utxo dict as returned by listunspent
+        """
+        txid = binascii.unhexlify(utxo['txid'])
+        script = binascii.unhexlify(utxo['scriptPubKey'])
+        value = int(Decimal(str(utxo['amount'])) * Decimal('1e8'))
+
+        wallet.add_utxo(txid, int(utxo['vout']), script, value)
+
     def get_deser_from_gettransaction(self, rpcretval):
         """Get full transaction deserialization from a call
         to `gettransaction`

jmclient/jmclient/client_protocol.py

@@ -15,10 +15,9 @@ import json
 import hashlib
 import os
 import sys
-import pprint
-from jmclient import (Taker, Wallet, jm_single, get_irc_mchannels,
-                        load_program_config, get_log, get_p2sh_vbyte,
-                        RegtestBitcoinCoreInterface)
+from jmclient import (jm_single, get_irc_mchannels, get_log, get_p2sh_vbyte,
+                      RegtestBitcoinCoreInterface)
+from .output import fmt_tx_data
 from jmbase import _byteify
 import btc
 
@@ -241,8 +240,9 @@ class JMMakerClientProtocol(JMClientProtocol):
             jlog.info("Failed to find notified unconfirmed transaction: " + txid)
             return
         removed_utxos = self.client.wallet.remove_old_utxos(txd)
-        jlog.info('saw tx on network, removed_utxos=\n{}'.format(
-                pprint.pformat(removed_utxos)))
+        jlog.info('saw tx on network, removed_utxos=\n{}'.format('\n'.join(
+            '{} - {}'.format(u, fmt_tx_data(tx_data, self.client.wallet))
+            for u, tx_data in removed_utxos.items())))
         to_cancel, to_announce = self.client.on_tx_unconfirmed(offerinfo,
                                                                txid, removed_utxos)
         self.client.modify_orders(to_cancel, to_announce)

jmclient/jmclient/configure.py

@@ -414,3 +414,7 @@ def get_blockchain_interface_instance(_config):
     else:
         raise ValueError("Invalid blockchain source")
     return bc_interface
+
+
+def is_segwit_mode():
+    return jm_single().config.get('POLICY', 'segwit') != 'false'

jmclient/jmclient/cryptoengine.py

@@ -0,0 +1,271 @@
+from __future__ import print_function, absolute_import, division, unicode_literals
+
+
+from binascii import hexlify, unhexlify
+from collections import OrderedDict
+
+
+from . import btc
+from .configure import get_network
+
+
+TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH = range(3)
+NET_MAINNET, NET_TESTNET = range(2)
+NET_MAP = {'mainnet': NET_MAINNET, 'testnet': NET_TESTNET}
+WIF_PREFIX_MAP = {'mainnet': 0x80, 'testnet': 0xef}
+BIP44_COIN_MAP = {'mainnet': 2**31, 'testnet': 2**31 + 1}
+
+
+#
+# library stuff that should be in btc/jmbitcoin
+#
+
+
+P2PKH_PRE, P2PKH_POST = b'\x76\xa9\x14', b'\x88\xac'
+P2SH_P2WPKH_PRE, P2SH_P2WPKH_POST = b'\xa9\x14', b'\x87'
+P2WPKH_PRE = b'\x00\x14'
+
+
+def _pubkey_to_script(pubkey, script_pre, script_post=b''):
+    # sanity check for public key
+    # see https://github.com/bitcoin/bitcoin/blob/master/src/pubkey.h
+    if not ((len(pubkey) == 33 and pubkey[0] in (b'\x02', b'\x03')) or
+            (len(pubkey) == 65 and pubkey[0] in (b'\x04', b'\x06', b'\x07'))):
+        raise Exception("Invalid public key!")
+    h = btc.bin_hash160(pubkey)
+    assert len(h) == 0x14
+    assert script_pre[-1] == b'\x14'
+    return script_pre + h + script_post
+
+
+def pubkey_to_p2pkh_script(pubkey):
+    return _pubkey_to_script(pubkey, P2PKH_PRE, P2PKH_POST)
+
+
+def pubkey_to_p2sh_p2wpkh_script(pubkey):
+    wscript = pubkey_to_p2wpkh_script(pubkey)
+    return P2SH_P2WPKH_PRE + btc.bin_hash160(wscript) + P2SH_P2WPKH_POST
+
+
+def pubkey_to_p2wpkh_script(pubkey):
+    return _pubkey_to_script(pubkey, P2WPKH_PRE)
+
+
+class classproperty(object):
+    """
+    from https://stackoverflow.com/a/5192374
+    """
+    def __init__(self, f):
+        self.f = f
+
+    def __get__(self, obj, owner):
+        return self.f(owner)
+
+
+class SimpleLruCache(OrderedDict):
+    """
+    note: python3.2 has a lru cache in functools
+    """
+    def __init__(self, max_size):
+        OrderedDict.__init__(self)
+        assert max_size > 0
+        self.max_size = max_size
+
+    def __setitem__(self, key, value):
+        OrderedDict.__setitem__(self, key, value)
+        self._adjust_size()
+
+    def __getitem__(self, item):
+        e = OrderedDict.__getitem__(self, item)
+        del self[item]
+        OrderedDict.__setitem__(self, item, e)
+        return e
+
+    def _adjust_size(self):
+        while len(self) > self.max_size:
+            self.popitem(last=False)
+
+
+#
+# library stuff end
+#
+
+
+class EngineError(Exception):
+    pass
+
+
+class BTCEngine(object):
+    # must be set by subclasses
+    VBYTE = None
+    __LRU_KEY_CACHE = SimpleLruCache(50)
+
+    @classproperty
+    def BIP32_priv_vbytes(cls):
+        return btc.PRIVATE[NET_MAP[get_network()]]
+
+    @classproperty
+    def WIF_PREFIX(cls):
+        return WIF_PREFIX_MAP[get_network()]
+
+    @classproperty
+    def BIP44_COIN_TYPE(cls):
+        return BIP44_COIN_MAP[get_network()]
+
+    @staticmethod
+    def privkey_to_pubkey(privkey):
+        return btc.privkey_to_pubkey(privkey, False)
+
+    @staticmethod
+    def address_to_script(addr):
+        return unhexlify(btc.address_to_script(addr))
+
+    @classmethod
+    def wif_to_privkey(cls, wif):
+        raw = btc.b58check_to_bin(wif)
+        vbyte = btc.get_version_byte(wif)
+
+        if (btc.BTC_P2PK_VBYTE[get_network()] + cls.WIF_PREFIX) & 0xff == vbyte:
+            key_type = TYPE_P2PKH
+        elif (btc.BTC_P2SH_VBYTE[get_network()] + cls.WIF_PREFIX) & 0xff == vbyte:
+            key_type = TYPE_P2SH_P2WPKH
+        else:
+            key_type = None
+
+        return raw, key_type
+
+    @classmethod
+    def privkey_to_wif(cls, priv):
+        return btc.bin_to_b58check(priv, cls.WIF_PREFIX)
+
+    @classmethod
+    def derive_bip32_master_key(cls, seed):
+        # FIXME: slight encoding mess
+        return btc.bip32_deserialize(
+            btc.bip32_master_key(seed, vbytes=cls.BIP32_priv_vbytes))
+
+    @classmethod
+    def derive_bip32_privkey(cls, master_key, path):
+        assert len(path) > 1
+        return cls._walk_bip32_path(master_key, path)[-1]
+
+    @classmethod
+    def derive_bip32_pub_export(cls, master_key, path):
+        priv = cls._walk_bip32_path(master_key, path)
+        return btc.bip32_serialize(btc.raw_bip32_privtopub(priv))
+
+    @classmethod
+    def derive_bip32_priv_export(cls, master_key, path):
+        return btc.bip32_serialize(cls._walk_bip32_path(master_key, path))
+
+    @classmethod
+    def _walk_bip32_path(cls, master_key, path):
+        key = master_key
+        for lvl in path[1:]:
+            assert 0 <= lvl < 2**32
+            if (key, lvl) in cls.__LRU_KEY_CACHE:
+                key = cls.__LRU_KEY_CACHE[(key, lvl)]
+            else:
+                cls.__LRU_KEY_CACHE[(key, lvl)] = btc.raw_bip32_ckd(key, lvl)
+                key = cls.__LRU_KEY_CACHE[(key, lvl)]
+        return key
+
+    @classmethod
+    def privkey_to_script(cls, privkey):
+        pub = cls.privkey_to_pubkey(privkey)
+        return cls.pubkey_to_script(pub)
+
+    @classmethod
+    def pubkey_to_script(cls, pubkey):
+        raise NotImplementedError()
+
+    @classmethod
+    def privkey_to_address(cls, privkey):
+        script = cls.privkey_to_script(privkey)
+        return btc.script_to_address(script, cls.VBYTE)
+
+    @classmethod
+    def pubkey_to_address(cls, pubkey):
+        script = cls.pubkey_to_script(pubkey)
+        return btc.script_to_address(script, cls.VBYTE)
+
+    @classmethod
+    def sign_transaction(cls, tx, index, privkey, amount):
+        raise NotImplementedError()
+
+    @staticmethod
+    def sign_message(privkey, message):
+        """
+        args:
+            privkey: bytes
+            message: bytes
+        returns:
+            base64-encoded signature
+        """
+        return btc.ecdsa_sign(message, privkey, True, False)
+
+    @classmethod
+    def script_to_address(cls, script):
+        return btc.script_to_address(script, vbyte=cls.VBYTE)
+
+
+class BTC_P2PKH(BTCEngine):
+    @classproperty
+    def VBYTE(cls):
+        return btc.BTC_P2PK_VBYTE[get_network()]
+
+    @classmethod
+    def pubkey_to_script(cls, pubkey):
+        return pubkey_to_p2pkh_script(pubkey)
+
+    @classmethod
+    def sign_transaction(cls, tx, index, privkey, *args, **kwargs):
+        hashcode = kwargs.get('hashcode') or btc.SIGHASH_ALL
+
+        pubkey = cls.privkey_to_pubkey(privkey)
+        script = cls.pubkey_to_script(pubkey)
+
+        signing_tx = btc.serialize(btc.signature_form(tx, index, script,
+                                                      hashcode=hashcode))
+        # FIXME: encoding mess
+        sig = unhexlify(btc.ecdsa_tx_sign(signing_tx, hexlify(privkey),
+                                          **kwargs))
+
+        tx['ins'][index]['script'] = btc.serialize_script([sig, pubkey])
+
+        return tx
+
+
+class BTC_P2SH_P2WPKH(BTCEngine):
+    # FIXME: implement different bip32 key export prefixes like electrum?
+    # see http://docs.electrum.org/en/latest/seedphrase.html#list-of-reserved-numbers
+
+    @classproperty
+    def VBYTE(cls):
+        return btc.BTC_P2SH_VBYTE[get_network()]
+
+    @classmethod
+    def pubkey_to_script(cls, pubkey):
+        return pubkey_to_p2sh_p2wpkh_script(pubkey)
+
+    @classmethod
+    def sign_transaction(cls, tx, index, privkey, amount,
+                         hashcode=btc.SIGHASH_ALL, **kwargs):
+        assert amount is not None
+
+        pubkey = cls.privkey_to_pubkey(privkey)
+        wpkscript = pubkey_to_p2wpkh_script(pubkey)
+        pkscript = pubkey_to_p2pkh_script(pubkey)
+
+        signing_tx = btc.segwit_signature_form(tx, index, pkscript, amount,
+                                               hashcode=hashcode,
+                                               decoder_func=lambda x: x)
+        # FIXME: encoding mess
+        sig = unhexlify(btc.ecdsa_tx_sign(signing_tx, hexlify(privkey),
+                                          hashcode=hashcode, **kwargs))
+
+        assert len(wpkscript) == 0x16
+        tx['ins'][index]['script'] = b'\x16' + wpkscript
+        tx['ins'][index]['txinwitness'] = [sig, pubkey]
+
+        return tx

jmclient/jmclient/electruminterface.py

@@ -6,15 +6,13 @@ import pprint
 import random
 import socket
 import threading
-import time
-import sys
 import ssl
-from twisted.python.log import startLogging
-from twisted.internet.protocol import ClientFactory, Protocol
+import binascii
+from twisted.internet.protocol import ClientFactory
 from twisted.internet.ssl import ClientContextFactory
 from twisted.protocols.basic import LineReceiver
 from twisted.internet import reactor, task, defer
-from .blockchaininterface import BlockchainInterface, is_index_ahead_of_cache
+from .blockchaininterface import BlockchainInterface
 from .configure import get_p2sh_vbyte
 from .support import get_log
 from .electrum_data import (get_default_ports, get_default_servers,
@@ -250,10 +248,10 @@ class ElectrumInterface(BlockchainInterface):
             reactor.callLater(0.2, self.sync_addresses, wallet, restart_cb)
             return
         log.debug("downloading wallet history from Electrum server ...")
-        for mixdepth in range(wallet.max_mix_depth):
+        for mixdepth in range(wallet.max_mixdepth + 1):
             for forchange in [0, 1]:
                 #start from a clean index
-                wallet.index[mixdepth][forchange] = 0
+                wallet.set_next_index(mixdepth, forchange, 0)
                 self.synchronize_batch(wallet, mixdepth, forchange, 0)
 
     def synchronize_batch(self, wallet, mixdepth, forchange, start_index):
@@ -297,9 +295,8 @@ class ElectrumInterface(BlockchainInterface):
             #existing index_cache from the wallet file; if both true, end, else, continue
             #to next batch
             if all([tah[i]['used'] is False for i in range(
-                start_index+self.BATCH_SIZE-wallet.gaplimit,
-                start_index+self.BATCH_SIZE)]) and is_index_ahead_of_cache(
-                    wallet, mixdepth, forchange):
+                start_index + self.BATCH_SIZE - wallet.gap_limit,
+                start_index + self.BATCH_SIZE)]):
                 last_used_addr = None
                 #to find last used, note that it may be in the *previous* batch;
                 #may as well just search from the start, since it takes no time.
@@ -307,12 +304,11 @@ class ElectrumInterface(BlockchainInterface):
                     if tah[i]['used']:
                         last_used_addr = tah[i]['addr']
                 if last_used_addr:
-                    wallet.index[mixdepth][forchange] = wallet.addr_cache[last_used_addr][2] + 1
+                    wallet.set_next_index(
+                        mixdepth, forchange,
+                        wallet.get_next_unused_index(mixdepth, forchange))
                 else:
-                    wallet.index[mixdepth][forchange] = 0
-                #account for index_cache
-                if not is_index_ahead_of_cache(wallet, mixdepth, forchange):
-                    wallet.index[mixdepth][forchange] = wallet.index_cache[mixdepth][forchange]
+                    wallet.set_next_index(mixdepth, forchange, 0)
                 tah["finished"] = True
                 #check if all branches are finished to trigger next stage of sync.
                 addr_sync_complete = True
@@ -328,10 +324,10 @@ class ElectrumInterface(BlockchainInterface):
 
     def sync_unspent(self, wallet):
         # finds utxos in the wallet
-        wallet.unspent = {}
+        wallet.reset_utxos()
         #Prepare list of all used addresses
-        addrs = []
-        for m in range(wallet.max_mix_depth):
+        addrs = set()
+        for m in range(wallet.max_mixdepth):
             for fc in [0, 1]:
                 branch_list = []
                 for k, v in self.temp_addr_history[m][fc].iteritems():
@@ -339,7 +335,7 @@ class ElectrumInterface(BlockchainInterface):
                         continue
                     if v["used"]:
                         branch_list.append(v["addr"])
-                addrs.extend(branch_list)
+                addrs.update(branch_list)
         if len(addrs) == 0:
             log.debug('no tx used')
             self.wallet_synced = True
@@ -348,21 +344,28 @@ class ElectrumInterface(BlockchainInterface):
             return
         #make sure to add any addresses during the run (a subset of those
         #added to the address cache)
-        addrs = list(set(self.wallet.addr_cache.keys()).union(set(addrs)))
-        self.listunspent_calls = 0
+        for md in range(wallet.max_mixdepth):
+            for internal in (True, False):
+                for index in range(wallet.get_next_unused_index(md, internal)):
+                    addrs.add(wallet.get_addr(md, internal, index))
+            for path in wallet.yield_imported_paths(md):
+                addrs.add(wallet.get_addr_path(path))
+
+        self.listunspent_calls = len(addrs)
         for a in addrs:
+            # FIXME: update to protocol version 1.1 and use scripthash instead
+            script = wallet.addr_to_script(a)
             d = self.get_from_electrum('blockchain.address.listunspent', a)
-            d.addCallback(self.process_listunspent_data, wallet, a, len(addrs))
+            d.addCallback(self.process_listunspent_data, wallet, script)
 
-    def process_listunspent_data(self, unspent_info, wallet, address, n):
-        self.listunspent_calls += 1
+    def process_listunspent_data(self, unspent_info, wallet, script):
         res = unspent_info['result']
         for u in res:
-            wallet.unspent[str(u['tx_hash']) + ':' + str(
-                u['tx_pos'])] = {'address': address, 'value': int(u['value'])}
-        if self.listunspent_calls == n:
-            for u in wallet.spent_utxos:
-                wallet.unspent.pop(u, None)
+            txid = binascii.unhexlify(u['tx_hash'])
+            wallet.add_utxo(txid, int(u['tx_pos']), script, int(u['value']))
+
+        self.listunspent_calls -= 1
+        if self.listunspent_calls == 0:
             self.wallet_synced = True
             if self.synctype == "sync-only":
                 reactor.stop()

jmclient/jmclient/maker.py

@@ -3,18 +3,14 @@ from __future__ import print_function
 
 import base64
 import pprint
-import random
 import sys
-import time
-import copy
+from binascii import unhexlify
 
 import btc
-from jmclient.configure import jm_single, get_p2pk_vbyte, get_p2sh_vbyte
+from jmclient.configure import jm_single
 from jmbase.support import get_log
 from jmclient.support import (calc_cj_fee)
-from jmclient.wallet import estimate_tx_fee
-from jmclient.podle import (generate_podle, get_podle_commitments, verify_podle,
-                                    PoDLE, PoDLEError, generate_podle_error_string)
+from jmclient.podle import verify_podle, PoDLE
 from twisted.internet import task
 jlog = get_log()
 
@@ -75,7 +71,11 @@ class Maker(object):
         if res[0]['value'] < reqd_amt:
             reason = "commitment utxo too small: " + str(res[0]['value'])
             return reject(reason)
-        if res[0]['address'] != self.wallet.pubkey_to_address(cr_dict['P']):
+
+        # FIXME: This only works if taker's commitment address is of same type
+        # as our wallet.
+        if res[0]['address'] != \
+                self.wallet.pubkey_to_addr(unhexlify(cr_dict['P'])):
             reason = "Invalid podle pubkey: " + str(cr_dict['P'])
             return reject(reason)
 
@@ -114,22 +114,25 @@ class Maker(object):
         jlog.info('goodtx')
         sigs = []
         utxos = offerinfo["utxos"]
+
+        our_inputs = {}
         for index, ins in enumerate(tx['ins']):
             utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
-            if utxo not in utxos.keys():
+            if utxo not in utxos:
                 continue
-            addr = utxos[utxo]['address']
-            amount = utxos[utxo]["value"]
-            txs = self.wallet.sign(txhex, index,
-                                   self.wallet.get_key_from_addr(addr),
-                                   amount=amount)
-            sigmsg = btc.deserialize(txs)["ins"][index]["script"].decode("hex")
-            if "txinwitness" in btc.deserialize(txs)["ins"][index].keys():
+            script = self.wallet.addr_to_script(utxos[utxo]['address'])
+            amount = utxos[utxo]['value']
+            our_inputs[index] = (script, amount)
+
+        txs = self.wallet.sign_tx(btc.deserialize(unhexlify(txhex)), our_inputs)
+
+        for index in our_inputs:
+            sigmsg = txs['ins'][index]['script']
+            if 'txinwitness' in txs['ins'][index]:
                 #We prepend the witness data since we want (sig, pub, scriptCode);
                 #also, the items in witness are not serialize_script-ed.
-                sigmsg = "".join([btc.serialize_script_unit(
-                    x.decode("hex")) for x in btc.deserialize(
-                        txs)["ins"][index]["txinwitness"]]) + sigmsg
+                sigmsg = b''.join(btc.serialize_script_unit(x)
+                                  for x in txs['ins'][index]['txinwitness']) + sigmsg
             sigs.append(base64.b64encode(sigmsg))
         return (True, sigs)
 
@@ -216,3 +219,11 @@ class Maker(object):
                 if len(oldorder_s) > 0:
                     self.offerlist.remove(oldorder_s[0])
             self.offerlist += to_announce
+
+    def import_new_addresses(self, addr_list):
+        # FIXME: same code as in taker.py
+        bci = jm_single().bc_interface
+        if not hasattr(bci, 'import_addresses'):
+            return
+        assert hasattr(bci, 'get_wallet_name')
+        bci.import_addresses(addr_list, bci.get_wallet_name(self.wallet))

jmclient/jmclient/output.py

@@ -0,0 +1,69 @@
+from binascii import hexlify
+
+
+def fmt_utxos(utxos, wallet, prefix=''):
+    output = []
+    for u in utxos:
+        utxo_str = '{}{} - {}'.format(
+            prefix, fmt_utxo(u), fmt_tx_data(utxos[u], wallet))
+        output.append(utxo_str)
+    return '\n'.join(output)
+
+
+def fmt_utxo(utxo):
+    return '{}:{}'.format(hexlify(utxo[0]), utxo[1])
+
+
+def fmt_tx_data(tx_data, wallet):
+    return 'path: {}, address: {}, value: {}'.format(
+        wallet.get_path_repr(wallet.script_to_path(tx_data['script'])),
+        wallet.script_to_addr(tx_data['script']), tx_data['value'])
+
+
+def generate_podle_error_string(priv_utxo_pairs, to, ts, wallet, cjamount,
+                                taker_utxo_age, taker_utxo_amtpercent):
+    """Gives detailed error information on why commitment sourcing failed.
+    """
+    errmsg = ""
+    errmsgheader = ("Failed to source a commitment; this debugging information"
+                    " may help:\n\n")
+    errmsg += ("1: Utxos that passed age and size limits, but have "
+                "been used too many times (see taker_utxo_retries "
+                "in the config):\n")
+    if len(priv_utxo_pairs) == 0:
+        errmsg += ("None\n")
+    else:
+        for p, u in priv_utxo_pairs:
+            errmsg += (str(u) + "\n")
+    errmsg += "2: Utxos that have less than " + taker_utxo_age + " confirmations:\n"
+    if len(to) == 0:
+        errmsg += ("None\n")
+    else:
+        for t in to:
+            errmsg += (str(t) + "\n")
+    errmsg += ("3: Utxos that were not at least " + taker_utxo_amtpercent + \
+               "% of the size of the coinjoin amount " + str(cjamount) + "\n")
+    if len(ts) == 0:
+        errmsg += ("None\n")
+    else:
+        for t in ts:
+            errmsg += (str(t) + "\n")
+    errmsg += ('***\n')
+    errmsg += ("Utxos that appeared in item 1 cannot be used again.\n")
+    errmsg += ("Utxos only in item 2 can be used by waiting for more "
+               "confirmations, (set by the value of taker_utxo_age).\n")
+    errmsg += ("Utxos only in item 3 are not big enough for this "
+               "coinjoin transaction, set by the value "
+               "of taker_utxo_amtpercent.\n")
+    errmsg += ("If you cannot source a utxo from your wallet according "
+               "to these rules, use the tool add-utxo.py to source a "
+               "utxo external to your joinmarket wallet. Read the help "
+               "with 'python add-utxo.py --help'\n\n")
+    errmsg += ("***\nFor reference, here are the utxos in your wallet:\n")
+
+    for md, utxos in wallet.get_utxos_by_mixdepth_().items():
+        if not utxos:
+            continue
+        errmsg += ("\nmixdepth {}:\n{}".format(
+            md, fmt_utxos(utxos, wallet, prefix='    ')))
+    return (errmsgheader, errmsg)

jmclient/jmclient/podle.py

@@ -21,49 +21,6 @@ def get_commitment_file():
     return PODLE_COMMIT_FILE
 
 
-def generate_podle_error_string(priv_utxo_pairs, to, ts, unspent, cjamount,
-                                taker_utxo_age, taker_utxo_amtpercent):
-    """Gives detailed error information on why commitment sourcing failed.
-    """
-    errmsg = ""
-    errmsgheader = ("Failed to source a commitment; this debugging information"
-                    " may help:\n\n")
-    errmsg += ("1: Utxos that passed age and size limits, but have "
-                "been used too many times (see taker_utxo_retries "
-                "in the config):\n")
-    if len(priv_utxo_pairs) == 0:
-        errmsg += ("None\n")
-    else:
-        for p, u in priv_utxo_pairs:
-            errmsg += (str(u) + "\n")
-    errmsg += "2: Utxos that have less than " + taker_utxo_age + " confirmations:\n"
-    if len(to) == 0:
-        errmsg += ("None\n")
-    else:
-        for t in to:
-            errmsg += (str(t) + "\n")
-    errmsg += ("3: Utxos that were not at least " + taker_utxo_amtpercent + \
-               "% of the size of the coinjoin amount " + str(cjamount) + "\n")
-    if len(ts) == 0:
-        errmsg += ("None\n")
-    else:
-        for t in ts:
-            errmsg += (str(t) + "\n")
-    errmsg += ('***\n')
-    errmsg += ("Utxos that appeared in item 1 cannot be used again.\n")
-    errmsg += ("Utxos only in item 2 can be used by waiting for more "
-               "confirmations, (set by the value of taker_utxo_age).\n")
-    errmsg += ("Utxos only in item 3 are not big enough for this "
-               "coinjoin transaction, set by the value "
-               "of taker_utxo_amtpercent.\n")
-    errmsg += ("If you cannot source a utxo from your wallet according "
-               "to these rules, use the tool add-utxo.py to source a "
-               "utxo external to your joinmarket wallet. Read the help "
-               "with 'python add-utxo.py --help'\n\n")
-    errmsg += ("***\nFor reference, here are the utxos in your wallet:\n")
-    errmsg += ("\n" + str(unspent))
-    return (errmsgheader, errmsg)
-
 class PoDLEError(Exception):
     pass
 

jmclient/jmclient/storage.py

@@ -0,0 +1,327 @@
+from __future__ import print_function, absolute_import, division, unicode_literals
+
+import os
+import shutil
+import atexit
+import bencoder
+import pyaes
+from hashlib import sha256
+from argon2 import low_level
+from .support import get_random_bytes
+
+
+class Argon2Hash(object):
+    def __init__(self, password, salt=None, hash_len=32, salt_len=16,
+                 time_cost=500, memory_cost=1000, parallelism=4,
+                 argon2_type=low_level.Type.I, version=19):
+        """
+        args:
+          password: password as bytes
+          salt: salt in bytes or None to create random one, must have length >= 8
+          hash_len: generated hash length in bytes
+          salt_len: salt length in bytes, ignored if salt is not None, must be >= 8
+
+        Other arguments are argon2 settings. Only change those if you know what
+        you're doing. Optimized for slow hashing suitable for file encryption.
+        """
+        # Default and recommended settings from argon2.PasswordHasher are for
+        # interactive logins. For encryption we want something much slower.
+        self.settings = {
+            'time_cost': time_cost,
+            'memory_cost': memory_cost,
+            'parallelism': parallelism,
+            'hash_len': hash_len,
+            'type': argon2_type,
+            'version': version
+        }
+        self.salt = salt if salt is not None else get_random_bytes(salt_len)
+        self.hash = low_level.hash_secret_raw(password, self.salt,
+                                              **self.settings)
+
+
+class StorageError(Exception):
+    pass
+
+
+class StoragePasswordError(StorageError):
+    pass
+
+
+class Storage(object):
+    """
+    Responsible for reading/writing [encrypted] data to disk.
+
+    All data to be stored must be added to self.data which defaults to an
+    empty dict.
+
+    self.data must contain serializable data (dict, list, tuple, bytes, numbers)
+    Having str objects anywhere in self.data will lead to undefined behaviour (py3).
+    All dict keys must be bytes.
+
+    KDF: argon2, ENC: AES-256-CBC
+    """
+    MAGIC_UNENC = b'JMWALLET'
+    MAGIC_ENC =   b'JMENCWLT'
+    MAGIC_DETECT_ENC = b'JMWALLET'
+
+    ENC_KEY_BYTES = 32  # AES-256
+    SALT_LENGTH = 16
+
+    def __init__(self, path, password=None, create=False, read_only=False):
+        """
+        args:
+          path: file path to storage
+          password: bytes or None for unencrypted file
+          create: create file if it does not exist
+          read_only: do not change anything on the file system
+        """
+        self.path = path
+        self._lock_file = None
+        self._hash = None
+        self._data_checksum = None
+        self.data = None
+        self.changed = False
+        self.read_only = read_only
+        self.newly_created = False
+
+        if not os.path.isfile(path):
+            if create and not read_only:
+                self._create_new(password)
+                self._save_file()
+                self.newly_created = True
+            else:
+                raise StorageError("File not found.")
+        else:
+            self._load_file(password)
+
+        assert self.data is not None
+        assert self._data_checksum is not None
+
+        self._create_lock()
+
+    def is_encrypted(self):
+        return self._hash is not None
+
+    def is_locked(self):
+        return self._lock_file and os.path.exists(self._lock_file)
+
+    def was_changed(self):
+        """
+        return True if data differs from data on disk
+        """
+        return self._data_checksum != self._get_data_checksum()
+
+    def change_password(self, password):
+        if self.read_only:
+            raise StorageError("Cannot change password of read-only file.")
+        self._set_hash(password)
+        self._save_file()
+
+    def save(self):
+        """
+        Write file to disk if data was modified
+        """
+        #if not self.was_changed():
+        #    return
+        if self.read_only:
+            raise StorageError("Read-only storage cannot be saved.")
+        self._save_file()
+
+    @classmethod
+    def is_storage_file(cls, path):
+        return cls._get_file_magic(path) in (cls.MAGIC_ENC, cls.MAGIC_UNENC)
+
+    @classmethod
+    def is_encrypted_storage_file(cls, path):
+        return cls._get_file_magic(path) == cls.MAGIC_ENC
+
+    @classmethod
+    def _get_file_magic(cls, path):
+        assert len(cls.MAGIC_ENC) == len(cls.MAGIC_UNENC)
+        with open(path, 'rb') as fh:
+            return fh.read(len(cls.MAGIC_ENC))
+
+    def _get_data_checksum(self):
+        if self.data is None:  #pragma: no cover
+            return None
+        return sha256(self._serialize(self.data)).digest()
+
+    def _update_data_hash(self):
+        self._data_checksum = self._get_data_checksum()
+
+    def _create_new(self, password):
+        self.data = {}
+        self._set_hash(password)
+
+    def _set_hash(self, password):
+        if password is None:
+            self._hash = None
+        else:
+            self._hash = self._hash_password(password)
+
+    def _save_file(self):
+        assert self.read_only == False
+        data = self._serialize(self.data)
+        enc_data = self._encrypt_file(data)
+
+        magic = self.MAGIC_UNENC if data is enc_data else self.MAGIC_ENC
+        self._write_file(magic + enc_data)
+        self._update_data_hash()
+
+    def _load_file(self, password):
+        data = self._read_file()
+        assert len(self.MAGIC_ENC) == len(self.MAGIC_UNENC) == 8
+        magic = data[:8]
+
+        if magic not in (self.MAGIC_ENC, self.MAGIC_UNENC):
+            raise StorageError("File does not appear to be a joinmarket wallet.")
+
+        data = data[8:]
+
+        if magic == self.MAGIC_ENC:
+            if password is None:
+                raise StorageError("Password required to open wallet.")
+            data = self._decrypt_file(password, data)
+        else:
+            assert magic == self.MAGIC_UNENC
+
+        self.data = self._deserialize(data)
+        self._update_data_hash()
+
+    def _write_file(self, data):
+        assert self.read_only is False
+
+        if not os.path.exists(self.path):
+            # newly created storage
+            with open(self.path, 'wb') as fh:
+                fh.write(data)
+            return
+
+        # using a tmpfile ensures the write is atomic
+        tmpfile = '{}.tmp'.format(self.path)
+
+        with open(tmpfile, 'wb') as fh:
+            shutil.copystat(self.path, tmpfile)
+            fh.write(data)
+
+        #FIXME: behaviour with symlinks might be weird
+        shutil.move(tmpfile, self.path)
+
+    def _read_file(self):
+        # this method mainly exists for easier mocking
+        with open(self.path, 'rb') as fh:
+            return fh.read()
+
+    @staticmethod
+    def _serialize(data):
+        return bencoder.bencode(data)
+
+    @staticmethod
+    def _deserialize(data):
+        return bencoder.bdecode(data)
+
+    def _encrypt_file(self, data):
+        if not self.is_encrypted():
+            return data
+
+        iv = get_random_bytes(16)
+        container = {
+            b'enc': {b'salt': self._hash.salt, b'iv': iv},
+            b'data': self._encrypt(data, iv)
+        }
+        return self._serialize(container)
+
+    def _decrypt_file(self, password, data):
+        assert password is not None
+
+        container = self._deserialize(data)
+        assert b'enc' in container
+        assert b'data' in container
+
+        self._hash = self._hash_password(password, container[b'enc'][b'salt'])
+
+        return self._decrypt(container[b'data'], container[b'enc'][b'iv'])
+
+    def _encrypt(self, data, iv):
+        encrypter = pyaes.Encrypter(
+                pyaes.AESModeOfOperationCBC(self._hash.hash, iv=iv))
+        enc_data = encrypter.feed(self.MAGIC_DETECT_ENC + data)
+        enc_data += encrypter.feed()
+
+        return enc_data
+
+    def _decrypt(self, data, iv):
+        decrypter = pyaes.Decrypter(
+                pyaes.AESModeOfOperationCBC(self._hash.hash, iv=iv))
+        try:
+            dec_data = decrypter.feed(data)
+            dec_data += decrypter.feed()
+        except ValueError:
+            # in most "wrong password" cases the pkcs7 padding will be wrong
+            raise StoragePasswordError("Wrong password.")
+
+        if not dec_data.startswith(self.MAGIC_DETECT_ENC):
+            raise StoragePasswordError("Wrong password.")
+        return dec_data[len(self.MAGIC_DETECT_ENC):]
+
+    @classmethod
+    def _hash_password(cls, password, salt=None):
+        return Argon2Hash(password, salt,
+                          hash_len=cls.ENC_KEY_BYTES, salt_len=cls.SALT_LENGTH)
+
+    def _create_lock(self):
+        if self.read_only:
+            return
+        self._lock_file = '{}.lock'.format(self.path)
+        if os.path.exists(self._lock_file):
+            self._lock_file = None
+            raise StorageError("File is currently in use. If this is a "
+                               "leftover from a crashed instance you need to "
+                               "remove the lock file manually")
+        #FIXME: in python >=3.3 use mode xb
+        with open(self._lock_file, 'wb'):
+            pass
+
+        atexit.register(self.close)
+
+    def _remove_lock(self):
+        if self._lock_file:
+            os.remove(self._lock_file)
+            self._lock_file = None
+
+    def close(self):
+        if not self.read_only and self.was_changed():
+            self._save_file()
+        self._remove_lock()
+        self.read_only = True
+
+    def __del__(self):
+        self.close()
+
+
+class VolatileStorage(Storage):
+    """
+    Storage that is never actually written to disk and only kept in memory.
+
+    This exists for easier testing.
+    """
+
+    def __init__(self, password=None, data=None):
+        self.file_data = None
+        super(VolatileStorage, self).__init__('VOLATILE', password,
+                                              create=True)
+        if data:
+            self.file_data = data
+            self._load_file(password)
+
+    def _create_lock(self):
+        pass
+
+    def _remove_lock(self):
+        pass
+
+    def _write_file(self, data):
+        self.file_data = data
+
+    def _read_file(self):
+        return self.file_data

jmclient/jmclient/support.py

@@ -1,9 +1,5 @@
 from __future__ import absolute_import, print_function
 
-import sys
-
-import logging
-import pprint
 import random
 from jmbase.support import get_log
 from decimal import Decimal
@@ -23,6 +19,15 @@ Only for sampling purposes
 """
 
 
+def get_random_bytes(num_bytes, cryptographically_secure=False):
+    if cryptographically_secure:
+        # uses os.urandom if available
+        generator = random.SystemRandom()
+    else:
+        generator = random
+    return bytes(bytearray((generator.randrange(256) for b in xrange(num_bytes))))
+
+
 def rand_norm_array(mu, sigma, n):
     # use normalvariate instead of gauss for thread safety
     return [random.normalvariate(mu, sigma) for _ in range(n)]

jmclient/jmclient/taker.py

@@ -4,9 +4,7 @@ from __future__ import print_function
 import base64
 import pprint
 import random
-import sys
-import time
-import copy
+from binascii import hexlify, unhexlify
 
 import btc
 from jmclient.configure import jm_single, get_p2pk_vbyte, get_p2sh_vbyte
@@ -14,8 +12,9 @@ from jmbase.support import get_log
 from jmclient.support import (calc_cj_fee, weighted_order_choose, choose_orders,
                               choose_sweep_orders)
 from jmclient.wallet import estimate_tx_fee
-from jmclient.podle import (generate_podle, get_podle_commitments,
-                                    PoDLE, PoDLEError, generate_podle_error_string)
+from jmclient.podle import generate_podle, get_podle_commitments, PoDLE
+from .output import generate_podle_error_string
+
 jlog = get_log()
 
 
@@ -28,7 +27,6 @@ class Taker(object):
                  wallet,
                  schedule,
                  order_chooser=weighted_order_choose,
-                 sign_method=None,
                  callbacks=None,
                  tdestaddrs=None,
                  ignored_makers=None):
@@ -99,10 +97,8 @@ class Taker(object):
         self.waiting_for_conf = False
         self.txid = None
         self.schedule_index = -1
+        self.utxos = {}
         self.tdestaddrs = [] if not tdestaddrs else tdestaddrs
-        #allow custom wallet-based clients to use their own signing code;
-        #currently only setting "wallet" is allowed, calls wallet.sign_tx(tx)
-        self.sign_method = sign_method
         self.filter_orders_callback = callbacks[0]
         self.taker_info_callback = callbacks[1]
         if not self.taker_info_callback:
@@ -181,10 +177,11 @@ class Taker(object):
             #if destination is flagged "INTERNAL", choose a destination
             #from the next mixdepth modulo the maxmixdepth
             if self.my_cj_addr == "INTERNAL":
-                next_mixdepth = (self.mixdepth + 1) % self.wallet.max_mix_depth
+                next_mixdepth = (self.mixdepth + 1) % (self.wallet.max_mixdepth + 1)
                 jlog.info("Choosing a destination from mixdepth: " + str(next_mixdepth))
                 self.my_cj_addr = self.wallet.get_internal_addr(next_mixdepth)
                 jlog.info("Chose destination address: " + self.my_cj_addr)
+                self.import_new_addresses([self.my_cj_addr])
             self.outputs = []
             self.cjfee_total = 0
             self.maker_txfee_contributions = 0
@@ -269,6 +266,7 @@ class Taker(object):
         if self.cjamount != 0:
             try:
                 self.my_change_addr = self.wallet.get_internal_addr(self.mixdepth)
+                self.import_new_addresses([self.my_change_addr])
             except:
                 self.taker_info_callback("ABORT", "Failed to get a change address")
                 return False
@@ -367,7 +365,9 @@ class Taker(object):
             #Construct the Bitcoin address for the auth_pub field
             #Ensure that at least one address from utxos corresponds.
             input_addresses = [d['address'] for d in utxo_data]
-            auth_address = self.wallet.pubkey_to_address(auth_pub)
+            # FIXME: This only works if taker's commitment address is of same type
+            # as our wallet.
+            auth_address = self.wallet.pubkey_to_addr(unhexlify(auth_pub))
             if not auth_address in input_addresses:
                 jlog.warn("ERROR maker's (" + nick + ")"
                          " authorising pubkey is not included "
@@ -665,9 +665,12 @@ class Taker(object):
             #in the transaction, about to be consumed, rather than use
             #random utxos that will persist after. At this step we also
             #allow use of external utxos in the json file.
-            if self.wallet.unspent:
+            if any(self.wallet.get_utxos_by_mixdepth_().values()):
+                utxos = {}
+                for mdutxo in self.wallet.get_utxos_by_mixdepth().values():
+                    utxos.update(mdutxo)
                 priv_utxo_pairs, to, ts = priv_utxo_pairs_from_utxos(
-                    self.wallet.unspent, age, amt)
+                    utxos, age, amt)
             #Pre-filter the set of external commitments that work for this
             #transaction according to its size and age.
             dummy, extdict = get_podle_commitments()
@@ -688,7 +691,7 @@ class Taker(object):
                     "Commitment sourced OK")
         else:
             errmsgheader, errmsg = generate_podle_error_string(priv_utxo_pairs,
-                        to, ts, self.wallet.unspent, self.cjamount,
+                        to, ts, self.wallet, self.cjamount,
                         jm_single().config.get("POLICY", "taker_utxo_age"),
                         jm_single().config.get("POLICY", "taker_utxo_amtpercent"))
 
@@ -707,37 +710,23 @@ class Taker(object):
             #Note: donation code removed (possibly temporarily)
             raise NotImplementedError
 
-    def sign_tx(self, tx, i, priv, amount):
-        if self.my_cj_addr:
-            return self.wallet.sign(tx, i, priv, amount)
-        else:
-            #Note: donation code removed (possibly temporarily)
-            raise NotImplementedError
-
     def self_sign(self):
         # now sign it ourselves
-        tx = btc.serialize(self.latest_tx)
-        if self.sign_method == "wallet":
-            #Currently passes addresses of to-be-signed inputs
-            #to backend wallet; this is correct for Electrum, may need
-            #different info for other backends.
-            addrs = {}
-            for index, ins in enumerate(self.latest_tx['ins']):
-                utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
-                if utxo not in self.input_utxos.keys():
-                    continue
-                addrs[index] = self.input_utxos[utxo]['address']
-            tx = self.wallet.sign_tx(tx, addrs)
-        else:
-            for index, ins in enumerate(self.latest_tx['ins']):
-                utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
-                if utxo not in self.input_utxos.keys():
-                    continue
-                addr = self.input_utxos[utxo]['address']
-                amount = self.input_utxos[utxo]["value"]
-                tx = self.sign_tx(tx, index, self.wallet.get_key_from_addr(addr),
-                                  amount)
-        self.latest_tx = btc.deserialize(tx)
+        our_inputs = {}
+        for index, ins in enumerate(self.latest_tx['ins']):
+            utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
+            if utxo not in self.input_utxos.keys():
+                continue
+            script = self.wallet.addr_to_script(self.input_utxos[utxo]['address'])
+            amount = self.input_utxos[utxo]['value']
+            our_inputs[index] = (script, amount)
+
+        # FIXME: ugly hack
+        tx_bin = btc.deserialize(unhexlify(btc.serialize(self.latest_tx)))
+        self.wallet.sign_tx(tx_bin, our_inputs)
+
+        self.latest_tx = btc.deserialize(hexlify(btc.serialize(tx_bin)))
+
 
     def push(self):
         tx = btc.serialize(self.latest_tx)
@@ -804,3 +793,11 @@ class Taker(object):
         waittime = self.schedule[self.schedule_index][4]
         self.on_finished_callback(True, fromtx=fromtx, waittime=waittime,
                                   txdetails=(txd, txid))
+
+    def import_new_addresses(self, addr_list):
+        # FIXME: same code as in maker.py
+        bci = jm_single().bc_interface
+        if not hasattr(bci, 'import_addresses'):
+            return
+        assert hasattr(bci, 'get_wallet_name')
+        bci.import_addresses(addr_list, bci.get_wallet_name(self.wallet))

jmclient/jmclient/taker_utils.py

@@ -5,10 +5,11 @@ import pprint
 import os
 import time
 import numbers
+from binascii import hexlify, unhexlify
 from .configure import get_log, jm_single, validate_address
 from .schedule import human_readable_schedule_entry, tweak_tumble_schedule
-from .wallet import Wallet, SegwitWallet, estimate_tx_fee
-from jmclient import mktx, deserialize, sign, txhash
+from .wallet import BaseWallet, estimate_tx_fee
+from .btc import mktx, serialize, deserialize, sign, txhash
 log = get_log()
 
 """
@@ -42,10 +43,10 @@ def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False,
     assert mixdepth >= 0
     assert isinstance(amount, numbers.Integral)
     assert amount >=0
-    assert isinstance(wallet, Wallet) or isinstance(wallet, SegwitWallet)
+    assert isinstance(wallet, BaseWallet)
 
     from pprint import pformat
-    txtype = 'p2sh-p2wpkh' if isinstance(wallet, SegwitWallet) else 'p2pkh'
+    txtype = wallet.get_txtype()
     if amount == 0:
         utxos = wallet.get_utxos_by_mixdepth()[mixdepth]
         if utxos == {}:
@@ -67,21 +68,14 @@ def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False,
         changeval = total_inputs_val - fee_est - amount
         outs = [{"value": amount, "address": destaddr}]
         change_addr = wallet.get_internal_addr(mixdepth)
+        import_new_addresses(wallet, [change_addr])
         outs.append({"value": changeval, "address": change_addr})
 
     #Now ready to construct transaction
     log.info("Using a fee of : " + str(fee_est) + " satoshis.")
     if amount != 0:
         log.info("Using a change value of: " + str(changeval) + " satoshis.")
-    tx = mktx(utxos.keys(), outs)
-    stx = deserialize(tx)
-    for index, ins in enumerate(stx['ins']):
-        utxo = ins['outpoint']['hash'] + ':' + str(
-                ins['outpoint']['index'])
-        addr = utxos[utxo]['address']
-        signing_amount = utxos[utxo]['value']
-        amt = signing_amount if isinstance(wallet, SegwitWallet) else None
-        tx = sign(tx, index, wallet.get_key_from_addr(addr), amount=amt)
+    tx = sign_tx(wallet, mktx(utxos.keys(), outs), utxos)
     txsigned = deserialize(tx)
     log.info("Got signed transaction:\n")
     log.info(tx + "\n")
@@ -105,6 +99,31 @@ def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False,
     cb(successmsg)
     return txid
 
+
+def sign_tx(wallet, tx, utxos):
+    stx = deserialize(tx)
+    our_inputs = {}
+    for index, ins in enumerate(stx['ins']):
+        utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
+        script = wallet.addr_to_script(utxos[utxo]['address'])
+        amount = utxos[utxo]['value']
+        our_inputs[index] = (script, amount)
+
+    # FIXME: ugly hack
+    tx_bin = deserialize(unhexlify(serialize(stx)))
+    wallet.sign_tx(tx_bin, our_inputs)
+    return hexlify(serialize(tx_bin))
+
+
+def import_new_addresses(wallet, addr_list):
+    # FIXME: same code as in maker.py and taker.py
+    bci = jm_single().bc_interface
+    if not hasattr(bci, 'import_addresses'):
+        return
+    assert hasattr(bci, 'get_wallet_name')
+    bci.import_addresses(addr_list, bci.get_wallet_name(wallet))
+
+
 def get_tumble_log(logsdir):
     tumble_log = logging.getLogger('tumbler')
     tumble_log.setLevel(logging.DEBUG)

jmclient/jmclient/wallet_utils.py

@@ -1,21 +1,20 @@
 from __future__ import print_function
 import json
 import os
-import pprint
 import sys
 import sqlite3
 import binascii
 from datetime import datetime
-from mnemonic import Mnemonic
 from optparse import OptionParser
-import getpass
-from jmclient import (get_network, get_wallet_cls, Bip39Wallet, podle,
-                      encryptData, get_p2sh_vbyte, get_p2pk_vbyte, jm_single,
-                      mn_decode, mn_encode, BitcoinCoreInterface,
-                      JsonRpcError, sync_wallet, WalletError)
+from jmclient import (get_network, WALLET_IMPLEMENTATIONS, Storage, podle,
+    jm_single, BitcoinCoreInterface, JsonRpcError, sync_wallet, WalletError,
+    VolatileStorage, StoragePasswordError,
+    is_segwit_mode, SegwitLegacyWallet, LegacyWallet)
 from jmbase.support import get_password
+from cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH
 import jmclient.btc as btc
 
+
 def get_wallettool_parser():
     description = (
         'Use this script to monitor and manage your Joinmarket wallet.\n'
@@ -43,8 +42,9 @@ def get_wallettool_parser():
                       '--maxmixdepth',
                       action='store',
                       type='int',
-                      dest='maxmixdepth',
-                      help='how many mixing depths to display, default=5')
+                      dest='mixdepths',
+                      help='how many mixing depths to initialize in the wallet',
+                      default=5)
     parser.add_option('-g',
                       '--gap-limit',
                       type="int",
@@ -83,9 +83,30 @@ def get_wallettool_parser():
                       type='str',
                       dest='hd_path',
                       help='hd wallet path (e.g. m/0/0/0/000)')
+    parser.add_option('--key-type',  # note: keep in sync with map_key_type
+                      type='choice',
+                      choices=('standard', 'segwit-p2sh'),
+                      action='store',
+                      dest='key_type',
+                      default=None,
+                      help=("Key type when importing private keys.\n"
+                            "If your address starts with '1' use 'standard', "
+                            "if your address starts with '3' use 'segwit-p2sh.\n"
+                            "Native segwit addresses (starting with 'bc') are"
+                            "not yet supported."))
     return parser
 
 
+def map_key_type(parser_key_choice):
+    if not parser_key_choice:
+        return parser_key_choice
+    if parser_key_choice == 'standard':
+        return TYPE_P2PKH
+    if parser_key_choice == 'segwit-p2sh':
+        return TYPE_P2SH_P2WPKH
+    raise Exception("Unknown key type choice '{}'.".format(parser_key_choice))
+
+
 """The classes in this module manage representations
 of wallet states; but they know nothing about Bitcoin,
 so do not attempt to validate addresses, keys, BIP32 or relationships.
@@ -123,10 +144,9 @@ WalletView* classes manage wallet representations.
 """
 
 class WalletViewBase(object):
-    def __init__(self, bip32path, children=None, serclass=str,
+    def __init__(self, wallet_path_repr, children=None, serclass=str,
                  custom_separator=None):
-        assert bip32pathparse(bip32path)
-        self.bip32path = bip32path
+        self.wallet_path_repr = wallet_path_repr
         self.children = children
         self.serclass = serclass
         self.separator = custom_separator if custom_separator else "\t"
@@ -140,11 +160,10 @@ class WalletViewBase(object):
         return "{0:.08f}".format(self.get_balance(include_unconf))
 
 class WalletViewEntry(WalletViewBase):
-    def __init__(self, bip32path, account, forchange, aindex, addr, amounts,
+    def __init__(self, wallet_path_repr, account, forchange, aindex, addr, amounts,
                  used = 'new', serclass=str, priv=None, custom_separator=None):
-        self.bip32path = bip32path
-        super(WalletViewEntry, self).__init__(bip32path, serclass=serclass,
-                                             custom_separator=custom_separator)
+        super(WalletViewEntry, self).__init__(wallet_path_repr, serclass=serclass,
+                                              custom_separator=custom_separator)
         self.account = account
         assert forchange in [0, 1, -1]
         self.forchange =forchange
@@ -172,10 +191,7 @@ class WalletViewEntry(WalletViewBase):
         return self.serclass(self.separator.join([left, addr, amounts, extradata]))
 
     def serialize_wallet_position(self):
-        bippath = self.bip32path + bip32sep + str(self.account) + "'" + \
-        bip32sep + str(self.forchange) + bip32sep + "{0:03d}".format(self.aindex)
-        assert bip32pathparse(bippath)
-        return self.serclass(bippath)
+        return self.wallet_path_repr.ljust(20)
 
     def serialize_address(self):
         return self.serclass(self.address)
@@ -195,11 +211,11 @@ class WalletViewEntry(WalletViewBase):
         return self.serclass(ed)
 
 class WalletViewBranch(WalletViewBase):
-    def __init__(self, bip32path, account, forchange, branchentries=None,
+    def __init__(self, wallet_path_repr, account, forchange, branchentries=None,
                  xpub=None, serclass=str, custom_separator=None):
-        super(WalletViewBranch, self).__init__(bip32path, children=branchentries,
-                                              serclass=serclass,
-                                              custom_separator=custom_separator)
+        super(WalletViewBranch, self).__init__(wallet_path_repr, children=branchentries,
+                                               serclass=serclass,
+                                               custom_separator=custom_separator)
         self.account = account
         assert forchange in [0, 1, -1]
         self.forchange = forchange
@@ -220,20 +236,18 @@ class WalletViewBranch(WalletViewBase):
             return self.serclass(entryseparator.join(lines))
 
     def serialize_branch_header(self):
-        bippath = self.bip32path + bip32sep + str(self.account) + "'" + \
-            bip32sep + str(self.forchange)
-        assert bip32pathparse(bippath)
         start = "external addresses" if self.forchange == 0 else "internal addresses"
         if self.forchange == -1:
             start = "Imported keys"
-        return self.serclass(self.separator.join([start, bippath, self.xpub]))
+        return self.serclass(self.separator.join([start, self.wallet_path_repr,
+                                                  self.xpub]))
 
 class WalletViewAccount(WalletViewBase):
-    def __init__(self, bip32path, account, branches=None, account_name="mixdepth",
+    def __init__(self, wallet_path_repr, account, branches=None, account_name="mixdepth",
                  serclass=str, custom_separator=None, xpub=None):
-        super(WalletViewAccount, self).__init__(bip32path, children=branches,
-                                               serclass=serclass,
-                                               custom_separator=custom_separator)
+        super(WalletViewAccount, self).__init__(wallet_path_repr, children=branches,
+                                                serclass=serclass,
+                                                custom_separator=custom_separator)
         self.account = account
         self.account_name = account_name
         self.xpub = xpub
@@ -256,12 +270,11 @@ class WalletViewAccount(WalletViewBase):
                 x.serialize(entryseparator) for x in self.branches] + [footer]))
 
 class WalletView(WalletViewBase):
-    def __init__(self, bip32path, accounts, wallet_name="JM wallet",
+    def __init__(self, wallet_path_repr, accounts, wallet_name="JM wallet",
                  serclass=str, custom_separator=None):
-        super(WalletView, self).__init__(bip32path, children=accounts,
-                                              serclass=serclass,
-                                              custom_separator=custom_separator)
-        self.bip32path = bip32path
+        super(WalletView, self).__init__(wallet_path_repr, children=accounts,
+                                         serclass=serclass,
+                                         custom_separator=custom_separator)
         self.wallet_name = wallet_name
         assert all([isinstance(x, WalletViewAccount) for x in accounts])
         self.accounts = accounts
@@ -277,40 +290,41 @@ class WalletView(WalletViewBase):
                 x.serialize(entryseparator, summarize=False) for x in self.accounts] + [footer]))
 
 def get_imported_privkey_branch(wallet, m, showprivkey):
-    if m in wallet.imported_privkeys:
-        entries = []
-        for i, privkey in enumerate(wallet.imported_privkeys[m]):
-            pub = btc.privkey_to_pubkey(privkey)
-            addr = btc.pubkey_to_p2sh_p2wpkh_address(pub, magicbyte=get_p2sh_vbyte())
-            balance = 0.0
-            for addrvalue in wallet.unspent.values():
-                if addr == addrvalue['address']:
-                    balance += addrvalue['value']
-            used = ('used' if balance > 0.0 else 'empty')
-            if showprivkey:
-                wip_privkey = btc.wif_compressed_privkey(
-                privkey, get_p2pk_vbyte())
-            else:
-                wip_privkey = ''
-            entries.append(WalletViewEntry("m/0", m, -1,
-                                           i, addr, [balance, balance],
-                                           used=used,priv=wip_privkey))
+    entries = []
+    for path in wallet.yield_imported_paths(m):
+        addr = wallet.get_addr_path(path)
+        script = wallet.get_script_path(path)
+        balance = 0.0
+        for data in wallet.get_utxos_by_mixdepth_()[m].values():
+            if script == data['script']:
+                balance += data['value']
+        used = ('used' if balance > 0.0 else 'empty')
+        if showprivkey:
+            wip_privkey = wallet.get_wif_path(path)
+        else:
+            wip_privkey = ''
+        entries.append(WalletViewEntry(wallet.get_path_repr(path), m, -1,
+                                       0, addr, [balance, balance],
+                                       used=used, priv=wip_privkey))
+
+    if entries:
         return WalletViewBranch("m/0", m, -1, branchentries=entries)
     return None
 
 def wallet_showutxos(wallet, showprivkey):
     unsp = {}
     max_tries = jm_single().config.getint("POLICY", "taker_utxo_retries")
-    for u, av in wallet.unspent.iteritems():
-        key = wallet.get_key_from_addr(av['address'])
-        tries = podle.get_podle_tries(u, key, max_tries)
-        tries_remaining = max(0, max_tries - tries)
-        unsp[u] = {'address': av['address'], 'value': av['value'],
-                   'tries': tries, 'tries_remaining': tries_remaining,
-                   'external': False}
-        if showprivkey:
-            wifkey = btc.wif_compressed_privkey(key, vbyte=get_p2pk_vbyte())
-            unsp[u]['privkey'] = wifkey
+    utxos = wallet.get_utxos_by_mixdepth()
+    for md in utxos:
+        for u, av in utxos[md].items():
+            key = wallet.get_key_from_addr(av['address'])
+            tries = podle.get_podle_tries(u, key, max_tries)
+            tries_remaining = max(0, max_tries - tries)
+            unsp[u] = {'address': av['address'], 'value': av['value'],
+                       'tries': tries, 'tries_remaining': tries_remaining,
+                       'external': False}
+            if showprivkey:
+                unsp[u]['privkey'] = wallet.get_wif_path(av['path'])
 
     used_commitments, external_commitments = podle.get_podle_commitments()
     for u, ec in external_commitments.iteritems():
@@ -329,44 +343,48 @@ def wallet_display(wallet, gaplimit, showprivkey, displayall=False,
     else return the WalletView object.
     """
     acctlist = []
-    rootpath = wallet.get_root_path()
-    for m in range(wallet.max_mix_depth):
+    for m in xrange(wallet.max_mixdepth + 1):
         branchlist = []
         for forchange in [0, 1]:
             entrylist = []
             if forchange == 0:
-                xpub_key = btc.bip32_privtopub(wallet.keys[m][forchange])
+                # users would only want to hand out the xpub for externals
+                xpub_key = wallet.get_bip32_pub_export(m, forchange)
             else:
                 xpub_key = ""
 
-            for k in range(wallet.index[m][forchange] + gaplimit):
-                addr = wallet.get_addr(m, forchange, k)
+            unused_index = wallet.get_next_unused_index(m, forchange)
+            for k in xrange(unused_index + gaplimit):
+                path = wallet.get_path(m, forchange, k)
+                addr = wallet.get_addr_path(path)
                 balance = 0
-                for addrvalue in wallet.unspent.values():
-                    if addr == addrvalue['address']:
-                        balance += addrvalue['value']
-                used = 'used' if k < wallet.index[m][forchange] else 'new'
+                for utxodata in wallet.get_utxos_by_mixdepth_()[m].values():
+                    if path == utxodata['path']:
+                        balance += utxodata['value']
+                used = 'used' if k < unused_index else 'new'
                 if showprivkey:
-                    privkey = btc.wif_compressed_privkey(
-                        wallet.get_key(m, forchange, k), get_p2pk_vbyte())
+                    privkey = wallet.get_wif_path(path)
                 else:
                     privkey = ''
                 if (displayall or balance > 0 or
-                    (used == 'new' and forchange == 0)):
-                    entrylist.append(WalletViewEntry(rootpath, m, forchange, k,
-                                                 addr, [balance, balance],
-                                                 priv=privkey, used=used))
-            branchlist.append(WalletViewBranch(rootpath, m, forchange,
-                entrylist, xpub=xpub_key))
+                        (used == 'new' and forchange == 0)):
+                    entrylist.append(WalletViewEntry(
+                        wallet.get_path_repr(path), m, forchange, k, addr,
+                        [balance, balance], priv=privkey, used=used))
+            wallet.set_next_index(m, forchange, unused_index)
+            path = wallet.get_path_repr(wallet.get_path(m, forchange))
+            branchlist.append(WalletViewBranch(path, m, forchange, entrylist,
+                                               xpub=xpub_key))
         ipb = get_imported_privkey_branch(wallet, m, showprivkey)
         if ipb:
             branchlist.append(ipb)
         #get the xpub key of the whole account
-        xpub_account = btc.bip32_privtopub(
-            wallet.get_mixing_depth_keys(wallet.get_master_key())[m])
-        acctlist.append(WalletViewAccount(rootpath, m, branchlist,
+        xpub_account = wallet.get_bip32_pub_export(mixdepth=m)
+        path = wallet.get_path_repr(wallet.get_path(m))
+        acctlist.append(WalletViewAccount(path, m, branchlist,
                                           xpub=xpub_account))
-    walletview = WalletView(rootpath, acctlist)
+    path = wallet.get_path_repr(wallet.get_path())
+    walletview = WalletView(path, acctlist)
     if serialized:
         return walletview.serialize(summarize=summarized)
     else:
@@ -381,7 +399,7 @@ def cli_get_wallet_passphrase_check():
     return password
 
 def cli_get_wallet_file_name():
-    return raw_input('Input wallet file name (default: wallet.json): ')
+    return raw_input('Input wallet file name (default: wallet.jmdat): ')
 
 def cli_display_user_words(words, mnemonic_extension):
     text = 'Write down this wallet recovery mnemonic\n\n' + words +'\n'
@@ -390,47 +408,24 @@ def cli_display_user_words(words, mnemonic_extension):
     print(text)
 
 def cli_user_mnemonic_entry():
-    mnemonic_phrase = raw_input("Input 12 word mnemonic recovery phrase: ")
+    mnemonic_phrase = raw_input("Input mnemonic recovery phrase: ")
     mnemonic_extension = raw_input("Input mnemonic extension, leave blank if there isnt one: ")
     if len(mnemonic_extension.strip()) == 0:
         mnemonic_extension = None
     return (mnemonic_phrase, mnemonic_extension)
 
 def cli_get_mnemonic_extension():
-    uin = raw_input('Would you like to use a two-factor mnemonic recovery'
-        + ' phrase? write \'n\' if you don\'t know what this is (y/n): ')
+    uin = raw_input("Would you like to use a two-factor mnemonic recovery "
+                    "phrase? write 'n' if you don't know what this is (y/n): ")
     if len(uin) == 0 or uin[0] != 'y':
-        print('Not using mnemonic extension')
+        print("Not using mnemonic extension")
         return None #no mnemonic extension
-    return raw_input('Enter mnemonic extension: ')
-
-def persist_walletfile(walletspath, default_wallet_name, encrypted_entropy,
-                       encrypted_mnemonic_extension=None,
-                       callbacks=(cli_get_wallet_file_name,)):
-    timestamp = datetime.now().strftime("%Y/%m/%d %H:%M:%S")
-    walletjson = {'creator': 'joinmarket project',
-                  'creation_time': timestamp,
-                  'encrypted_entropy': encrypted_entropy.encode('hex'),
-                  'network': get_network()}
-    if encrypted_mnemonic_extension:
-        walletjson['encrypted_mnemonic_extension'] = encrypted_mnemonic_extension.encode('hex')
-    walletfile = json.dumps(walletjson)
-    walletname = callbacks[0]()
-    if len(walletname) == 0:
-        walletname = default_wallet_name
-    walletpath = os.path.join(walletspath, walletname)
-    # Does a wallet with the same name exist?
-    if os.path.isfile(walletpath):
-        print('ERROR: ' + walletpath + ' already exists. Aborting.')
-        return False
-    else:
-        fd = open(walletpath, 'w')
-        fd.write(walletfile)
-        fd.close()
-        print('saved to ' + walletname)
-    return True
+    print("Note: This will be stored in a reversible way. Do not reuse!")
+    return raw_input("Enter mnemonic extension: ")
+
 
 def wallet_generate_recover_bip39(method, walletspath, default_wallet_name,
+                                  mixdepths=5,
                                   callbacks=(cli_display_user_words,
                                              cli_user_mnemonic_entry,
                                              cli_get_wallet_passphrase_check,
@@ -444,68 +439,77 @@ def wallet_generate_recover_bip39(method, walletspath, default_wallet_name,
     4 - enter mnemonic extension
     The defaults are for terminal entry.
     """
-    #using 128 bit entropy, 12 words, mnemonic module
-    m = Mnemonic("english")
+    entropy = None
+    mnemonic_extension = None
     if method == "generate":
         mnemonic_extension = callbacks[4]()
-        words = m.generate()
-        callbacks[0](words, mnemonic_extension)
     elif method == 'recover':
         words, mnemonic_extension = callbacks[1]()
+        mnemonic_extension = mnemonic_extension and mnemonic_extension.strip()
         if not words:
             return False
-    entropy = str(m.to_entropy(words))
+        try:
+            entropy = SegwitLegacyWallet.entropy_from_mnemonic(words)
+        except WalletError:
+            return False
+    else:
+        raise Exception("unknown method for wallet creation: '{}'"
+                        .format(method))
+
     password = callbacks[2]()
     if not password:
         return False
-    password_key = btc.bin_dbl_sha256(password)
-    encrypted_entropy = encryptData(password_key, entropy)
-    encrypted_mnemonic_extension = None
-    if mnemonic_extension:
-        mnemonic_extension = mnemonic_extension.strip()
-        #check all ascii printable
-        if not all([a > '\x19' and a < '\x7f' for a in mnemonic_extension]):
-            return False
-        #padding to stop an adversary easily telling how long the mn extension is
-        #padding at the start because of how aes blocks are combined
-        #checksum in order to tell whether the decryption was successful
-        cleartext_length = 79
-        padding_length = cleartext_length - 10 - len(mnemonic_extension)
-        if padding_length > 0:
-            padding = os.urandom(padding_length).replace('\xff', '\xfe')
-        else:
-            padding = ''
-        cleartext = (padding + '\xff' + mnemonic_extension + '\xff'
-            + btc.dbl_sha256(mnemonic_extension)[:8])
-        encrypted_mnemonic_extension = encryptData(password_key, cleartext)
-    return persist_walletfile(walletspath, default_wallet_name, encrypted_entropy,
-                              encrypted_mnemonic_extension, callbacks=(callbacks[3],))
+
+    wallet_name = callbacks[3]()
+    if not wallet_name:
+        wallet_name = default_wallet_name
+    wallet_path = os.path.join(walletspath, wallet_name)
+
+    wallet = create_wallet(wallet_path, password, mixdepths - 1,
+                           entropy=entropy,
+                           entropy_extension=mnemonic_extension)
+    mnemonic, mnext = wallet.get_mnemonic_words()
+    callbacks[0] and callbacks[0](mnemonic, mnext or '')
+    wallet.close()
+    return True
+
 
 def wallet_generate_recover(method, walletspath,
-                            default_wallet_name='wallet.json'):
-    if jm_single().config.get("POLICY", "segwit") == "true":
+                            default_wallet_name='wallet.jmdat',
+                            mixdepths=5):
+    if is_segwit_mode():
         #Here using default callbacks for scripts (not used in Qt)
-        return wallet_generate_recover_bip39(method, walletspath,
-                                             default_wallet_name)
-    if method == 'generate':
-        seed = btc.sha256(os.urandom(64))[:32]
-        words = mn_encode(seed)
-        print('Write down this wallet recovery seed\n\n' + ' '.join(words) +
-              '\n')
-    elif method == 'recover':
-        words = raw_input('Input 12 word recovery seed: ')
-        words = words.split()  # default for split is 1 or more whitespace chars
-        if len(words) != 12:
-            print('ERROR: Recovery seed phrase must be exactly 12 words.')
+        return wallet_generate_recover_bip39(
+            method, walletspath, default_wallet_name, mixdepths=mixdepths)
+
+    entropy = None
+    if method == 'recover':
+        seed = raw_input("Input 12 word recovery seed: ")
+        try:
+            entropy = LegacyWallet.entropy_from_mnemonic(seed)
+        except WalletError as e:
+            print("Unable to restore seed: {}".format(e.message))
             return False
-        seed = mn_decode(words)
-        print(seed)
+    elif method != 'generate':
+        raise Exception("unknown method for wallet creation: '{}'"
+                        .format(method))
+
     password = cli_get_wallet_passphrase_check()
     if not password:
         return False
-    password_key = btc.bin_dbl_sha256(password)
-    encrypted_seed = encryptData(password_key, seed.decode('hex'))
-    return persist_walletfile(walletspath, default_wallet_name, encrypted_seed)
+
+    wallet_name = cli_get_wallet_file_name()
+    if not wallet_name:
+        wallet_name = default_wallet_name
+    wallet_path = os.path.join(walletspath, wallet_name)
+
+    wallet = create_wallet(wallet_path, password, mixdepths - 1,
+                           wallet_cls=LegacyWallet, entropy=entropy)
+    print("Write down and safely store this wallet recovery seed\n\n{}\n"
+          .format(wallet.get_mnemonic_words()[0]))
+    wallet.close()
+    return True
+
 
 def wallet_fetch_history(wallet, options):
     # sort txes in a db because python can be really bad with large lists
@@ -528,10 +532,12 @@ def wallet_fetch_history(wallet, options):
                     in tx)
             tx_db.executemany('INSERT INTO transactions VALUES(?, ?, ?);',
                     tx_data)
-            txes = tx_db.execute('SELECT DISTINCT txid, blockhash, blocktime '
-                    'FROM transactions ORDER BY blocktime').fetchall()
-            wallet_addr_cache = wallet.addr_cache
-    wallet_addr_set = set(wallet_addr_cache.keys())
+
+    txes = tx_db.execute(
+        'SELECT DISTINCT txid, blockhash, blocktime '
+        'FROM transactions ORDER BY blocktime').fetchall()
+    wallet_script_set = set(wallet.get_script_path(p)
+                            for p in wallet.yield_known_paths())
 
     def s():
         return ',' if options.csv else ' '
@@ -570,13 +576,13 @@ def wallet_fetch_history(wallet, options):
         rpctx = jm_single().bc_interface.rpc('gettransaction', [tx['txid']])
         txhex = str(rpctx['hex'])
         txd = btc.deserialize(txhex)
-        output_addr_values = dict(((btc.script_to_address(sv['script'],
-            get_p2sh_vbyte()), sv['value']) for sv in txd['outs']))
-        our_output_addrs = wallet_addr_set.intersection(
-                output_addr_values.keys())
+        output_script_values = {binascii.unhexlify(sv['script']): sv['value']
+                                for sv in txd['outs']}
+        our_output_scripts = wallet_script_set.intersection(
+                output_script_values.keys())
 
         from collections import Counter
-        value_freq_list = sorted(Counter(output_addr_values.values())
+        value_freq_list = sorted(Counter(output_script_values.values())
                 .most_common(), key=lambda x: -x[1])
         non_cj_freq = 0 if len(value_freq_list)==1 else sum(zip(
             *value_freq_list[1:])[1])
@@ -596,12 +602,12 @@ def wallet_fetch_history(wallet, options):
                 'outpoint']['index']]
             rpc_inputs.append(input_dict)
 
-        rpc_input_addrs = set((btc.script_to_address(ind['script'],
-            get_p2sh_vbyte()) for ind in rpc_inputs))
-        our_input_addrs = wallet_addr_set.intersection(rpc_input_addrs)
-        our_input_values = [ind['value'] for ind in rpc_inputs if btc.
-                script_to_address(ind['script'], get_p2sh_vbyte()) in
-                our_input_addrs]
+        rpc_input_scripts = set(binascii.unhexlify(ind['script'])
+                                for ind in rpc_inputs)
+        our_input_scripts = wallet_script_set.intersection(rpc_input_scripts)
+        our_input_values = [
+            ind['value'] for ind in rpc_inputs
+            if binascii.unhexlify(ind['script']) in our_input_scripts]
         our_input_value = sum(our_input_values)
         utxos_consumed = len(our_input_values)
 
@@ -613,19 +619,19 @@ def wallet_fetch_history(wallet, options):
         mixdepth_dst = -1
         #TODO this seems to assume all the input addresses are from the same
         # mixdepth, which might not be true
-        if len(our_input_addrs) == 0 and len(our_output_addrs) > 0:
+        if len(our_input_scripts) == 0 and len(our_output_scripts) > 0:
             #payment to us
-            amount = sum([output_addr_values[a] for a in our_output_addrs])
+            amount = sum([output_script_values[a] for a in our_output_scripts])
             tx_type = 'deposit    '
             cj_n = -1
             delta_balance = amount
-            mixdepth_dst = tuple(wallet_addr_cache[a][0] for a in
-                    our_output_addrs)
+            mixdepth_dst = tuple(wallet.get_script_mixdepth(a)
+                                 for a in our_output_scripts)
             if len(mixdepth_dst) == 1:
                 mixdepth_dst = mixdepth_dst[0]
-        elif len(our_input_addrs) == 0 and len(our_output_addrs) == 0:
+        elif len(our_input_scripts) == 0 and len(our_output_scripts) == 0:
             continue            # skip those that don't belong to our wallet
-        elif len(our_input_addrs) > 0 and len(our_output_addrs) == 0:
+        elif len(our_input_scripts) > 0 and len(our_output_scripts) == 0:
             # we swept coins elsewhere
             if is_coinjoin:
                 tx_type = 'cj sweepout'
@@ -633,13 +639,13 @@ def wallet_fetch_history(wallet, options):
                 fees = our_input_value - cj_amount
             else:
                 tx_type = 'sweep out  '
-                amount = sum([v for v in output_addr_values.values()])
+                amount = sum([v for v in output_script_values.values()])
                 fees = our_input_value - amount
             delta_balance = -our_input_value
-            mixdepth_src = wallet_addr_cache[list(our_input_addrs)[0]][0]
-        elif len(our_input_addrs) > 0 and len(our_output_addrs) == 1:
+            mixdepth_src = wallet.get_script_mixdepth(list(our_input_scripts)[0])
+        elif len(our_input_scripts) > 0 and len(our_output_scripts) == 1:
             # payment to somewhere with our change address getting the remaining
-            change_value = output_addr_values[list(our_output_addrs)[0]]
+            change_value = output_script_values[list(our_output_scripts)[0]]
             if is_coinjoin:
                 tx_type = 'cj withdraw'
                 amount = cj_amount
@@ -650,25 +656,25 @@ def wallet_fetch_history(wallet, options):
                 cj_n = -1
             delta_balance = change_value - our_input_value
             fees = our_input_value - change_value - cj_amount
-            mixdepth_src = wallet_addr_cache[list(our_input_addrs)[0]][0]
-        elif len(our_input_addrs) > 0 and len(our_output_addrs) == 2:
+            mixdepth_src = wallet.get_script_mixdepth(list(our_input_scripts)[0])
+        elif len(our_input_scripts) > 0 and len(our_output_scripts) == 2:
             #payment to self
-            out_value = sum([output_addr_values[a] for a in our_output_addrs])
+            out_value = sum([output_script_values[a] for a in our_output_scripts])
             if not is_coinjoin:
                 print('this is wrong TODO handle non-coinjoin internal')
             tx_type = 'cj internal'
             amount = cj_amount
             delta_balance = out_value - our_input_value
-            mixdepth_src = wallet_addr_cache[list(our_input_addrs)[0]][0]
-            cj_addr = list(set([a for a,v in output_addr_values.iteritems()
-                if v == cj_amount]).intersection(our_output_addrs))[0]
-            mixdepth_dst = wallet_addr_cache[cj_addr][0]
+            mixdepth_src = wallet.get_script_mixdepth(list(our_input_scripts)[0])
+            cj_script = list(set([a for a, v in output_script_values.iteritems()
+                if v == cj_amount]).intersection(our_output_scripts))[0]
+            mixdepth_dst = wallet.get_script_mixdepth(cj_script)
         else:
             tx_type = 'unknown type'
-            print('our utxos: ' + str(len(our_input_addrs)) \
-                  + ' in, ' + str(len(our_output_addrs)) + ' out')
+            print('our utxos: ' + str(len(our_input_scripts)) \
+                  + ' in, ' + str(len(our_output_scripts)) + ' out')
         balance += delta_balance
-        utxo_count += (len(our_output_addrs) - utxos_consumed)
+        utxo_count += (len(our_output_scripts) - utxos_consumed)
         index = '% 4d'%(i)
         timestamp = datetime.fromtimestamp(rpctx['blocktime']
                 ).strftime("%Y-%m-%d %H:%M")
@@ -755,89 +761,200 @@ def wallet_fetch_history(wallet, options):
         print(('BUG ERROR: wallet balance (%s) does not match balance from ' +
             'history (%s)') % (sat_to_str(total_wallet_balance),
                 sat_to_str(balance)))
-    if utxo_count != len(wallet.unspent):
+    wallet_utxo_count = sum(map(len, wallet.get_utxos_by_mixdepth_().values()))
+    if utxo_count != wallet_utxo_count:
         print(('BUG ERROR: wallet utxo count (%d) does not match utxo count from ' +
-            'history (%s)') % (len(wallet.unspent), utxo_count))
+            'history (%s)') % (wallet_utxo_count, utxo_count))
 
 
 def wallet_showseed(wallet):
-    if isinstance(wallet, Bip39Wallet):
-        if not wallet.entropy:
-            return "Entropy is not initialized."
-        m = Mnemonic("english")
-        text = "Wallet mnemonic recovery phrase:\n\n" + m.to_mnemonic(wallet.entropy) + "\n"
-        if wallet.mnemonic_extension:
-            text += '\nWallet mnemonic extension: ' + wallet.mnemonic_extension + '\n'
-        return text
-    hexseed = wallet.seed
-    print("hexseed = " + hexseed)
-    words = mn_encode(hexseed)
-    return "Wallet mnemonic seed phrase:\n\n" + " ".join(words) + "\n"
-
-def wallet_importprivkey(wallet, mixdepth):
-    print('WARNING: This imported key will not be recoverable with your 12 ' +
-          'word mnemonic phrase. Make sure you have backups.')
-    print('WARNING: Handling of raw ECDSA bitcoin private keys can lead to '
-          'non-intuitive behaviour and loss of funds.\n  Recommended instead '
-          'is to use the \'sweep\' feature of sendpayment.py ')
-    privkeys = raw_input('Enter private key(s) to import: ')
+    seed, extension = wallet.get_mnemonic_words()
+    text = "Wallet mnemonic recovery phrase:\n\n{}\n".format(seed)
+    if extension:
+        text += "\nWallet mnemonic extension: {}\n".format(extension)
+    return text
+
+
+def wallet_importprivkey(wallet, mixdepth, key_type):
+    print("WARNING: This imported key will not be recoverable with your 12 "
+          "word mnemonic phrase. Make sure you have backups.")
+    print("WARNING: Handling of raw ECDSA bitcoin private keys can lead to "
+          "non-intuitive behaviour and loss of funds.\n  Recommended instead "
+          "is to use the \'sweep\' feature of sendpayment.py.")
+    privkeys = raw_input("Enter private key(s) to import: ")
     privkeys = privkeys.split(',') if ',' in privkeys else privkeys.split()
+    imported_addr = []
+    import_failed = 0
     # TODO read also one key for each line
-    for privkey in privkeys:
+    for wif in privkeys:
         # TODO is there any point in only accepting wif format? check what
         # other wallets do
-        privkey_bin = btc.from_wif_privkey(privkey,
-                                        vbyte=get_p2pk_vbyte()).decode('hex')[:-1]
-        encrypted_privkey = encryptData(wallet.password_key, privkey_bin)
-        if 'imported_keys' not in wallet.walletdata:
-            wallet.walletdata['imported_keys'] = []
-        wallet.walletdata['imported_keys'].append(
-            {'encrypted_privkey': encrypted_privkey.encode('hex'),
-             'mixdepth': mixdepth})
-    if wallet.walletdata['imported_keys']:
-        fd = open(wallet.path, 'w')
-        fd.write(json.dumps(wallet.walletdata))
-        fd.close()
-        print('Private key(s) successfully imported')
+        try:
+            path = wallet.import_private_key(mixdepth, wif, key_type=key_type)
+        except WalletError as e:
+            print("Failed to import key {}: {}".format(wif, e))
+            import_failed += 1
+        else:
+            imported_addr.append(wallet.get_addr_path(path))
+
+    if not imported_addr:
+        print("Warning: No keys imported!")
+        return
+
+    wallet.save()
+
+    # show addresses to user so they can verify everything went as expected
+    print("Imported keys for addresses:\n{}".format('\n'.join(imported_addr)))
+    if import_failed:
+        print("Warning: failed to import {} keys".format(import_failed))
+
 
 def wallet_dumpprivkey(wallet, hdpath):
-    pathlist = bip32pathparse(hdpath)
-    print('got pathlist: ' + str(pathlist))
-    if pathlist and len(pathlist) in [5, 4]:
-        #note here we assume the path conforms to Wallet or SegwitWallet(BIP49) standard
-        m, forchange, k = pathlist[-3:]
-        key = wallet.get_key(m, forchange, k)
-        wifkey = btc.wif_compressed_privkey(key, vbyte=get_p2pk_vbyte())
-        return wifkey
-    else:
-        return hdpath + " is not a valid hd wallet path"
+    if not hdpath:
+        print("Error: no hd wallet path supplied")
+        return False
+    path = wallet.path_repr_to_path(hdpath)
+    return wallet.get_wif_path(path)  # will raise exception on invalid path
+
 
 def wallet_signmessage(wallet, hdpath, message):
-    if hdpath.startswith(wallet.get_root_path()):
-        hp = bip32pathparse(hdpath)
-        m, forchange, k = hp[-3:]
-        key = wallet.get_key(m, forchange, k)
-        addr = wallet.pubkey_to_address(btc.privkey_to_pubkey(key))
-        print('Using address: ' + addr)
+    msg = message.encode('utf-8')
+
+    if not hdpath:
+        return "Error: no key path for signing specified"
+    if not message:
+        return "Error: no message specified"
+
+    path = wallet.path_repr_to_path(hdpath)
+    sig = wallet.sign_message(msg, path)
+    return ("Signature: {}\n"
+            "To verify this in Bitcoin Core use the RPC command 'verifymessage'"
+            .format(sig))
+
+
+def get_wallet_type():
+    if is_segwit_mode():
+        return TYPE_P2SH_P2WPKH
+    return TYPE_P2PKH
+
+
+def get_wallet_cls(wtype=None):
+    if wtype is None:
+        wtype = get_wallet_type()
+
+    cls = WALLET_IMPLEMENTATIONS.get(wtype)
+
+    if not cls:
+        raise WalletError("No wallet implementation found for type {}."
+                          "".format(wtype))
+    return cls
+
+
+def create_wallet(path, password, max_mixdepth, wallet_cls=None, **kwargs):
+    storage = Storage(path, password, create=True)
+    wallet_cls = wallet_cls or get_wallet_cls()
+    wallet_cls.initialize(storage, get_network(), max_mixdepth=max_mixdepth,
+                          **kwargs)
+    storage.save()
+    return wallet_cls(storage)
+
+
+def open_test_wallet_maybe(path, seed, max_mixdepth,
+                           test_wallet_cls=SegwitLegacyWallet, **kwargs):
+    """
+    Create a volatile test wallet if path is a hex-encoded string of length 64,
+    otherwise run open_wallet().
+
+    params:
+        path: path to wallet file, ignored for test wallets
+        seed: hex-encoded test seed
+        max_mixdepth: see create_wallet(), ignored when calling open_wallet()
+        kwargs: see open_wallet()
+
+    returns:
+        wallet object
+    """
+    if len(seed) == test_wallet_cls.ENTROPY_BYTES * 2:
+        try:
+            seed = binascii.unhexlify(seed)
+        except binascii.Error:
+            pass
+        else:
+            storage = VolatileStorage()
+            test_wallet_cls.initialize(
+                storage, get_network(), max_mixdepth=max_mixdepth,
+                entropy=seed)
+            assert 'ask_for_password' not in kwargs
+            assert 'read_only' not in kwargs
+            return test_wallet_cls(storage, **kwargs)
+
+    return open_wallet(path, **kwargs)
+
+
+def open_wallet(path, ask_for_password=True, password=None, read_only=False,
+                **kwargs):
+    """
+    Open the wallet file at path and return the corresponding wallet object.
+
+    params:
+        path: str, full path to wallet file
+        ask_for_password: bool, if False password is assumed unset and user
+            will not be asked to type it
+        password: password for storage, ignored if ask_for_password is True
+        read_only: bool, if True, open wallet in read-only mode
+        kwargs: additional options to pass to wallet's init method
+
+    returns:
+        wallet object
+    """
+    if ask_for_password:
+        while True:
+            try:
+                # do not try empty password, assume unencrypted on empty password
+                pwd = get_password("Enter wallet decryption passphrase: ") or None
+                storage = Storage(path, password=pwd, read_only=read_only)
+            except StoragePasswordError:
+                print("Wrong password, try again.")
+                continue
+            except Exception as e:
+                print("Failed to load wallet, error message: " + repr(e))
+                raise e
+            break
     else:
-        print('%s is not a valid hd wallet path' % hdpath)
-        return None
-    sig = btc.ecdsa_sign(message, key, formsg=True)
-    retval = "Signature: " + str(sig) + "\n"
-    retval += "To verify this in Bitcoin Core use the RPC command 'verifymessage'"
-    return retval
+        storage = Storage(path, password, read_only=read_only)
+
+    wallet_cls = get_wallet_cls_from_storage(storage)
+    wallet = wallet_cls(storage, **kwargs)
+    wallet_sanity_check(wallet)
+    return wallet
+
+
+def get_wallet_cls_from_storage(storage):
+    wtype = storage.data.get(b'wallet_type')
+
+    if wtype is None:
+        raise WalletError("File {} is not a valid wallet.".format(storage.path))
+
+    return get_wallet_cls(wtype)
+
+
+def wallet_sanity_check(wallet):
+    if wallet.network != get_network():
+        raise Exception("Wallet network mismatch: we are on '{}' but wallet "
+                        "is on '{}'.".format(get_network(), wallet.network))
+
+
+def get_wallet_path(file_name, wallet_dir):
+    # TODO: move default wallet path to ~/.joinmarket
+    wallet_dir = wallet_dir or 'wallets'
+    return os.path.join(wallet_dir, file_name)
+
 
 def wallet_tool_main(wallet_root_path):
     """Main wallet tool script function; returned is a string (output or error)
     """
     parser = get_wallettool_parser()
     (options, args) = parser.parse_args()
-    # if the index_cache stored in wallet.json is longer than the default
-    # then set maxmixdepth to the length of index_cache
-    maxmixdepth_configured = True
-    if not options.maxmixdepth:
-        maxmixdepth_configured = False
-        options.maxmixdepth = 5
 
     noseed_methods = ['generate', 'recover']
     methods = ['display', 'displayall', 'summary', 'showseed', 'importprivkey',
@@ -849,44 +966,33 @@ def wallet_tool_main(wallet_root_path):
         parser.error('Needs a wallet file or method')
         sys.exit(0)
 
+    if options.mixdepths < 1:
+        parser.error("Must have at least one mixdepth.")
+        sys.exit(0)
+
     if args[0] in noseed_methods:
         method = args[0]
     else:
         seed = args[0]
+        wallet_path = get_wallet_path(seed, wallet_root_path)
         method = ('display' if len(args) == 1 else args[1].lower())
-        if not os.path.exists(os.path.join(wallet_root_path, seed)):
-            wallet = get_wallet_cls()(seed, None, options.maxmixdepth,
-                    options.gaplimit, extend_mixdepth= not maxmixdepth_configured,
-                    storepassword=(method == 'importprivkey'),
-                    wallet_dir=wallet_root_path)
-        else:
-            while True:
-                try:
-                    pwd = get_password("Enter wallet decryption passphrase: ")
-                    wallet = get_wallet_cls()(seed, pwd,
-                            options.maxmixdepth,
-                            options.gaplimit,
-                            extend_mixdepth=not maxmixdepth_configured,
-                            storepassword=(method == 'importprivkey'),
-                            wallet_dir=wallet_root_path)
-                except WalletError:
-                    print("Wrong password, try again.")
-                    continue
-                except Exception as e:
-                    print("Failed to load wallet, error message: " + repr(e))
-                    sys.exit(0)
-                break
+
+        wallet = open_test_wallet_maybe(
+            wallet_path, seed, options.mixdepths - 1, gap_limit=options.gaplimit)
+
         if method not in noscan_methods:
             # if nothing was configured, we override bitcoind's options so that
             # unconfirmed balance is included in the wallet display by default
             if 'listunspent_args' not in jm_single().config.options('POLICY'):
                 jm_single().config.set('POLICY','listunspent_args', '[0]')
-            sync_wallet(wallet, fast=options.fastsync)
+            while not jm_single().bc_interface.wallet_synced:
+                sync_wallet(wallet, fast=options.fastsync)
     #Now the wallet/data is prepared, execute the script according to the method
     if method == "display":
         return wallet_display(wallet, options.gaplimit, options.showprivkey)
     elif method == "displayall":
-        return wallet_display(wallet, options.gaplimit, options.showprivkey, displayall=True)
+        return wallet_display(wallet, options.gaplimit, options.showprivkey,
+                              displayall=True)
     elif method == "summary":
         return wallet_display(wallet, options.gaplimit, options.showprivkey, summarized=True)
     elif method == "history":
@@ -897,10 +1003,12 @@ def wallet_tool_main(wallet_root_path):
         else:
             return wallet_fetch_history(wallet, options)
     elif method == "generate":
-        retval = wallet_generate_recover("generate", wallet_root_path)
+        retval = wallet_generate_recover("generate", wallet_root_path,
+                                         mixdepths=options.mixdepths)
         return retval if retval else "Failed"
     elif method == "recover":
-        retval = wallet_generate_recover("recover", wallet_root_path)
+        retval = wallet_generate_recover("recover", wallet_root_path,
+                                         mixdepths=options.mixdepths)
         return retval if retval else "Failed"
     elif method == "showutxos":
         return wallet_showutxos(wallet, options.showprivkey)
@@ -910,11 +1018,13 @@ def wallet_tool_main(wallet_root_path):
         return wallet_dumpprivkey(wallet, options.hd_path)
     elif method == "importprivkey":
         #note: must be interactive (security)
-        wallet_importprivkey(wallet, options.mixdepth)
+        wallet_importprivkey(wallet, options.mixdepth,
+                             map_key_type(options.key_type))
         return "Key import completed."
     elif method == "signmessage":
         return wallet_signmessage(wallet, options.hd_path, args[2])
 
+
 #Testing (can port to test modules, TODO)
 if __name__ == "__main__":
     if not test_bip32_pathparse():

jmclient/jmclient/yieldgenerator.py

@@ -7,10 +7,10 @@ import time
 import abc
 from twisted.python.log import startLogging
 from optparse import OptionParser
-from jmbase import get_password
 from jmclient import (Maker, jm_single, get_network, load_program_config, get_log,
-                      get_wallet_cls, sync_wallet, JMClientProtocolFactory,
+                      sync_wallet, JMClientProtocolFactory,
                       start_reactor, calc_cj_fee, WalletError)
+from .wallet_utils import open_test_wallet_maybe, get_wallet_path
 
 jlog = get_log()
 
@@ -129,9 +129,10 @@ class YieldGeneratorBasic(YieldGenerator):
         jlog.info('filling offer, mixdepth=' + str(mixdepth))
 
         # mixdepth is the chosen depth we'll be spending from
-        cj_addr = self.wallet.get_internal_addr((mixdepth + 1) %
-                                                self.wallet.max_mix_depth)
+        cj_addr = self.wallet.get_internal_addr(
+            (mixdepth + 1) % (self.wallet.max_mixdepth + 1))
         change_addr = self.wallet.get_internal_addr(mixdepth)
+        self.import_new_addresses([cj_addr, change_addr])
 
         utxos = self.wallet.select_utxos(mixdepth, total_amount)
         my_total_in = sum([va['value'] for va in utxos.values()])
@@ -232,26 +233,16 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffe
     nickserv_password = options.password
 
     load_program_config()
-    if not os.path.exists(os.path.join('wallets', wallet_name)):
-        wallet = get_wallet_cls()(wallet_name, None, max_mix_depth=MAX_MIX_DEPTH,
-                                  gaplimit=options.gaplimit)
-    else:
-        while True:
-            try:
-                pwd = get_password("Enter wallet decryption passphrase: ")
-                wallet = get_wallet_cls()(wallet_name, pwd,
-                                          max_mix_depth=MAX_MIX_DEPTH,
-                                          gaplimit=options.gaplimit)
-            except WalletError:
-                print("Wrong password, try again.")
-                continue
-            except Exception as e:
-                print("Failed to load wallet, error message: " + repr(e))
-                sys.exit(0)
-            break
+
+    wallet_path = get_wallet_path(wallet_name, 'wallets')
+    wallet = open_test_wallet_maybe(
+        wallet_path, wallet_name, 4, gap_limit=options.gaplimit)
+
     if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == "electrum-server":
         jm_single().bc_interface.synctype = "with-script"
-    sync_wallet(wallet, fast=options.fastsync)
+
+    while not jm_single().bc_interface.wallet_synced:
+        sync_wallet(wallet, fast=options.fastsync)
 
     maker = ygclass(wallet, [options.txfee, cjfee_a, cjfee_r,
                              options.ordertype, options.minsize])
@@ -265,4 +256,3 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffe
     start_reactor(jm_single().config.get("DAEMON", "daemon_host"),
                       jm_single().config.getint("DAEMON", "daemon_port"),
                       clientfactory, daemon=daemon)
-

jmclient/setup.py

@@ -9,5 +9,5 @@ setup(name='joinmarketclient',
       author_email='',
       license='GPL',
       packages=['jmclient'],
-      install_requires=['joinmarketbase==0.3.5', 'mnemonic', 'qt4reactor'],
+      install_requires=['joinmarketbase==0.3.5', 'mnemonic', 'qt4reactor', 'argon2_cffi', 'bencoder.pyx', 'pyaes'],
       zip_safe=False)

jmclient/test/commontest.py

@@ -1,19 +1,15 @@
 #! /usr/bin/env python
-from __future__ import absolute_import
+from __future__ import absolute_import, print_function
 '''Some helper functions for testing'''
 
-import sys
 import os
-import time
 import binascii
-import pexpect
 import random
-import subprocess
-import platform
 from decimal import Decimal
 
-from jmclient import (jm_single, Wallet, get_log, estimate_tx_fee,
-                      BlockchainInterface, get_p2sh_vbyte)
+from jmclient import (
+    jm_single, open_test_wallet_maybe, get_log, estimate_tx_fee,
+    BlockchainInterface, get_p2sh_vbyte, BIP32Wallet, SegwitLegacyWallet)
 from jmbase.support import chunks
 import jmbitcoin as btc
 
@@ -35,6 +31,8 @@ class DummyBlockchainInterface(BlockchainInterface):
         pass
     def sync_unspent(self, wallet):
         pass
+    def import_addresses(self, addr_list, wallet_name):
+        pass
     def outputs_watcher(self, wallet_name, notifyaddr,
                         tx_output_set, uf, cf, tf):
         pass
@@ -106,29 +104,21 @@ class DummyBlockchainInterface(BlockchainInterface):
     def estimate_fee_per_kb(self, N):
         return 30000
 
-class TestWallet(Wallet):
-    """Implementation of wallet
-    that allows passing in a password
-    for removal of command line interrupt.
-    """
 
-    def __init__(self,
-                 seedarg,
-                 max_mix_depth=2,
-                 gaplimit=6,
-                 extend_mixdepth=False,
-                 storepassword=False,
-                 pwd=None):
-        self.given_pwd = pwd
-        super(TestWallet, self).__init__(seedarg,
-                                     max_mix_depth,
-                                     gaplimit,
-                                     extend_mixdepth,
-                                     storepassword)
-
-    def read_wallet_file_data(self, filename):
-        return super(TestWallet, self).read_wallet_file_data(
-            filename, self.given_pwd)
+def create_wallet_for_sync(wallet_structure, a, **kwargs):
+    #We need a distinct seed for each run so as not to step over each other;
+    #make it through a deterministic hash
+    seedh = btc.sha256("".join([str(x) for x in a]))[:32]
+    return make_wallets(
+        1, [wallet_structure], fixed_seeds=[seedh], **kwargs)[0]['wallet']
+
+
+def binarize_tx(tx):
+    for o in tx['outs']:
+        o['script'] = binascii.unhexlify(o['script'])
+    for i in tx['ins']:
+        i['outpoint']['hash'] = binascii.unhexlify(i['outpoint']['hash'])
+
 
 def make_sign_and_push(ins_full,
                        wallet,
@@ -150,17 +140,17 @@ def make_sign_and_push(ins_full,
              'address': output_addr}, {'value': total - amount - fee_est,
                                        'address': change_addr}]
 
-    tx = btc.mktx(ins, outs)
-    de_tx = btc.deserialize(tx)
+    de_tx = btc.deserialize(btc.mktx(ins, outs))
+    scripts = {}
     for index, ins in enumerate(de_tx['ins']):
         utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
-        addr = ins_full[utxo]['address']
-        priv = wallet.get_key_from_addr(addr)
-        if index % 2:
-            priv = binascii.unhexlify(priv)
-        tx = btc.sign(tx, index, priv, hashcode=hashcode)
+        script = wallet.addr_to_script(ins_full[utxo]['address'])
+        scripts[index] = (script, ins_full[utxo]['value'])
+    binarize_tx(de_tx)
+    de_tx = wallet.sign_tx(de_tx, scripts, hashcode=hashcode)
     #pushtx returns False on any error
-    print btc.deserialize(tx)
+    print(de_tx)
+    tx = binascii.hexlify(btc.serialize(de_tx))
     push_succeed = jm_single().bc_interface.pushtx(tx)
     if push_succeed:
         return btc.txhash(tx)
@@ -173,8 +163,9 @@ def make_wallets(n,
                  sdev_amt=0,
                  start_index=0,
                  fixed_seeds=None,
-                 test_wallet=False,
-                 passwords=None):
+                 wallet_cls=SegwitLegacyWallet,
+                 mixdepths=5,
+                 populate_internal=False):
     '''n: number of wallets to be created
        wallet_structure: array of n arrays , each subarray
        specifying the number of addresses to be populated with coins
@@ -182,34 +173,33 @@ def make_wallets(n,
        mean_amt: the number of coins (in btc units) in each address as above
        sdev_amt: if randomness in amouts is desired, specify here.
        Returns: a dict of dicts of form {0:{'seed':seed,'wallet':Wallet object},1:..,}
-       Default Wallet constructor is joinmarket.Wallet, else use TestWallet,
-       which takes a password parameter as in the list passwords.
        '''
+    # FIXME: this is basically the same code as test/common.py
+    assert mixdepths > 0
     if len(wallet_structures) != n:
         raise Exception("Number of wallets doesn't match wallet structures")
     if not fixed_seeds:
-        seeds = chunks(binascii.hexlify(os.urandom(15 * n)), 15 * 2)
+        seeds = chunks(binascii.hexlify(os.urandom(BIP32Wallet.ENTROPY_BYTES * n)),
+                       BIP32Wallet.ENTROPY_BYTES * 2)
     else:
         seeds = fixed_seeds
     wallets = {}
     for i in range(n):
-        if test_wallet:
-            w = Wallet(seeds[i], passwords[i], max_mix_depth=5)
-        else:
-            w = Wallet(seeds[i], None, max_mix_depth=5)
+        assert len(seeds[i]) == BIP32Wallet.ENTROPY_BYTES * 2
+
+        w = open_test_wallet_maybe(seeds[i], seeds[i], mixdepths - 1,
+                                   test_wallet_cls=wallet_cls)
+
         wallets[i + start_index] = {'seed': seeds[i],
                                     'wallet': w}
-        for j in range(5):
+        for j in range(mixdepths):
             for k in range(wallet_structures[i][j]):
                 deviation = sdev_amt * random.random()
                 amt = mean_amt - sdev_amt / 2.0 + deviation
                 if amt < 0: amt = 0.001
                 amt = float(Decimal(amt).quantize(Decimal(10)**-8))
                 jm_single().bc_interface.grab_coins(
-                    wallets[i + start_index]['wallet'].get_external_addr(j),
-                    amt)
-            #reset the index so the coins can be seen if running in same script
-            wallets[i + start_index]['wallet'].index[j][0] -= wallet_structures[i][j]
+                    w.get_new_addr(j, populate_internal), amt)
     return wallets
 
 

jmclient/test/taker_test_data.py

@@ -45,11 +45,11 @@ t_chosen_orders = {u'J659UPUSLLjHJpaB': {u'cjfee': u'0.0002',
 """
 t_utxos_by_mixdepth = {0: {u'534b635ed8891f16c4ec5b8236ae86164783903e8e8bb47fa9ef2ca31f3c2d7a:0': {'address': u'mrcNu71ztWjAQA6ww9kHiW3zBWSQidHXTQ',
                                                                              'value': 200000000}},
- 1: {u'0780d6e5e381bff01a3519997bb4fcba002493103a198fde334fd264f9835d75:1': {'address': u'mvtY8DVgn3TtvjHbVsauYoSQjAhNqVyqmM',
+ 1: {u'0780d6e5e381bff01a3519997bb4fcba002493103a198fde334fd264f9835d75:1': {'address': u'n31WD8pkfAjg2APV78GnbDTdZb1QonBi5D',
                                                                              'value': 200000000},
-     u'7e574db96a4d43a99786b3ea653cda9e4388f377848f489332577e018380cff1:0': {'address': u'n3nELhmU2D7ebGYzJnGFWgVDK3cYErmTcQ',
+     u'7e574db96a4d43a99786b3ea653cda9e4388f377848f489332577e018380cff1:0': {'address': u'mmVEKH61BZbLbnVEmk9VmojreB4G4PmBPd',
                                                                              'value': 200000000},
-     u'dd9711a2ef340750db21efb761f5f7d665d94b312332dc354e252c77e9c48349:0': {'address': u'mxeLuX8PP7qLkcM8uarHmdZyvP1b5e1Ynf',
+     u'dd9711a2ef340750db21efb761f5f7d665d94b312332dc354e252c77e9c48349:0': {'address': u'msxyyydNXTiBmt3SushXbH5Qh2ukBAThk3',
                                                                              'value': 200000000}},
  2: {},
  3: {},

jmclient/test/test_argon2.py

@@ -0,0 +1,35 @@
+from __future__ import print_function, absolute_import, division, unicode_literals
+
+from jmclient import Argon2Hash, get_random_bytes
+import pytest
+
+
+def test_argon2_sanity():
+    pwd = b'password'
+    salt = b'saltsalt'
+
+    h = Argon2Hash(pwd, salt, 16)
+
+    assert len(h.hash) == 16
+    assert h.salt == salt
+    assert h.hash == b'\x05;V\xd7fy\xdfI\xa4\xe7F$_\\3\xcb'
+
+
+def test_get_random_bytes():
+    assert len(get_random_bytes(16)) == 16
+    assert get_random_bytes(16) != get_random_bytes(16)
+
+
+def test_argon2():
+    pwd = b'testpass'
+    h = Argon2Hash(pwd, hash_len=16, salt_len=22)
+
+    assert len(h.hash) == 16
+    assert len(h.salt) == 22
+
+    h2 = Argon2Hash(pwd, h.salt, hash_len=16)
+
+    assert h.settings == h2.settings
+    assert h.hash == h2.hash
+    assert h.salt == h2.salt
+

jmclient/test/test_blockchaininterface.py

@@ -0,0 +1,149 @@
+from __future__ import absolute_import, print_function
+
+"""Blockchaininterface functionality tests."""
+
+import binascii
+from commontest import create_wallet_for_sync, make_sign_and_push
+
+import pytest
+from jmclient import load_program_config, jm_single, sync_wallet, get_log
+
+log = get_log()
+
+
+def sync_test_wallet(fast, wallet):
+    sync_count = 0
+    jm_single().bc_interface.wallet_synced = False
+    while not jm_single().bc_interface.wallet_synced:
+        sync_wallet(wallet, fast=fast)
+        sync_count += 1
+        # avoid infinite loop
+        assert sync_count < 10
+        log.debug("Tried " + str(sync_count) + " times")
+
+
+@pytest.mark.parametrize('fast', (False, True))
+def test_empty_wallet_sync(setup_wallets, fast):
+    wallet = create_wallet_for_sync([0, 0, 0, 0, 0], ['test_empty_wallet_sync'])
+
+    sync_test_wallet(fast, wallet)
+
+    broken = True
+    for md in range(wallet.max_mixdepth + 1):
+        for internal in (True, False):
+            broken = False
+            assert 0 == wallet.get_next_unused_index(md, internal)
+    assert not broken
+
+
+@pytest.mark.parametrize('fast,internal', (
+        (False, False), (False, True),
+        (True, False), (True, True)))
+def test_sequentially_used_wallet_sync(setup_wallets, fast, internal):
+    used_count = [1, 3, 6, 2, 23]
+    wallet = create_wallet_for_sync(
+        used_count, ['test_sequentially_used_wallet_sync'],
+        populate_internal=internal)
+
+    sync_test_wallet(fast, wallet)
+
+    broken = True
+    for md in range(len(used_count)):
+        broken = False
+        assert used_count[md] == wallet.get_next_unused_index(md, internal)
+    assert not broken
+
+
+@pytest.mark.parametrize('fast', (False, True))
+def test_gap_used_wallet_sync(setup_wallets, fast):
+    used_count = [1, 3, 6, 2, 23]
+    wallet = create_wallet_for_sync(used_count, ['test_gap_used_wallet_sync'])
+    wallet.gap_limit = 20
+
+    for md in range(len(used_count)):
+        x = -1
+        for x in range(md):
+            assert x <= wallet.gap_limit, "test broken"
+            # create some unused addresses
+            wallet.get_new_script(md, True)
+            wallet.get_new_script(md, False)
+        used_count[md] += x + 2
+        jm_single().bc_interface.grab_coins(wallet.get_new_addr(md, True), 1)
+        jm_single().bc_interface.grab_coins(wallet.get_new_addr(md, False), 1)
+
+    # reset indices to simulate completely unsynced wallet
+    for md in range(wallet.max_mixdepth + 1):
+        wallet.set_next_index(md, True, 0)
+        wallet.set_next_index(md, False, 0)
+
+    sync_test_wallet(fast, wallet)
+
+    broken = True
+    for md in range(len(used_count)):
+        broken = False
+        assert md + 1 == wallet.get_next_unused_index(md, True)
+        assert used_count[md] == wallet.get_next_unused_index(md, False)
+    assert not broken
+
+
+@pytest.mark.parametrize('fast', (False, True))
+def test_multigap_used_wallet_sync(setup_wallets, fast):
+    start_index = 5
+    used_count = [start_index, 0, 0, 0, 0]
+    wallet = create_wallet_for_sync(used_count, ['test_multigap_used_wallet_sync'])
+    wallet.gap_limit = 5
+
+    mixdepth = 0
+    for w in range(5):
+        for x in range(int(wallet.gap_limit * 0.6)):
+            assert x <= wallet.gap_limit, "test broken"
+            # create some unused addresses
+            wallet.get_new_script(mixdepth, True)
+            wallet.get_new_script(mixdepth, False)
+        used_count[mixdepth] += x + 2
+        jm_single().bc_interface.grab_coins(wallet.get_new_addr(mixdepth, True), 1)
+        jm_single().bc_interface.grab_coins(wallet.get_new_addr(mixdepth, False), 1)
+
+    # reset indices to simulate completely unsynced wallet
+    for md in range(wallet.max_mixdepth + 1):
+        wallet.set_next_index(md, True, 0)
+        wallet.set_next_index(md, False, 0)
+
+    sync_test_wallet(fast, wallet)
+
+    assert used_count[mixdepth] - start_index == wallet.get_next_unused_index(mixdepth, True)
+    assert used_count[mixdepth] == wallet.get_next_unused_index(mixdepth, False)
+
+
+@pytest.mark.parametrize('fast', (False, True))
+def test_retain_unused_indices_wallet_sync(setup_wallets, fast):
+    used_count = [0, 0, 0, 0, 0]
+    wallet = create_wallet_for_sync(used_count, ['test_retain_unused_indices_wallet_sync'])
+
+    for x in range(9):
+        wallet.get_new_script(0, 1)
+
+    sync_test_wallet(fast, wallet)
+
+    assert wallet.get_next_unused_index(0, 1) == 9
+
+
+@pytest.mark.parametrize('fast', (False, True))
+def test_imported_wallet_sync(setup_wallets, fast):
+    used_count = [0, 0, 0, 0, 0]
+    wallet = create_wallet_for_sync(used_count, ['test_imported_wallet_sync'])
+    source_wallet = create_wallet_for_sync(used_count, ['test_imported_wallet_sync_origin'])
+
+    address = source_wallet.get_new_addr(0, 1)
+    wallet.import_private_key(0, source_wallet.get_wif(0, 1, 0))
+    txid = binascii.unhexlify(jm_single().bc_interface.grab_coins(address, 1))
+
+    sync_test_wallet(fast, wallet)
+
+    assert wallet._utxos.have_utxo(txid, 0) == 0
+
+
+@pytest.fixture(scope='module')
+def setup_wallets():
+    load_program_config()
+    jm_single().bc_interface.tick_forward_chain_interval = 1

jmclient/test/test_coinjoin.py

@@ -10,10 +10,11 @@ import pytest
 from twisted.internet import reactor
 
 from jmclient import load_program_config, jm_single, get_log,\
-    YieldGeneratorBasic, Taker, sync_wallet
+    YieldGeneratorBasic, Taker, sync_wallet, LegacyWallet, SegwitLegacyWallet
 from jmclient.podle import set_commitment_file
-from commontest import make_wallets
+from commontest import make_wallets, binarize_tx
 from test_taker import dummy_filter_orderbook
+import jmbitcoin as btc
 
 testdir = os.path.dirname(os.path.realpath(__file__))
 log = get_log()
@@ -51,8 +52,11 @@ def create_orderbook(makers):
 
 def create_taker(wallet, schedule, monkeypatch):
     def on_finished_callback(*args, **kwargs):
+        log.debug("on finished called with: {}, {}".format(args, kwargs))
+        on_finished_callback.status = args[0]
         on_finished_callback.called = True
     on_finished_callback.called = False
+    on_finished_callback.status = None
     taker = Taker(wallet, schedule, callbacks=(dummy_filter_orderbook, None,
                                                on_finished_callback))
 
@@ -104,7 +108,8 @@ def do_tx_signing(taker, makers, active_orders, txdata):
     return taker_final_result
 
 
-def test_simple_coinjoin(monkeypatch, tmpdir, setup_cj):
+@pytest.mark.parametrize('wallet_cls', (LegacyWallet, SegwitLegacyWallet))
+def test_simple_coinjoin(monkeypatch, tmpdir, setup_cj, wallet_cls):
     def raise_exit(i):
         raise Exception("sys.exit called")
     monkeypatch.setattr(sys, 'exit', raise_exit)
@@ -113,7 +118,7 @@ def test_simple_coinjoin(monkeypatch, tmpdir, setup_cj):
     MAKER_NUM = 3
     wallets = make_wallets_to_list(make_wallets(
         MAKER_NUM + 1, wallet_structures=[[4, 0, 0, 0, 0]] * (MAKER_NUM + 1),
-        mean_amt=1))
+        mean_amt=1, wallet_cls=wallet_cls))
 
     jm_single().bc_interface.tickchain()
     sync_wallets(wallets)
@@ -138,6 +143,115 @@ def test_simple_coinjoin(monkeypatch, tmpdir, setup_cj):
 
     taker_final_result = do_tx_signing(taker, makers, active_orders, txdata)
     assert taker_final_result is not False
+    assert taker.on_finished_callback.status is not False
+
+
+def test_coinjoin_mixdepth_wrap_taker(monkeypatch, tmpdir, setup_cj):
+    def raise_exit(i):
+        raise Exception("sys.exit called")
+    monkeypatch.setattr(sys, 'exit', raise_exit)
+    set_commitment_file(str(tmpdir.join('commitments.json')))
+
+    MAKER_NUM = 3
+    wallets = make_wallets_to_list(make_wallets(
+        MAKER_NUM + 1,
+        wallet_structures=[[4, 0, 0, 0, 0]] * MAKER_NUM + [[0, 0, 0, 0, 3]],
+        mean_amt=1))
+
+    for w in wallets:
+        assert w.max_mixdepth == 4
+
+    jm_single().bc_interface.tickchain()
+    jm_single().bc_interface.tickchain()
+    sync_wallets(wallets)
+
+    cj_fee = 2000
+    makers = [YieldGeneratorBasic(
+        wallets[i],
+        [0, cj_fee, 0, 'swabsoffer', 10**7]) for i in range(MAKER_NUM)]
+
+    orderbook = create_orderbook(makers)
+    assert len(orderbook) == MAKER_NUM
+
+    cj_amount = int(1.1 * 10**8)
+    # mixdepth, amount, counterparties, dest_addr, waittime
+    schedule = [(4, cj_amount, MAKER_NUM, 'INTERNAL', 0)]
+    taker = create_taker(wallets[-1], schedule, monkeypatch)
+
+    active_orders, maker_data = init_coinjoin(taker, makers,
+                                              orderbook, cj_amount)
+
+    txdata = taker.receive_utxos(maker_data)
+    assert txdata[0], "taker.receive_utxos error"
+
+    taker_final_result = do_tx_signing(taker, makers, active_orders, txdata)
+    assert taker_final_result is not False
+
+    tx = btc.deserialize(txdata[2])
+    binarize_tx(tx)
+
+    w = wallets[-1]
+    w.remove_old_utxos_(tx)
+    w.add_new_utxos_(tx, b'\x00' * 32)  # fake txid
+
+    balances = w.get_balance_by_mixdepth()
+    assert balances[0] == cj_amount
+    # <= because of tx fee
+    assert balances[4] <= 3 * 10**8 - cj_amount - (cj_fee * MAKER_NUM)
+
+
+def test_coinjoin_mixdepth_wrap_maker(monkeypatch, tmpdir, setup_cj):
+    def raise_exit(i):
+        raise Exception("sys.exit called")
+    monkeypatch.setattr(sys, 'exit', raise_exit)
+    set_commitment_file(str(tmpdir.join('commitments.json')))
+
+    MAKER_NUM = 2
+    wallets = make_wallets_to_list(make_wallets(
+        MAKER_NUM + 1,
+        wallet_structures=[[0, 0, 0, 0, 4]] * MAKER_NUM + [[3, 0, 0, 0, 0]],
+        mean_amt=1))
+
+    for w in wallets:
+        assert w.max_mixdepth == 4
+
+    jm_single().bc_interface.tickchain()
+    jm_single().bc_interface.tickchain()
+    sync_wallets(wallets)
+
+    cj_fee = 2000
+    makers = [YieldGeneratorBasic(
+        wallets[i],
+        [0, cj_fee, 0, 'swabsoffer', 10**7]) for i in range(MAKER_NUM)]
+
+    orderbook = create_orderbook(makers)
+    assert len(orderbook) == MAKER_NUM
+
+    cj_amount = int(1.1 * 10**8)
+    # mixdepth, amount, counterparties, dest_addr, waittime
+    schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0)]
+    taker = create_taker(wallets[-1], schedule, monkeypatch)
+
+    active_orders, maker_data = init_coinjoin(taker, makers,
+                                              orderbook, cj_amount)
+
+    txdata = taker.receive_utxos(maker_data)
+    assert txdata[0], "taker.receive_utxos error"
+
+    taker_final_result = do_tx_signing(taker, makers, active_orders, txdata)
+    assert taker_final_result is not False
+
+    tx = btc.deserialize(txdata[2])
+    binarize_tx(tx)
+
+    for i in range(MAKER_NUM):
+        w = wallets[i]
+        w.remove_old_utxos_(tx)
+        w.add_new_utxos_(tx, b'\x00' * 32)  # fake txid
+
+        balances = w.get_balance_by_mixdepth()
+        assert balances[0] == cj_amount
+        assert balances[4] == 4 * 10**8 - cj_amount + cj_fee
 
 
 @pytest.fixture(scope='module')

jmclient/test/test_maker.py

@@ -2,10 +2,11 @@
 
 from __future__ import print_function
 
-from jmclient import AbstractWallet, Maker, btc, get_p2sh_vbyte, get_p2pk_vbyte, \
+from jmclient import Maker, btc, get_p2sh_vbyte, get_p2pk_vbyte, \
     load_program_config, jm_single
 import jmclient
 from commontest import DummyBlockchainInterface
+from test_taker import DummyWallet
 
 import struct
 import binascii
@@ -13,10 +14,6 @@ from itertools import chain
 import pytest
 
 
-class MockWallet(AbstractWallet):
-    pass
-
-
 class OfflineMaker(Maker):
     def try_to_create_my_orders(self):
         self.sync_wait_loop.stop()
@@ -116,7 +113,7 @@ def test_verify_unsigned_tx_sw_valid(setup_env_nodeps):
     p2sh_gen = address_p2sh_generator()
     p2pkh_gen = address_p2pkh_generator()
 
-    wallet = MockWallet()
+    wallet = DummyWallet()
     maker = OfflineMaker(wallet)
 
     cj_addr, cj_script = next(p2sh_gen)
@@ -149,7 +146,7 @@ def test_verify_unsigned_tx_nonsw_valid(setup_env_nodeps):
     p2sh_gen = address_p2sh_generator()
     p2pkh_gen = address_p2pkh_generator()
 
-    wallet = MockWallet()
+    wallet = DummyWallet()
     maker = OfflineMaker(wallet)
 
     cj_addr, cj_script = next(p2pkh_gen)

jmclient/test/test_podle.py

@@ -7,16 +7,12 @@ import binascii
 import json
 import pytest
 import copy
-import subprocess
-import signal
-from commontest import make_wallets
-import time
-from pprint import pformat
 from jmclient import (load_program_config, get_log, jm_single, generate_podle,
                       generate_podle_error_string, set_commitment_file,
                       get_commitment_file, PoDLE, get_podle_commitments,
                       add_external_commitments, update_commitments)
 from jmclient.podle import verify_all_NUMS, verify_podle, PoDLEError
+from commontest import make_wallets
 log = get_log()
 
 def test_commitments_empty(setup_podle):
@@ -197,14 +193,14 @@ def test_podle_error_string(setup_podle):
                              ('fakepriv2', 'fakeutxo2')]
     to = ['tooold1', 'tooold2']
     ts = ['toosmall1', 'toosmall2']
-    unspent = "dummyunspent"
+    wallet = make_wallets(1, [[1, 0, 0, 0, 0]])[0]['wallet']
     cjamt = 100
     tua = "3"
     tuamtper = "20"
     errmgsheader, errmsg = generate_podle_error_string(priv_utxo_pairs,
                                                        to,
                                                        ts,
-                                                       unspent,
+                                                       wallet,
                                                        cjamt,
                                                        tua,
                                                        tuamtper)
@@ -213,7 +209,7 @@ def test_podle_error_string(setup_podle):
     y = [x[1] for x in priv_utxo_pairs]
     assert all([errmsg.find(x) != -1 for x in to + ts + y])
     #ensure OK with nothing
-    errmgsheader, errmsg = generate_podle_error_string([], [], [], unspent,
+    errmgsheader, errmsg = generate_podle_error_string([], [], [], wallet,
                                                        cjamt, tua, tuamtper)
 
 @pytest.fixture(scope="module")

jmclient/test/test_storage.py

@@ -0,0 +1,128 @@
+from __future__ import print_function, absolute_import, division, unicode_literals
+
+from jmclient import storage
+import pytest
+
+
+class MockStorage(storage.Storage):
+    def __init__(self, data, *args, **kwargs):
+        self.file_data = data
+        self.locked = False
+        super(type(self), self).__init__(*args, **kwargs)
+
+    def _read_file(self):
+        if hasattr(self, 'file_data'):
+            return self.file_data
+        return b''
+
+    def _write_file(self, data):
+        self.file_data = data
+
+    def _create_lock(self):
+        self.locked = not self.read_only
+
+    def _remove_lock(self):
+        self.locked = False
+
+
+def test_storage():
+    s = MockStorage(None, 'nonexistant', b'password', create=True)
+    assert s.file_data.startswith(s.MAGIC_ENC)
+    assert s.locked
+    assert s.is_encrypted()
+    assert not s.was_changed()
+
+    old_data = s.file_data
+
+    s.data[b'mydata'] = b'test'
+    assert s.was_changed()
+    s.save()
+    assert s.file_data != old_data
+    enc_data = s.file_data
+
+    old_data = s.file_data
+    s.change_password(b'newpass')
+    assert s.is_encrypted()
+    assert not s.was_changed()
+    assert s.file_data != old_data
+
+    old_data = s.file_data
+    s.change_password(None)
+    assert not s.is_encrypted()
+    assert not s.was_changed()
+    assert s.file_data != old_data
+    assert s.file_data.startswith(s.MAGIC_UNENC)
+
+    s2 = MockStorage(enc_data, __file__, b'password')
+    assert s2.locked
+    assert s2.is_encrypted()
+    assert not s2.was_changed()
+    assert s2.data[b'mydata'] == b'test'
+
+
+def test_storage_invalid():
+    with pytest.raises(storage.StorageError, message="File does not exist"):
+        MockStorage(None, 'nonexistant', b'password')
+
+    s = MockStorage(None, 'nonexistant', b'password', create=True)
+    with pytest.raises(storage.StorageError, message="Wrong password"):
+        MockStorage(s.file_data, __file__, b'wrongpass')
+
+    with pytest.raises(storage.StorageError, message="No password"):
+        MockStorage(s.file_data, __file__)
+
+    with pytest.raises(storage.StorageError, message="Non-wallet file, unencrypted"):
+        MockStorage(b'garbagefile', __file__)
+
+    with pytest.raises(storage.StorageError, message="Non-wallet file, encrypted"):
+        MockStorage(b'garbagefile', __file__, b'password')
+
+
+def test_storage_readonly():
+    s = MockStorage(None, 'nonexistant', b'password', create=True)
+    s = MockStorage(s.file_data, __file__, b'password', read_only=True)
+    s.data[b'mydata'] = b'test'
+
+    assert not s.locked
+    assert s.was_changed()
+
+    with pytest.raises(storage.StorageError):
+        s.save()
+
+    with pytest.raises(storage.StorageError):
+        s.change_password(b'newpass')
+
+
+def test_storage_lock(tmpdir):
+    p = str(tmpdir.join('test.jmdat'))
+    pw = None
+
+    with pytest.raises(storage.StorageError, message="File does not exist"):
+        storage.Storage(p, pw)
+
+    s = storage.Storage(p, pw, create=True)
+    assert s.is_locked()
+    assert not s.is_encrypted()
+    assert s.data == {}
+
+    with pytest.raises(storage.StorageError, message="File is locked"):
+        storage.Storage(p, pw)
+
+    assert storage.Storage.is_storage_file(p)
+    assert not storage.Storage.is_encrypted_storage_file(p)
+
+    s.data[b'test'] = b'value'
+    s.save()
+    s.close()
+    del s
+
+    s = storage.Storage(p, pw, read_only=True)
+    assert not s.is_locked()
+    assert s.data == {b'test': b'value'}
+    s.close()
+    del s
+
+    s = storage.Storage(p, pw)
+    assert s.is_locked()
+    assert s.data == {b'test': b'value'}
+

jmclient/test/test_taker.py

@@ -9,26 +9,56 @@ import shutil
 import pytest
 import json
 from base64 import b64encode
-from jmclient import (load_program_config, jm_single, set_commitment_file,
-                      get_commitment_file, AbstractWallet, Taker, SegwitWallet,
-                      get_p2sh_vbyte, get_p2pk_vbyte)
+from jmclient import (
+    load_program_config, jm_single, set_commitment_file, get_commitment_file,
+    SegwitLegacyWallet, Taker, VolatileStorage, get_p2sh_vbyte, get_network)
 from taker_test_data import (t_utxos_by_mixdepth, t_selected_utxos, t_orderbook,
                              t_maker_response, t_chosen_orders, t_dummy_ext)
 
-class DummyWallet(AbstractWallet):
 
+class DummyWallet(SegwitLegacyWallet):
     def __init__(self):
-        super(DummyWallet, self).__init__()
-        self.max_mix_depth = 5
+        storage = VolatileStorage()
+        super(DummyWallet, self).initialize(storage, get_network(),
+                                            max_mixdepth=5)
+        super(DummyWallet, self).__init__(storage)
+        self._add_utxos()
         self.inject_addr_get_failure = False
 
+    def _add_utxos(self):
+        for md, utxo in t_utxos_by_mixdepth.items():
+            for i, (txid, data) in enumerate(utxo.items()):
+                txid, index = txid.split(':')
+                path = (b'dummy', md, i)
+                self._utxos.add_utxo(binascii.unhexlify(txid), int(index),
+                                     path, data['value'], md)
+                script = self._ENGINE.address_to_script(data['address'])
+                self._script_map[script] = path
+
     def get_utxos_by_mixdepth(self, verbose=True):
         return t_utxos_by_mixdepth
 
+    def get_utxos_by_mixdepth_(self, verbose=True):
+        utxos = self.get_utxos_by_mixdepth(verbose)
+
+        utxos_conv = {}
+        for md, utxo_data in utxos.items():
+            md_utxo = utxos_conv.setdefault(md, {})
+            for i, (utxo_hex, data) in enumerate(utxo_data.items()):
+                utxo, index = utxo_hex.split(':')
+                data_conv = {
+                    'script': self._ENGINE.address_to_script(data['address']),
+                    'path': (b'dummy', md, i),
+                    'value': data['value']
+                }
+                md_utxo[(binascii.unhexlify(utxo), int(index))] = data_conv
+
+        return utxos_conv
+
     def select_utxos(self, mixdepth, amount):
         if amount > self.get_balance_by_mixdepth()[mixdepth]:
             raise Exception("Not enough funds")
-        return self.get_utxos_by_mixdepth()[mixdepth]
+        return t_utxos_by_mixdepth[mixdepth]
 
     def get_internal_addr(self, mixing_depth):
         if self.inject_addr_get_failure:
@@ -52,10 +82,6 @@ class DummyWallet(AbstractWallet):
         """
         return 'p2sh-p2wpkh'
 
-    @classmethod
-    def pubkey_to_address(cls, pubkey):
-        return SegwitWallet.pubkey_to_address(pubkey)
-
     def get_key_from_addr(self, addr):
         """usable addresses: privkey all 1s, 2s, 3s, ... :"""
         privs = [x*32 + "\x01" for x in [chr(y) for y in range(1,6)]]
@@ -74,6 +100,10 @@ class DummyWallet(AbstractWallet):
                 return binascii.hexlify(p)
         raise ValueError("No such keypair")
 
+    def _is_my_bip32_path(self, path):
+        return True
+
+
 def dummy_order_chooser():
     return t_chosen_orders
 
@@ -84,7 +114,7 @@ def dummy_filter_orderbook(orders_fees, cjamount):
     print("calling dummy filter orderbook")
     return True
 
-def get_taker(schedule=None, schedule_len=0, sign_method=None, on_finished=None,
+def get_taker(schedule=None, schedule_len=0, on_finished=None,
               filter_orders=None):
     if not schedule:
         #note, for taker.initalize() this will result in junk
@@ -93,8 +123,7 @@ def get_taker(schedule=None, schedule_len=0, sign_method=None, on_finished=None,
     on_finished_callback = on_finished if on_finished else taker_finished
     filter_orders_callback = filter_orders if filter_orders else dummy_filter_orderbook
     return Taker(DummyWallet(), schedule,
-                  callbacks=[filter_orders_callback, None, on_finished_callback],
-                  sign_method=sign_method)
+                  callbacks=[filter_orders_callback, None, on_finished_callback])
 
 def test_filter_rejection(createcmtdata):
     def filter_orders_reject(orders_feesl, cjamount):
@@ -135,22 +164,8 @@ def test_make_commitment(createcmtdata, failquery, external):
     mixdepth = 0
     amount = 110000000
     taker = get_taker([(mixdepth, amount, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw")])
-    taker.wallet.unspent = {'f34b635ed8891f16c4ec5b8236ae86164783903e8e8bb47fa9ef2ca31f3c2d7a:0':
-                            {'address': u'n31WD8pkfAjg2APV78GnbDTdZb1QonBi5D',
-                             'value': 10000000},
-                            'f780d6e5e381bff01a3519997bb4fcba002493103a198fde334fd264f9835d75:1':
-                            {'address': u'mmVEKH61BZbLbnVEmk9VmojreB4G4PmBPd',
-                             'value': 20000000},
-                            'fe574db96a4d43a99786b3ea653cda9e4388f377848f489332577e018380cff1:0':
-                            {'address': u'msxyyydNXTiBmt3SushXbH5Qh2ukBAThk3',
-                             'value': 500000000},
-                            'fd9711a2ef340750db21efb761f5f7d665d94b312332dc354e252c77e9c48349:0':
-                            {'address': u'musGZczug3BAbqobmYherywCwL9REgNaNm',
-                             'value': 500000000}}
     taker.cjamount = amount
-    taker.input_utxos = {'f34b635ed8891f16c4ec5b8236ae86164783903e8e8bb47fa9ef2ca31f3c2d7a:0':
-                            {'address': u'n31WD8pkfAjg2APV78GnbDTdZb1QonBi5D',
-                             'value': 10000000}}
+    taker.input_utxos = t_utxos_by_mixdepth[0]
     if failquery:
         jm_single().bc_interface.setQUSFail(True)
     taker.make_commitment()
@@ -327,8 +342,6 @@ def test_taker_init(createcmtdata, schedule, highfee, toomuchcoins, minmakers,
     taker.my_cj_addr = None
     with pytest.raises(NotImplementedError) as e_info:
         taker.prepare_my_bitcoin_data()
-    with pytest.raises(NotImplementedError) as e_info:
-        taker.sign_tx("a", "b", "c", "d")
     with pytest.raises(NotImplementedError) as e_info:
         a = taker.coinjoin_address()
     taker.wallet.inject_addr_get_failure = True
@@ -364,14 +377,12 @@ def test_unconfirm_confirm(schedule_len):
     assert not test_unconfirm_confirm.txflag
     
 @pytest.mark.parametrize(
-    "dummyaddr, signmethod, schedule",
+    "dummyaddr, schedule",
     [
-        ("mrcNu71ztWjAQA6ww9kHiW3zBWSQidHXTQ", None,
-         [(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)]),
-        ("mrcNu71ztWjAQA6ww9kHiW3zBWSQidHXTQ", "wallet",
-         [(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)]),
+        ("mrcNu71ztWjAQA6ww9kHiW3zBWSQidHXTQ",
+         [(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)])
     ])
-def test_on_sig(createcmtdata, dummyaddr, signmethod, schedule):
+def test_on_sig(createcmtdata, dummyaddr, schedule):
     #plan: create a new transaction with known inputs and dummy outputs;
     #then, create a signature with various inputs, pass in in b64 to on_sig.
     #in order for it to verify, the DummyBlockchainInterface will have to 
@@ -397,7 +408,7 @@ def test_on_sig(createcmtdata, dummyaddr, signmethod, schedule):
     
     de_tx = bitcoin.deserialize(tx)
     #prepare the Taker with the right intermediate data
-    taker = get_taker(schedule=schedule, sign_method=signmethod)
+    taker = get_taker(schedule=schedule)
     taker.nonrespondants=["cp1", "cp2", "cp3"]
     taker.latest_tx = de_tx
     #my inputs are the first 2 utxos
@@ -461,7 +472,3 @@ def createcmtdata(request):
     load_program_config()
     jm_single().bc_interface = DummyBlockchainInterface()
     jm_single().config.set("BLOCKCHAIN", "network", "testnet")
-
-            
-        
-    

jmclient/test/test_tx_creation.py

@@ -3,18 +3,15 @@ from __future__ import absolute_import
 '''Test of unusual transaction types creation and push to
 network to check validity.'''
 
-import sys
-import os
 import time
 import binascii
-import random
 from commontest import make_wallets, make_sign_and_push
 
 import jmbitcoin as bitcoin
 import pytest
-from jmclient import (load_program_config, jm_single, sync_wallet,
-                      get_p2pk_vbyte, get_log, Wallet, select_gradual,
-                      select, select_greedy, select_greediest, estimate_tx_fee)
+from jmclient import (
+    load_program_config, jm_single, sync_wallet, get_p2pk_vbyte, get_log,
+    select_gradual, select, select_greedy, select_greediest, estimate_tx_fee)
 
 log = get_log()
 #just a random selection of pubkeys for receiving multisigs;
@@ -177,15 +174,6 @@ def test_create_sighash_txs(setup_tx_creation):
         txid = make_sign_and_push(ins_full, wallet, amount, hashcode=sighash)
         assert txid
 
-    #Create an invalid sighash single (too many inputs)
-    extra = wallet.select_utxos(4, 100000000)  #just a few more inputs
-    ins_full.update(extra)
-    with pytest.raises(Exception) as e_info:
-        txid = make_sign_and_push(ins_full,
-                                  wallet,
-                                  amount,
-                                  hashcode=bitcoin.SIGHASH_SINGLE)
-
     #trigger insufficient funds
     with pytest.raises(Exception) as e_info:
         fake_utxos = wallet.select_utxos(4, 1000000000)

jmclient/test/test_utxomanager.py

@@ -0,0 +1,93 @@
+from __future__ import print_function, absolute_import, division, unicode_literals
+
+from jmclient.wallet import UTXOManager
+from test_storage import MockStorage
+import pytest
+
+from jmclient import load_program_config
+import jmclient
+from commontest import DummyBlockchainInterface
+
+
+def select(unspent, value):
+    return unspent
+
+
+def test_utxomanager_persist(setup_env_nodeps):
+    storage = MockStorage(None, 'wallet.jmdat', None, create=True)
+    UTXOManager.initialize(storage)
+    um = UTXOManager(storage, select)
+
+    txid = b'\x00' * UTXOManager.TXID_LEN
+    index = 0
+    path = (0,)
+    mixdepth = 0
+    value = 500
+
+    um.add_utxo(txid, index, path, value, mixdepth)
+    um.add_utxo(txid, index+1, path, value, mixdepth+1)
+
+    um.save()
+    del um
+
+    um = UTXOManager(storage, select)
+
+    assert um.have_utxo(txid, index) is mixdepth
+    assert um.have_utxo(txid, index+1) is mixdepth + 1
+    assert um.have_utxo(txid, index+2) is False
+
+    utxos = um.get_utxos_by_mixdepth()
+    assert len(utxos[mixdepth]) == 1
+    assert len(utxos[mixdepth+1]) == 1
+    assert len(utxos[mixdepth+2]) == 0
+
+    balances = um.get_balance_by_mixdepth()
+    assert balances[mixdepth] == value
+    assert balances[mixdepth+1] == value
+
+    um.remove_utxo(txid, index, mixdepth)
+    assert um.have_utxo(txid, index) is False
+
+    um.save()
+    del um
+
+    um = UTXOManager(storage, select)
+
+    assert um.have_utxo(txid, index) is False
+    assert um.have_utxo(txid, index+1) is mixdepth + 1
+
+    utxos = um.get_utxos_by_mixdepth()
+    assert len(utxos[mixdepth]) == 0
+    assert len(utxos[mixdepth+1]) == 1
+
+    balances = um.get_balance_by_mixdepth()
+    assert balances[mixdepth] == 0
+    assert balances[mixdepth+1] == value
+    assert balances[mixdepth+2] == 0
+
+
+def test_utxomanager_select(setup_env_nodeps):
+    storage = MockStorage(None, 'wallet.jmdat', None, create=True)
+    UTXOManager.initialize(storage)
+    um = UTXOManager(storage, select)
+
+    txid = b'\x00' * UTXOManager.TXID_LEN
+    index = 0
+    path = (0,)
+    mixdepth = 0
+    value = 500
+
+    um.add_utxo(txid, index, path, value, mixdepth)
+
+    assert len(um.select_utxos(mixdepth, value)) is 1
+    assert len(um.select_utxos(mixdepth+1, value)) is 0
+
+    um.add_utxo(txid, index+1, path, value, mixdepth)
+    assert len(um.select_utxos(mixdepth, value)) is 2
+
+
+@pytest.fixture
+def setup_env_nodeps(monkeypatch):
+    monkeypatch.setattr(jmclient.configure, 'get_blockchain_interface_instance',
+                        lambda x: DummyBlockchainInterface())
+    load_program_config()

jmclient/test/test_wallet.py

@@ -0,0 +1,590 @@
+from __future__ import print_function, absolute_import, division, unicode_literals
+'''Wallet functionality tests.'''
+
+import os
+import json
+from binascii import hexlify, unhexlify
+
+import pytest
+import jmbitcoin as btc
+from commontest import binarize_tx
+from jmclient import load_program_config, jm_single, get_log,\
+    SegwitLegacyWallet,BIP32Wallet, BIP49Wallet, LegacyWallet,\
+    VolatileStorage, get_network, cryptoengine, WalletError
+
+testdir = os.path.dirname(os.path.realpath(__file__))
+log = get_log()
+
+
+def signed_tx_is_segwit(tx):
+    for inp in tx['ins']:
+        if 'txinwitness' not in inp:
+            return False
+    return True
+
+
+def assert_segwit(tx):
+    assert signed_tx_is_segwit(tx)
+
+
+def assert_not_segwit(tx):
+    assert not signed_tx_is_segwit(tx)
+
+
+def get_populated_wallet(amount=10**8, num=3):
+    storage = VolatileStorage()
+    SegwitLegacyWallet.initialize(storage, get_network())
+    wallet = SegwitLegacyWallet(storage)
+
+    # fund three wallet addresses at mixdepth 0
+    for i in range(num):
+        fund_wallet_addr(wallet, wallet.get_internal_addr(0), amount / 10**8)
+
+    return wallet
+
+
+def fund_wallet_addr(wallet, addr, value_btc=1):
+    txin_id = jm_single().bc_interface.grab_coins(addr, value_btc)
+    txinfo = jm_single().bc_interface.rpc('gettransaction', [txin_id])
+    txin = btc.deserialize(unhexlify(txinfo['hex']))
+    utxo_in = wallet.add_new_utxos_(txin, unhexlify(txin_id))
+    assert len(utxo_in) == 1
+    return list(utxo_in.keys())[0]
+
+
+def get_bip39_vectors():
+    fh = open(os.path.join(testdir, 'bip39vectors.json'))
+    data = json.load(fh)['english']
+    fh.close()
+    return data
+
+
+@pytest.mark.parametrize('entropy,mnemonic,key,xpriv', get_bip39_vectors())
+def test_bip39_seeds(monkeypatch, setup_wallet, entropy, mnemonic, key, xpriv):
+    jm_single().config.set('BLOCKCHAIN', 'network', 'mainnet')
+    created_entropy = SegwitLegacyWallet.entropy_from_mnemonic(mnemonic)
+    assert entropy == hexlify(created_entropy)
+    storage = VolatileStorage()
+    SegwitLegacyWallet.initialize(
+        storage, get_network(), entropy=created_entropy,
+        entropy_extension=b'TREZOR', max_mixdepth=4)
+    wallet = SegwitLegacyWallet(storage)
+    assert (mnemonic, b'TREZOR') == wallet.get_mnemonic_words()
+    assert key == hexlify(wallet._create_master_key())
+
+    # need to monkeypatch this, else we'll default to the BIP-49 path
+    monkeypatch.setattr(SegwitLegacyWallet, '_get_bip32_base_path',
+                        BIP32Wallet._get_bip32_base_path)
+    assert xpriv == wallet.get_bip32_priv_export()
+
+
+def test_bip49_seed(monkeypatch, setup_wallet):
+    jm_single().config.set('BLOCKCHAIN', 'network', 'testnet')
+    mnemonic = 'abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about'
+    master_xpriv = 'tprv8ZgxMBicQKsPe5YMU9gHen4Ez3ApihUfykaqUorj9t6FDqy3nP6eoXiAo2ssvpAjoLroQxHqr3R5nE3a5dU3DHTjTgJDd7zrbniJr6nrCzd'
+    account0_xpriv = 'tprv8gRrNu65W2Msef2BdBSUgFdRTGzC8EwVXnV7UGS3faeXtuMVtGfEdidVeGbThs4ELEoayCAzZQ4uUji9DUiAs7erdVskqju7hrBcDvDsdbY'
+    addr0_script_hash = '336caa13e08b96080a32b5d818d59b4ab3b36742'
+
+    entropy = SegwitLegacyWallet.entropy_from_mnemonic(mnemonic)
+    storage = VolatileStorage()
+    SegwitLegacyWallet.initialize(
+        storage, get_network(), entropy=entropy, max_mixdepth=0)
+    wallet = SegwitLegacyWallet(storage)
+    assert (mnemonic, None) == wallet.get_mnemonic_words()
+    assert account0_xpriv == wallet.get_bip32_priv_export(0)
+    assert addr0_script_hash == hexlify(wallet.get_external_script(0)[2:-1])
+
+    # FIXME: is this desired behaviour? BIP49 wallet will not return xpriv for
+    # the root key but only for key after base path
+    monkeypatch.setattr(SegwitLegacyWallet, '_get_bip32_base_path',
+                        BIP32Wallet._get_bip32_base_path)
+    assert master_xpriv == wallet.get_bip32_priv_export()
+
+
+def test_bip32_test_vector_1(monkeypatch, setup_wallet):
+    jm_single().config.set('BLOCKCHAIN', 'network', 'mainnet')
+
+    entropy = unhexlify('000102030405060708090a0b0c0d0e0f')
+    storage = VolatileStorage()
+    LegacyWallet.initialize(
+        storage, get_network(), entropy=entropy, max_mixdepth=0)
+
+    # test vector 1 is using hardened derivation for the account/mixdepth level
+    monkeypatch.setattr(LegacyWallet, '_get_mixdepth_from_path',
+                        BIP49Wallet._get_mixdepth_from_path)
+    monkeypatch.setattr(LegacyWallet, '_get_bip32_mixdepth_path_level',
+                        BIP49Wallet._get_bip32_mixdepth_path_level)
+    monkeypatch.setattr(LegacyWallet, '_get_bip32_base_path',
+                        BIP32Wallet._get_bip32_base_path)
+    monkeypatch.setattr(LegacyWallet, '_create_master_key',
+                        BIP32Wallet._create_master_key)
+
+    wallet = LegacyWallet(storage)
+
+    assert wallet.get_bip32_priv_export() == 'xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi'
+    assert wallet.get_bip32_pub_export() == 'xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8'
+    assert wallet.get_bip32_priv_export(0) == 'xprv9uHRZZhk6KAJC1avXpDAp4MDc3sQKNxDiPvvkX8Br5ngLNv1TxvUxt4cV1rGL5hj6KCesnDYUhd7oWgT11eZG7XnxHrnYeSvkzY7d2bhkJ7'
+    assert wallet.get_bip32_pub_export(0) == 'xpub68Gmy5EdvgibQVfPdqkBBCHxA5htiqg55crXYuXoQRKfDBFA1WEjWgP6LHhwBZeNK1VTsfTFUHCdrfp1bgwQ9xv5ski8PX9rL2dZXvgGDnw'
+    assert wallet.get_bip32_priv_export(0, 1) == 'xprv9wTYmMFdV23N2TdNG573QoEsfRrWKQgWeibmLntzniatZvR9BmLnvSxqu53Kw1UmYPxLgboyZQaXwTCg8MSY3H2EU4pWcQDnRnrVA1xe8fs'
+    assert wallet.get_bip32_pub_export(0, 1) == 'xpub6ASuArnXKPbfEwhqN6e3mwBcDTgzisQN1wXN9BJcM47sSikHjJf3UFHKkNAWbWMiGj7Wf5uMash7SyYq527Hqck2AxYysAA7xmALppuCkwQ'
+    # there are more test vectors but those don't match joinmarket's wallet
+    # structure, hence they make litte sense to test here
+
+
+def test_bip32_test_vector_2(monkeypatch, setup_wallet):
+    jm_single().config.set('BLOCKCHAIN', 'network', 'mainnet')
+
+    entropy = unhexlify('fffcf9f6f3f0edeae7e4e1dedbd8d5d2cfccc9c6c3c0bdbab7b4b1aeaba8a5a29f9c999693908d8a8784817e7b7875726f6c696663605d5a5754514e4b484542')
+    storage = VolatileStorage()
+    LegacyWallet.initialize(
+        storage, get_network(), entropy=entropy, max_mixdepth=0)
+
+    monkeypatch.setattr(LegacyWallet, '_get_bip32_base_path',
+                        BIP32Wallet._get_bip32_base_path)
+    monkeypatch.setattr(LegacyWallet, '_create_master_key',
+                        BIP32Wallet._create_master_key)
+
+    wallet = LegacyWallet(storage)
+
+    assert wallet.get_bip32_priv_export() == 'xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U'
+    assert wallet.get_bip32_pub_export() == 'xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB'
+    assert wallet.get_bip32_priv_export(0) == 'xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt'
+    assert wallet.get_bip32_pub_export(0) == 'xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH'
+    # there are more test vectors but those don't match joinmarket's wallet
+    # structure, hence they make litte sense to test here
+
+
+def test_bip32_test_vector_3(monkeypatch, setup_wallet):
+    jm_single().config.set('BLOCKCHAIN', 'network', 'mainnet')
+
+    entropy = unhexlify('4b381541583be4423346c643850da4b320e46a87ae3d2a4e6da11eba819cd4acba45d239319ac14f863b8d5ab5a0d0c64d2e8a1e7d1457df2e5a3c51c73235be')
+    storage = VolatileStorage()
+    LegacyWallet.initialize(
+        storage, get_network(), entropy=entropy, max_mixdepth=0)
+
+    # test vector 3 is using hardened derivation for the account/mixdepth level
+    monkeypatch.setattr(LegacyWallet, '_get_mixdepth_from_path',
+                        BIP49Wallet._get_mixdepth_from_path)
+    monkeypatch.setattr(LegacyWallet, '_get_bip32_mixdepth_path_level',
+                        BIP49Wallet._get_bip32_mixdepth_path_level)
+    monkeypatch.setattr(LegacyWallet, '_get_bip32_base_path',
+                        BIP32Wallet._get_bip32_base_path)
+    monkeypatch.setattr(LegacyWallet, '_create_master_key',
+                        BIP32Wallet._create_master_key)
+
+    wallet = LegacyWallet(storage)
+
+    assert wallet.get_bip32_priv_export() == 'xprv9s21ZrQH143K25QhxbucbDDuQ4naNntJRi4KUfWT7xo4EKsHt2QJDu7KXp1A3u7Bi1j8ph3EGsZ9Xvz9dGuVrtHHs7pXeTzjuxBrCmmhgC6'
+    assert wallet.get_bip32_pub_export() == 'xpub661MyMwAqRbcEZVB4dScxMAdx6d4nFc9nvyvH3v4gJL378CSRZiYmhRoP7mBy6gSPSCYk6SzXPTf3ND1cZAceL7SfJ1Z3GC8vBgp2epUt13'
+    assert wallet.get_bip32_priv_export(0) == 'xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L'
+    assert wallet.get_bip32_pub_export(0) == 'xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y'
+
+
+@pytest.mark.parametrize('mixdepth,internal,index,address,wif', [
+    [0, 0, 0, 'mpCX9EbdXpcrKMtjEe1fqFhvzctkfzMYTX', 'cVqtSSoVxFyPqTRGfeESi31uCYfgTF4tGWRtGeVs84fzybiX5TPk'],
+    [0, 0, 5, 'mtj85a3pFppRhrxNcFig1k7ECshrZjJ9XC', 'cMsFXc4TRw9PTcCTv7x9mr88rDeGXBTLEV67mKaw2cxCkjkhL32G'],
+    [0, 1, 3, 'n1EaQuqvTRm719hsSJ7yRsj49JfoG1C86q', 'cUgSTqnAtvYoQRXCYy4wCFfaks2Zrz1d55m6mVhFyVhQbkDi7JGJ'],
+    [2, 1, 2, 'mfxkBk7uDhmF5PJGS9d1NonGiAxPwJqQP4', 'cPcZXSiXPuS5eiT4oDrDKi1mFumw5D1RcWzK2gkGdEHjEz99eyXn']
+])
+def test_bip32_addresses_p2pkh(monkeypatch, setup_wallet, mixdepth, internal, index, address, wif):
+    """
+    Test with a random but fixed entropy
+    """
+    jm_single().config.set('BLOCKCHAIN', 'network', 'testnet')
+
+    entropy = unhexlify('2e0339ba89b4a1272cdf78b27ee62669ee01992a59e836e2807051be128ca817')
+    storage = VolatileStorage()
+    LegacyWallet.initialize(
+        storage, get_network(), entropy=entropy, max_mixdepth=3)
+
+    monkeypatch.setattr(LegacyWallet, '_get_bip32_base_path',
+                        BIP32Wallet._get_bip32_base_path)
+    monkeypatch.setattr(LegacyWallet, '_create_master_key',
+                        BIP32Wallet._create_master_key)
+
+    wallet = LegacyWallet(storage)
+
+    # wallet needs to know about all intermediate keys
+    for i in range(index + 1):
+        wallet.get_new_script(mixdepth, internal)
+
+    assert wif == wallet.get_wif(mixdepth, internal, index)
+    assert address == wallet.get_addr(mixdepth, internal, index)
+
+
+@pytest.mark.parametrize('mixdepth,internal,index,address,wif', [
+    [0, 0, 0, '2MzY5yyonUY7zpHspg7jB7WQs1uJxKafQe4', 'cRAGLvPmhpzJNgdMT4W2gVwEW3fusfaDqdQWM2vnWLgXKzCWKtcM'],
+    [0, 0, 5, '2MsKvqPGStp3yXT8UivuAaGwfPzT7xYwSWk', 'cSo3h7nRuV4fwhVPXeTDJx6cBCkjAzS9VM8APXViyjoSaMq85ZKn'],
+    [0, 1, 3, '2N7k6wiQqkuMaApwGhk3HKrifprUSDydqUv', 'cTwq3UsZa8STVmwZR94dDphgqgdLFeuaRFD1Ea44qjbjFfKEb1n5'],
+    [2, 1, 2, '2MtE6gzHgmEXeWzKsmCJFEqkrpNuBDvoRnz', 'cPV8FZuCvrRpk4RhmhpjnSucHhaQZUan4Vbyo1NVQtuAxurW9grb']
+])
+def test_bip32_addresses_p2sh_p2wpkh(setup_wallet, mixdepth, internal, index, address, wif):
+    """
+    Test with a random but fixed entropy
+    """
+    jm_single().config.set('BLOCKCHAIN', 'network', 'testnet')
+
+    entropy = unhexlify('2e0339ba89b4a1272cdf78b27ee62669ee01992a59e836e2807051be128ca817')
+    storage = VolatileStorage()
+    SegwitLegacyWallet.initialize(
+        storage, get_network(), entropy=entropy, max_mixdepth=3)
+    wallet = SegwitLegacyWallet(storage)
+
+    # wallet needs to know about all intermediate keys
+    for i in range(index + 1):
+        wallet.get_new_script(mixdepth, internal)
+
+    assert wif == wallet.get_wif(mixdepth, internal, index)
+    assert address == wallet.get_addr(mixdepth, internal, index)
+
+
+def test_import_key(setup_wallet):
+    jm_single().config.set('BLOCKCHAIN', 'network', 'testnet')
+    storage = VolatileStorage()
+    SegwitLegacyWallet.initialize(storage, get_network())
+    wallet = SegwitLegacyWallet(storage)
+
+    wallet.import_private_key(
+        0, 'cRAGLvPmhpzJNgdMT4W2gVwEW3fusfaDqdQWM2vnWLgXKzCWKtcM',
+        cryptoengine.TYPE_P2SH_P2WPKH)
+    wallet.import_private_key(
+        1, 'cVqtSSoVxFyPqTRGfeESi31uCYfgTF4tGWRtGeVs84fzybiX5TPk',
+        cryptoengine.TYPE_P2PKH)
+
+    with pytest.raises(WalletError):
+        wallet.import_private_key(
+            1, 'cRAGLvPmhpzJNgdMT4W2gVwEW3fusfaDqdQWM2vnWLgXKzCWKtcM',
+            cryptoengine.TYPE_P2SH_P2WPKH)
+
+    # test persist imported keys
+    wallet.save()
+    data = storage.file_data
+
+    del wallet
+    del storage
+
+    storage = VolatileStorage(data=data)
+    wallet = SegwitLegacyWallet(storage)
+
+    imported_paths_md0 = list(wallet.yield_imported_paths(0))
+    imported_paths_md1 = list(wallet.yield_imported_paths(1))
+    assert len(imported_paths_md0) == 1
+    assert len(imported_paths_md1) == 1
+
+    # verify imported addresses
+    assert wallet.get_addr_path(imported_paths_md0[0]) == '2MzY5yyonUY7zpHspg7jB7WQs1uJxKafQe4'
+    assert wallet.get_addr_path(imported_paths_md1[0]) == 'mpCX9EbdXpcrKMtjEe1fqFhvzctkfzMYTX'
+
+    # test remove key
+    wallet.remove_imported_key(path=imported_paths_md0[0])
+    assert not list(wallet.yield_imported_paths(0))
+
+    assert wallet.get_details(imported_paths_md1[0]) == (1, 'imported', 0)
+
+
+@pytest.mark.parametrize('wif,keytype,type_check', [
+    ['cVqtSSoVxFyPqTRGfeESi31uCYfgTF4tGWRtGeVs84fzybiX5TPk',
+     cryptoengine.TYPE_P2PKH, assert_not_segwit],
+    ['cRAGLvPmhpzJNgdMT4W2gVwEW3fusfaDqdQWM2vnWLgXKzCWKtcM',
+     cryptoengine.TYPE_P2SH_P2WPKH, assert_segwit]
+])
+def test_signing_imported(setup_wallet, wif, keytype, type_check):
+    jm_single().config.set('BLOCKCHAIN', 'network', 'testnet')
+    storage = VolatileStorage()
+    SegwitLegacyWallet.initialize(storage, get_network())
+    wallet = SegwitLegacyWallet(storage)
+
+    MIXDEPTH = 0
+    path = wallet.import_private_key(MIXDEPTH, wif, keytype)
+    utxo = fund_wallet_addr(wallet, wallet.get_addr_path(path))
+    tx = btc.deserialize(btc.mktx(['{}:{}'.format(hexlify(utxo[0]), utxo[1])],
+                                  ['00'*17 + ':' + str(10**8 - 9000)]))
+    binarize_tx(tx)
+    script = wallet.get_script_path(path)
+    wallet.sign_tx(tx, {0: (script, 10**8)})
+    type_check(tx)
+    txout = jm_single().bc_interface.pushtx(hexlify(btc.serialize(tx)))
+    assert txout
+
+
+@pytest.mark.parametrize('wallet_cls,type_check', [
+    [LegacyWallet, assert_not_segwit],
+    [SegwitLegacyWallet, assert_segwit]
+])
+def test_signing_simple(setup_wallet, wallet_cls, type_check):
+    jm_single().config.set('BLOCKCHAIN', 'network', 'testnet')
+    storage = VolatileStorage()
+    wallet_cls.initialize(storage, get_network())
+    wallet = wallet_cls(storage)
+    utxo = fund_wallet_addr(wallet, wallet.get_internal_addr(0))
+    tx = btc.deserialize(btc.mktx(['{}:{}'.format(hexlify(utxo[0]), utxo[1])],
+                                  ['00'*17 + ':' + str(10**8 - 9000)]))
+    binarize_tx(tx)
+    script = wallet.get_script(0, 1, 0)
+    wallet.sign_tx(tx, {0: (script, 10**8)})
+    type_check(tx)
+    txout = jm_single().bc_interface.pushtx(hexlify(btc.serialize(tx)))
+    assert txout
+
+
+def test_add_utxos(setup_wallet):
+    jm_single().config.set('BLOCKCHAIN', 'network', 'testnet')
+    amount = 10**8
+    num_tx = 3
+
+    wallet = get_populated_wallet(amount, num_tx)
+
+    balances = wallet.get_balance_by_mixdepth()
+    assert balances[0] == num_tx * amount
+    for md in range(1, wallet.max_mixdepth + 1):
+        assert balances[md] == 0
+
+    utxos = wallet.get_utxos_by_mixdepth_()
+    assert len(utxos[0]) == num_tx
+    for md in range(1, wallet.max_mixdepth + 1):
+        assert not utxos[md]
+
+    with pytest.raises(Exception):
+        # no funds in mixdepth
+        wallet.select_utxos_(1, amount)
+
+    with pytest.raises(Exception):
+        # not enough funds
+        wallet.select_utxos_(0, amount * (num_tx + 1))
+
+    wallet.reset_utxos()
+    assert wallet.get_balance_by_mixdepth()[0] == 0
+
+
+def test_select_utxos(setup_wallet):
+    jm_single().config.set('BLOCKCHAIN', 'network', 'testnet')
+    amount = 10**8
+
+    wallet = get_populated_wallet(amount)
+    utxos = wallet.select_utxos_(0, amount // 2)
+
+    assert len(utxos) == 1
+    utxos = list(utxos.keys())
+
+    more_utxos = wallet.select_utxos_(0, int(amount * 1.5), utxo_filter=utxos)
+    assert len(more_utxos) == 2
+    assert utxos[0] not in more_utxos
+
+
+def test_add_new_utxos(setup_wallet):
+    jm_single().config.set('BLOCKCHAIN', 'network', 'testnet')
+    wallet = get_populated_wallet(num=1)
+
+    scripts = [wallet.get_new_script(x, True) for x in range(3)]
+    tx_scripts = list(scripts)
+    tx_scripts.append(b'\x22'*17)
+
+    tx = btc.deserialize(btc.mktx(
+        ['0'*64 + ':2'], [{'script': hexlify(s), 'value': 10**8}
+                          for s in tx_scripts]))
+    binarize_tx(tx)
+    txid = b'\x01' * 32
+    added = wallet.add_new_utxos_(tx, txid)
+    assert len(added) == len(scripts)
+
+    added_scripts = {x['script'] for x in added.values()}
+    for s in scripts:
+        assert s in added_scripts
+
+    balances = wallet.get_balance_by_mixdepth()
+    assert balances[0] == 2 * 10**8
+    assert balances[1] == 10**8
+    assert balances[2] == 10**8
+    assert len(balances) == wallet.max_mixdepth + 1
+
+
+def test_remove_old_utxos(setup_wallet):
+    jm_single().config.set('BLOCKCHAIN', 'network', 'testnet')
+    wallet = get_populated_wallet()
+
+    # add some more utxos to mixdepth 1
+    for i in range(3):
+        txin = jm_single().bc_interface.grab_coins(
+            wallet.get_internal_addr(1), 1)
+        wallet.add_utxo(unhexlify(txin), 0, wallet.get_script(1, 1, i), 10**8)
+
+    inputs = wallet.select_utxos_(0, 10**8)
+    inputs.update(wallet.select_utxos_(1, 2 * 10**8))
+    assert len(inputs) == 3
+
+    tx_inputs = list(inputs.keys())
+    tx_inputs.append((b'\x12'*32, 6))
+
+    tx = btc.deserialize(btc.mktx(
+        ['{}:{}'.format(hexlify(txid), i) for txid, i in tx_inputs],
+        ['0' * 36 + ':' + str(3 * 10**8 - 1000)]))
+    binarize_tx(tx)
+
+    removed = wallet.remove_old_utxos_(tx)
+    assert len(removed) == len(inputs)
+
+    for txid in removed:
+        assert txid in inputs
+
+    balances = wallet.get_balance_by_mixdepth()
+    assert balances[0] == 2 * 10**8
+    assert balances[1] == 10**8
+    assert balances[2] == 0
+    assert len(balances) == wallet.max_mixdepth + 1
+
+
+def test_initialize_twice(setup_wallet):
+    wallet = get_populated_wallet(num=0)
+    storage = wallet._storage
+    with pytest.raises(WalletError):
+        SegwitLegacyWallet.initialize(storage, get_network())
+
+
+def test_is_known(setup_wallet):
+    wallet = get_populated_wallet(num=0)
+    script = wallet.get_new_script(1, True)
+    addr = wallet.get_new_addr(2, False)
+
+    assert wallet.is_known_script(script)
+    assert wallet.is_known_addr(addr)
+    assert wallet.is_known_addr(wallet.script_to_addr(script))
+    assert wallet.is_known_script(wallet.addr_to_script(addr))
+
+    assert not wallet.is_known_script(b'\x12' * len(script))
+    assert not wallet.is_known_addr('2MzY5yyonUY7zpHspg7jB7WQs1uJxKafQe4')
+
+
+def test_wallet_save(setup_wallet):
+    wallet = get_populated_wallet()
+
+    script = wallet.get_external_script(1)
+
+    wallet.save()
+    storage = wallet._storage
+    data = storage.file_data
+
+    del wallet
+    del storage
+
+    storage = VolatileStorage(data=data)
+    wallet = SegwitLegacyWallet(storage)
+
+    assert wallet.get_next_unused_index(0, True) == 3
+    assert wallet.get_next_unused_index(0, False) == 0
+    assert wallet.get_next_unused_index(1, True) == 0
+    assert wallet.get_next_unused_index(1, False) == 1
+    assert wallet.is_known_script(script)
+
+
+def test_set_next_index(setup_wallet):
+    wallet = get_populated_wallet()
+
+    assert wallet.get_next_unused_index(0, True) == 3
+
+    with pytest.raises(Exception):
+        # cannot advance index without force=True
+        wallet.set_next_index(0, True, 5)
+
+    wallet.set_next_index(0, True, 1)
+    assert wallet.get_next_unused_index(0, True) == 1
+
+    wallet.set_next_index(0, True, 20, force=True)
+    assert wallet.get_next_unused_index(0, True) == 20
+
+    script = wallet.get_new_script(0, True)
+    path = wallet.script_to_path(script)
+    index = wallet.get_details(path)[2]
+    assert index == 20
+
+
+def test_path_repr(setup_wallet):
+    wallet = get_populated_wallet()
+    path = wallet.get_path(2, False, 0)
+    path_repr = wallet.get_path_repr(path)
+    path_new = wallet.path_repr_to_path(path_repr)
+
+    assert path_new == path
+
+
+def test_path_repr_imported(setup_wallet):
+    wallet = get_populated_wallet(num=0)
+    path = wallet.import_private_key(
+        0, 'cRAGLvPmhpzJNgdMT4W2gVwEW3fusfaDqdQWM2vnWLgXKzCWKtcM',
+        cryptoengine.TYPE_P2SH_P2WPKH)
+    path_repr = wallet.get_path_repr(path)
+    path_new = wallet.path_repr_to_path(path_repr)
+
+    assert path_new == path
+
+
+def test_wrong_wallet_cls(setup_wallet):
+    storage = VolatileStorage()
+    SegwitLegacyWallet.initialize(storage, get_network())
+    wallet = SegwitLegacyWallet(storage)
+
+    wallet.save()
+    data = storage.file_data
+
+    del wallet
+    del storage
+
+    storage = VolatileStorage(data=data)
+
+    with pytest.raises(Exception):
+        LegacyWallet(storage)
+
+
+def test_wallet_id(setup_wallet):
+    storage1 = VolatileStorage()
+    SegwitLegacyWallet.initialize(storage1, get_network())
+    wallet1 = SegwitLegacyWallet(storage1)
+
+    storage2 = VolatileStorage()
+    LegacyWallet.initialize(storage2, get_network(), entropy=wallet1._entropy)
+    wallet2 = LegacyWallet(storage2)
+
+    assert wallet1.get_wallet_id() != wallet2.get_wallet_id()
+
+    storage2 = VolatileStorage()
+    SegwitLegacyWallet.initialize(storage2, get_network(),
+                                  entropy=wallet1._entropy)
+    wallet2 = SegwitLegacyWallet(storage2)
+
+    assert wallet1.get_wallet_id() == wallet2.get_wallet_id()
+
+
+def test_addr_script_conversion(setup_wallet):
+    wallet = get_populated_wallet(num=1)
+
+    path = wallet.get_path(0, True, 0)
+    script = wallet.get_script_path(path)
+    addr = wallet.script_to_addr(script)
+
+    assert script == wallet.addr_to_script(addr)
+    addr_path = wallet.addr_to_path(addr)
+    assert path == addr_path
+
+
+def test_imported_key_removed(setup_wallet):
+    wif = 'cRAGLvPmhpzJNgdMT4W2gVwEW3fusfaDqdQWM2vnWLgXKzCWKtcM'
+    key_type = cryptoengine.TYPE_P2SH_P2WPKH
+
+    storage = VolatileStorage()
+    SegwitLegacyWallet.initialize(storage, get_network())
+    wallet = SegwitLegacyWallet(storage)
+
+    path = wallet.import_private_key(1, wif, key_type)
+    script = wallet.get_script_path(path)
+    assert wallet.is_known_script(script)
+
+    wallet.remove_imported_key(path=path)
+    assert not wallet.is_known_script(script)
+
+    with pytest.raises(WalletError):
+        wallet.get_script_path(path)
+
+
+@pytest.fixture(scope='module')
+def setup_wallet():
+    load_program_config()
+    jm_single().bc_interface.tick_forward_chain_interval = 2

jmclient/test/test_wallets.py

@@ -6,27 +6,15 @@ import sys
 import os
 import time
 import binascii
-import random
-import subprocess
-import datetime
-import unittest
 from mnemonic import Mnemonic
-from ConfigParser import SafeConfigParser, NoSectionError
-from decimal import Decimal
-from commontest import (interact, make_wallets,
-                        make_sign_and_push)
+from commontest import create_wallet_for_sync, make_sign_and_push
 import json
 
-import jmbitcoin as bitcoin
 import pytest
-from jmclient import (load_program_config, jm_single, sync_wallet,
-                      AbstractWallet, get_p2pk_vbyte, get_log, Wallet, select,
-                      select_gradual, select_greedy, select_greediest,
-                      estimate_tx_fee, encryptData, get_network, WalletError,
-                      BitcoinCoreWallet, BitcoinCoreInterface, SegwitWallet,
-                      wallet_generate_recover_bip39, decryptData, encryptData)
-from jmbase.support import chunks
-from taker_test_data import t_obtained_tx, t_raw_signed_tx
+from jmclient import (
+    load_program_config, jm_single, sync_wallet, get_log,
+    estimate_tx_fee, BitcoinCoreInterface)
+from taker_test_data import t_raw_signed_tx
 testdir = os.path.dirname(os.path.realpath(__file__))
 log = get_log()
 
@@ -51,8 +39,7 @@ def do_tx(wallet, amount):
 def test_query_utxo_set(setup_wallets):
     load_program_config()
     jm_single().bc_interface.tick_forward_chain_interval = 1
-    wallet = create_wallet_for_sync("wallet4utxo.json", "4utxo",
-                                    [2, 3, 0, 0, 0],
+    wallet = create_wallet_for_sync([2, 3, 0, 0, 0],
                                     ["wallet4utxo.json", "4utxo", [2, 3]])
     sync_wallet(wallet, fast=True)
     txid = do_tx(wallet, 90000000)
@@ -72,212 +59,9 @@ def test_query_utxo_set(setup_wallets):
     assert res3 == [None]
 
 
-def create_wallet_for_sync(wallet_file, password, wallet_structure, a):
-    #Prepare a testnet wallet file for this wallet
-    password_key = bitcoin.bin_dbl_sha256(password)
-    #We need a distinct seed for each run so as not to step over each other;
-    #make it through a deterministic hash
-    seedh = bitcoin.sha256("".join([str(x) for x in a]))[:32]
-    encrypted_seed = encryptData(password_key, seedh.decode('hex'))
-    timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")
-    walletfilejson = {'creator': 'joinmarket project',
-                      'creation_time': timestamp,
-                      'encrypted_seed': encrypted_seed.encode('hex'),
-                      'network': get_network()}
-    walletfile = json.dumps(walletfilejson)
-    if not os.path.exists('wallets'):
-        os.makedirs('wallets')
-    with open(os.path.join('wallets', wallet_file), "wb") as f:
-        f.write(walletfile)
-    #The call to Wallet() in make_wallets should now find the file
-    #and read from it:
-    return make_wallets(1,
-                        [wallet_structure],
-                        fixed_seeds=[wallet_file],
-                        test_wallet=True,
-                        passwords=[password])[0]['wallet']
-
-
-@pytest.mark.parametrize(
-    "num_txs, fake_count, wallet_structure, amount, wallet_file, password",
-    [
-        (3, 13, [11, 3, 4, 5, 6], 150000000, 'test_import_wallet.json',
-         'import-pwd'),
-        #Uncomment all these for thorough tests. Passing currently.
-        #Lots of used addresses
-        #(7, 1, [51, 3, 4, 5, 6], 150000000, 'test_import_wallet.json',
-        # 'import-pwd'),
-        #(3, 1, [3, 1, 4, 5, 6], 50000000, 'test_import_wallet.json',
-        # 'import-pwd'),
-        #No spams/fakes
-        #(2, 0, [5, 20, 1, 1, 1], 50000000, 'test_import_wallet.json',
-        # 'import-pwd'),
-        #Lots of transactions and fakes
-        #(25, 30, [30, 20, 1, 1, 1], 50000000, 'test_import_wallet.json',
-        # 'import-pwd'),
-    ])
-def test_wallet_sync_with_fast(setup_wallets, num_txs, fake_count,
-                               wallet_structure, amount, wallet_file, password):
-    jm_single().bc_interface.tick_forward_chain_interval = 1
-    wallet = create_wallet_for_sync(wallet_file, password, wallet_structure,
-                                    [num_txs, fake_count, wallet_structure,
-                                     amount, wallet_file, password])
-    sync_count = 0
-    jm_single().bc_interface.wallet_synced = False
-    while not jm_single().bc_interface.wallet_synced:
-        sync_wallet(wallet)
-        sync_count += 1
-        #avoid infinite loop
-        assert sync_count < 10
-        log.debug("Tried " + str(sync_count) + " times")
-
-    assert jm_single().bc_interface.wallet_synced
-    assert not jm_single().bc_interface.fast_sync_called
-    #do some transactions with the wallet, then close, then resync
-    for i in range(num_txs):
-        do_tx(wallet, amount)
-        log.debug("After doing a tx, index is now: " + str(wallet.index))
-        #simulate a spammer requesting a bunch of transactions. This
-        #mimics what happens in CoinJoinOrder.__init__()
-        for j in range(fake_count):
-            #Note that as in a real script run,
-            #the initial call to sync_wallet will
-            #have set wallet_synced to True, so these will
-            #trigger actual imports.
-            cj_addr = wallet.get_internal_addr(0)
-            change_addr = wallet.get_internal_addr(0)
-            wallet.update_cache_index()
-            log.debug("After doing a spam, index is now: " + str(wallet.index))
-
-    assert wallet.index[0][1] == num_txs + fake_count * 2 * num_txs
-
-    #Attempt re-sync, simulating a script restart.
-
-    jm_single().bc_interface.wallet_synced = False
-    sync_count = 0
-    #Probably should be fixed in main code:
-    #wallet.index_cache is only assigned in Wallet.__init__(),
-    #meaning a second sync in the same script, after some transactions,
-    #will not know about the latest index_cache value (see is_index_ahead_of_cache),
-    #whereas a real re-sync will involve reading the cache from disk.
-    #Hence, simulation of the fact that the cache index will
-    #be read from the file on restart:
-    wallet.index_cache = wallet.index
-
-    while not jm_single().bc_interface.wallet_synced:
-        #Wallet.__init__() resets index to zero.
-        wallet.index = []
-        for i in range(5):
-            wallet.index.append([0, 0])
-        #Wallet.__init__() also updates the cache index
-        #from file, but we can reuse from the above pre-loop setting,
-        #since nothing else in sync will overwrite the cache.
-
-        #for regtest add_watchonly_addresses does not exit(), so can
-        #just repeat as many times as possible. This might
-        #be usable for non-test code (i.e. no need to restart the
-        #script over and over again)?
-        sync_count += 1
-        log.debug("TRYING SYNC NUMBER: " + str(sync_count))
-        sync_wallet(wallet, fast=True)
-        assert jm_single().bc_interface.fast_sync_called
-        #avoid infinite loop on failure.
-        assert sync_count < 10
-    #Wallet should recognize index_cache on fast sync, so should not need to
-    #run sync process more than once.
-    assert sync_count == 1
-    #validate the wallet index values after sync
-    for i, ws in enumerate(wallet_structure):
-        assert wallet.index[i][0] == ws  #spends into external only
-    #Same number as above; note it includes the spammer's extras.
-    assert wallet.index[0][1] == num_txs + fake_count * 2 * num_txs
-    assert wallet.index[1][1] == num_txs  #one change per transaction
-    for i in range(2, 5):
-        assert wallet.index[i][1] == 0  #unused
-
-    #Now try to do more transactions as sanity check.
-    do_tx(wallet, 50000000)
-
-
-@pytest.mark.parametrize(
-    "wallet_structure, wallet_file, password, ic",
-    [
-        #As usual, more test cases are preferable but time
-        #of build test is too long, so only one activated.
-        #([11,3,4,5,6], 'test_import_wallet.json', 'import-pwd',
-        # [(12,3),(100,99),(7, 40), (200, 201), (10,0)]
-        # ),
-        ([1, 3, 0, 2, 9], 'test_import_wallet.json', 'import-pwd',
-         [(1, 7), (100, 99), (0, 0), (200, 201), (21, 41)]),
-    ])
-def test_wallet_sync_from_scratch(setup_wallets, wallet_structure, wallet_file,
-                                  password, ic):
-    """Simulate a scenario in which we use a new bitcoind, thusly:
-    generate a new wallet and simply pretend that it has an existing
-    index_cache. This will force import of all addresses up to
-    the index_cache values.
-    """
-    wallet = create_wallet_for_sync(wallet_file, password, wallet_structure,
-                                    [wallet_structure, wallet_file, password,
-                                     ic])
-    sync_count = 0
-    jm_single().bc_interface.wallet_synced = False
-    wallet.index_cache = ic
-    while not jm_single().bc_interface.wallet_synced:
-        wallet.index = []
-        for i in range(5):
-            wallet.index.append([0, 0])
-        #will call with fast=False but index_cache exists; should use slow-sync
-        sync_wallet(wallet)
-        sync_count += 1
-        #avoid infinite loop
-        assert sync_count < 10
-        log.debug("Tried " + str(sync_count) + " times")
-    #after #586 we expect to ALWAYS succeed within 2 rounds
-    assert sync_count <= 2
-    #for each external branch, the new index may be higher than
-    #the original index_cache if there was a higher used address
-    expected_wallet_index = []
-    for i, val in enumerate(wallet_structure):
-        if val > wallet.index_cache[i][0]:
-            expected_wallet_index.append([val, wallet.index_cache[i][1]])
-        else:
-            expected_wallet_index.append([wallet.index_cache[i][0],
-                                          wallet.index_cache[i][1]])
-    assert wallet.index == expected_wallet_index
-    log.debug("This is wallet unspent: ")
-    log.debug(json.dumps(wallet.unspent, indent=4))
-
-
 """Purely blockchaininterface related error condition tests"""
 
 
-def test_index_ahead_cache(setup_wallets):
-    """Artificial test; look into finding a sync mode that triggers this
-    """
-
-    class NonWallet(object):
-        pass
-
-    wallet = NonWallet()
-    wallet.index_cache = [[0, 0], [0, 2]]
-    from jmclient.blockchaininterface import is_index_ahead_of_cache
-    assert is_index_ahead_of_cache(wallet, 3, 1)
-
-
-def test_core_wallet_no_sync(setup_wallets):
-    """Ensure BitcoinCoreWallet sync attempt does nothing
-    """
-    wallet = BitcoinCoreWallet("")
-    #this will not trigger sync due to absence of non-zero index_cache, usually.
-    wallet.index_cache = [[1, 1]]
-    jm_single().bc_interface.wallet_synced = False
-    jm_single().bc_interface.sync_wallet(wallet, fast=True)
-    assert not jm_single().bc_interface.wallet_synced
-    jm_single().bc_interface.sync_wallet(wallet)
-    assert not jm_single().bc_interface.wallet_synced
-
-
 def test_wrong_network_bci(setup_wallets):
     rpc = jm_single().bc_interface.jsonRpc
     with pytest.raises(Exception) as e_info:
@@ -306,31 +90,6 @@ def test_absurd_fee(setup_wallets):
     load_program_config()
 
 
-def test_abstract_wallet(setup_wallets):
-
-    class DoNothingWallet(AbstractWallet):
-        pass
-
-    for algo in ["default", "gradual", "greedy", "greediest", "none"]:
-        jm_single().config.set("POLICY", "merge_algorithm", algo)
-        if algo == "none":
-            with pytest.raises(Exception) as e_info:
-                dnw = DoNothingWallet()
-            #also test if the config is blank
-            jm_single().config = SafeConfigParser()
-            dnw = DoNothingWallet()
-            assert dnw.utxo_selector == select
-        else:
-            dnw = DoNothingWallet()
-        assert not dnw.get_key_from_addr("a")
-        assert not dnw.get_utxos_by_mixdepth()
-        assert not dnw.get_external_addr(1)
-        assert not dnw.get_internal_addr(0)
-        dnw.update_cache_index()
-        dnw.remove_old_utxos("a")
-        dnw.add_new_utxos("b", "c")
-        load_program_config()
-
 def check_bip39_case(vectors, language="english"):
     mnemo = Mnemonic(language)
     for v in vectors:
@@ -359,288 +118,6 @@ def test_bip39_vectors(setup_wallets):
     vectors = filter(lambda x: len(x[1].split())==12, vectors)
     check_bip39_case(vectors)
 
-@pytest.mark.parametrize(
-    "pwd, me, valid", [
-        ("asingleword", "1234aaaaaaaaaaaaaaaaa", True),
-        ("a whole set of words", "a whole set of words", True),
-        ("wordwithtrailingspaces   ", "A few words with trailing  ", True),
-        ("monkey", "verylongpasswordindeedxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", True),
-        ("blablah", "invalidcontainsnonascii\xee", False)
-    ])
-def test_create_bip39_with_me(setup_wallets, pwd, me, valid):
-    def dummyDisplayWords(a, b):
-        pass
-    def getMnemonic():
-        return ("legal winner thank year wave sausage worth useful legal winner thank yellow",
-                me)
-    def getPassword():
-        return pwd
-    def getWalletFileName():
-        return "bip39-test-wallet-name-from-callback.json"
-    def promptMnemonicExtension():
-        return me
-    if os.path.exists(os.path.join("wallets", getWalletFileName())):
-        os.remove(os.path.join("wallets", getWalletFileName()))
-    success = wallet_generate_recover_bip39("generate",
-                                            "wallets",
-                                            "wallet.json",
-                                            callbacks=(dummyDisplayWords,
-                                                       getMnemonic,
-                                                       getPassword,
-                                                       getWalletFileName,
-                                                       promptMnemonicExtension))
-    if not valid:
-        #wgrb39 returns false for failed wallet creation case
-        assert not success
-        return
-    assert success
-    #open the wallet file, and decrypt the encrypted mnemonic extension and check
-    #it's the one we intended.
-    with open(os.path.join("wallets", getWalletFileName()), 'r') as f:
-        walletdata = json.load(f)
-    password_key = bitcoin.bin_dbl_sha256(getPassword())
-    cleartext = decryptData(password_key,
-                            walletdata['encrypted_mnemonic_extension'].decode('hex'))
-    assert len(cleartext) >= 79
-    #throws if not len == 3
-    padding, me2, checksum = cleartext.split('\xff')
-    strippedme = me.strip()
-    assert strippedme == me2
-    assert checksum == bitcoin.dbl_sha256(strippedme)[:8]
-    #also test recovery from this combination of mnemonic + extension
-    if os.path.exists(os.path.join("wallets", getWalletFileName())):
-        os.remove(os.path.join("wallets", getWalletFileName()))
-    success = wallet_generate_recover_bip39("recover", "wallets", "wallet.json",
-                                            callbacks=(dummyDisplayWords,
-                                                       getMnemonic,
-                                                       getPassword,
-                                                       getWalletFileName,
-                                                       None))
-    assert success
-    with open(os.path.join("wallets", getWalletFileName()), 'r') as f:
-        walletdata = json.load(f)
-        password_key = bitcoin.bin_dbl_sha256(getPassword())
-        cleartext = decryptData(password_key,
-                    walletdata['encrypted_entropy'].decode('hex')).encode('hex')
-        assert cleartext == "7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f"
-
-def create_default_testnet_wallet():
-    walletdir = "wallets"
-    testwalletname = "testwallet.json"
-    pathtowallet = os.path.join(walletdir, testwalletname)
-    if os.path.exists(pathtowallet):
-        os.remove(pathtowallet)
-    seed = "deadbeef"
-    return (walletdir, pathtowallet, testwalletname,
-            SegwitWallet(seed,
-                   None,
-                   5,
-                   6,
-                   extend_mixdepth=False,
-                   storepassword=False))
-
-@pytest.mark.parametrize(
-    "includecache, wrongnet, storepwd, extendmd, pwdnumtries", [
-        (False, False, False, False, 100),
-        (True, False, False, True, 1),
-        (False, True, False, False, 1),
-        (False, False, True, False, 1)
-    ])
-def test_wallet_create(setup_wallets, includecache, wrongnet, storepwd,
-                       extendmd, pwdnumtries):
-    walletdir, pathtowallet, testwalletname, wallet = create_default_testnet_wallet(
-    )
-    assert wallet.get_key(
-        4, 1,
-        17) == "96095d7542e4e832c476b9df7e49ca9e5be61ad3bb8c8a3bdd8e141e2f4caf9101"
-    assert wallet.get_addr(2, 0, 5) == "2NBUxbEQrGPKrYCV6d4o7Y4AtJ34Uy6gZZg"
-    jm_single().bc_interface.wallet_synced = True
-    assert wallet.get_new_addr(1, 0) == "2Mz817RE6zqywgkG2h9cATUoiXwnFSxufk2"
-    assert wallet.get_external_addr(3) == "2N3gn65WXEzbLnjk5FLDZPc1pL6ebvZAmoA"
-    addr3internal = wallet.get_internal_addr(3)
-    assert addr3internal == "2N5NMTYogAyrGhDtWBnVQUp1kgwwFzcf7UM"
-    assert wallet.get_key_from_addr(
-        addr3internal) == "089a7173314d29f99e02a37e36da517ce41537a317c83284db1f33dda0af0cc201"
-    dummyaddr = "mvw1NazKDRbeNufFANqpYNAANafsMC2zVU"
-    assert not wallet.get_key_from_addr(dummyaddr)
-    #Make a new Wallet(), and prepare a testnet wallet file for this wallet
-
-    password = "dummypassword"
-    password_key = bitcoin.bin_dbl_sha256(password)
-    seed = bitcoin.sha256("\xaa" * 64)[:32]
-    encrypted_seed = encryptData(password_key, seed.decode('hex'))
-    timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")
-    net = get_network() if not wrongnet else 'mainnnet'
-    walletfilejson = {'creator': 'joinmarket project',
-                      'creation_time': timestamp,
-                      'encrypted_seed': encrypted_seed.encode('hex'),
-                      'network': net}
-    if includecache:
-        mmd = wallet.max_mix_depth if not extendmd else wallet.max_mix_depth + 5
-        print("using mmd: " + str(mmd))
-        walletfilejson.update({'index_cache': [[0, 0]] * mmd})
-    walletfile = json.dumps(walletfilejson)
-    if not os.path.exists(walletdir):
-        os.makedirs(walletdir)
-    with open(pathtowallet, "wb") as f:
-        f.write(walletfile)
-    if wrongnet:
-        with pytest.raises(ValueError) as e_info:
-            SegwitWallet(testwalletname,
-                   password,
-                   5,
-                   6,
-                   extend_mixdepth=extendmd,
-                   storepassword=storepwd)
-        return
-    from string import ascii_letters
-    for i in range(
-            pwdnumtries):  #multiple tries to ensure pkcs7 error is triggered
-        with pytest.raises(WalletError) as e_info:
-            wrongpwd = "".join([random.choice(ascii_letters) for _ in range(20)
-                               ])
-            SegwitWallet(testwalletname,
-                   wrongpwd,
-                   5,
-                   6,
-                   extend_mixdepth=extendmd,
-                   storepassword=storepwd)
-
-    with pytest.raises(WalletError) as e_info:
-        SegwitWallet(testwalletname,
-               None,
-               5,
-               6,
-               extend_mixdepth=extendmd,
-               storepassword=storepwd)
-    newwallet = SegwitWallet(testwalletname,
-                       password,
-                       5,
-                       6,
-                       extend_mixdepth=extendmd,
-                       storepassword=storepwd)
-    assert newwallet.seed == wallet.wallet_data_to_seed(seed)
-    #now we have a functional wallet + file, update the cache; first try
-    #with failed paths
-    oldpath = newwallet.path
-    newwallet.path = None
-    newwallet.update_cache_index()
-    newwallet.path = "fake-path-definitely-doesnt-exist"
-    newwallet.update_cache_index()
-    #with real path
-    newwallet.path = oldpath
-    newwallet.index = [[1, 1]] * 5
-    newwallet.update_cache_index()
-
-    #ensure we cannot find a mainnet wallet from seed
-    seed = "goodbye"
-    jm_single().config.set("BLOCKCHAIN", "network", "mainnet")
-    with pytest.raises(IOError) as e_info:
-        Wallet(seed, 5, 6, False, False)
-    load_program_config()
-
-
-def test_imported_privkey(setup_wallets):
-    for n in ["mainnet", "testnet"]:
-        privkey = "7d998b45c219a1e38e99e7cbd312ef67f77a455a9b50c730c27f02c6f730dfb401"
-        jm_single().config.set("BLOCKCHAIN", "network", n)
-        password = "dummypassword"
-        password_key = bitcoin.bin_dbl_sha256(password)
-        wifprivkey = bitcoin.wif_compressed_privkey(privkey, get_p2pk_vbyte())
-        #mainnet is "L1RrrnXkcKut5DEMwtDthjwRcTTwED36thyL1DebVrKuwvohjMNi"
-        #to verify use from_wif_privkey and privkey_to_address
-        if n == "mainnet":
-            iaddr = "1LDsjB43N2NAQ1Vbc2xyHca4iBBciN8iwC"
-        else:
-            iaddr = "mzjq2E92B3oRB7yDKbwM7XnPaAnKfRERw2"
-        privkey_bin = bitcoin.from_wif_privkey(
-            wifprivkey,
-            vbyte=get_p2pk_vbyte()).decode('hex')[:-1]
-        encrypted_privkey = encryptData(password_key, privkey_bin)
-        encrypted_privkey_bad = encryptData(password_key, privkey_bin[:6])
-        walletdir = "wallets"
-        testwalletname = "test" + n
-        pathtowallet = os.path.join(walletdir, testwalletname)
-        seed = bitcoin.sha256("\xaa" * 64)[:32]
-        encrypted_seed = encryptData(password_key, seed.decode('hex'))
-        timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")
-        for ep in [encrypted_privkey, encrypted_privkey_bad]:
-            walletfilejson = {'creator': 'joinmarket project',
-                              'creation_time': timestamp,
-                              'encrypted_seed': encrypted_seed.encode('hex'),
-                              'network': n,
-                              'index_cache': [[0, 0]] * 5,
-                              'imported_keys': [
-                                  {'encrypted_privkey': ep.encode('hex'),
-                                   'mixdepth': 0}
-                              ]}
-            walletfile = json.dumps(walletfilejson)
-            if not os.path.exists(walletdir):
-                os.makedirs(walletdir)
-            with open(pathtowallet, "wb") as f:
-                f.write(walletfile)
-            if ep == encrypted_privkey_bad:
-                with pytest.raises(Exception) as e_info:
-                    Wallet(testwalletname, password, 5, 6, False, False)
-                continue
-            newwallet = Wallet(testwalletname, password, 5, 6, False, False)
-            assert newwallet.seed == seed
-            #test accessing the key from the addr
-            assert newwallet.get_key_from_addr(
-                iaddr) == bitcoin.from_wif_privkey(wifprivkey,
-                                                   vbyte=get_p2pk_vbyte())
-            if n == "testnet":
-                jm_single().bc_interface.sync_wallet(newwallet)
-    load_program_config()
-
-def test_add_remove_utxos(setup_wallets):
-    #Make a fake wallet and inject and then remove fake utxos
-    walletdir, pathtowallet, testwalletname, wallet = create_default_testnet_wallet()
-    assert wallet.get_addr(2, 0, 5) == "2NBUxbEQrGPKrYCV6d4o7Y4AtJ34Uy6gZZg"
-    wallet.addr_cache["2NBUxbEQrGPKrYCV6d4o7Y4AtJ34Uy6gZZg"] = (2, 0, 5)
-    #'a914c80b3c03b96c0da5ef983942d9e541cb788aed8787'
-    #these calls automatically update the addr_cache:
-    assert wallet.get_new_addr(1, 0) == "2Mz817RE6zqywgkG2h9cATUoiXwnFSxufk2"
-    #a9144b6b3836a1708fd38d4728e41b86e69d5bb15d5187
-    assert wallet.get_external_addr(3) == "2N3gn65WXEzbLnjk5FLDZPc1pL6ebvZAmoA"
-    #a914728673d95ceafa892ed82f9cc23c8bf1700b6c6187
-    #using the above pubkey scripts:
-    faketxforwallet = {'outs': [
-        {'script': 'a914c80b3c03b96c0da5ef983942d9e541cb788aed8787',
-         'value': 110000000},
-        {'script': 'a9144b6b3836a1708fd38d4728e41b86e69d5bb15d5187',
-         'value': 89910900},
-        {'script': 'a914728673d95ceafa892ed82f9cc23c8bf1700b6c6187',
-         'value': 90021000},
-        {'script':
-         '76a9145ece2dac945c8ff5b2b6635360ca0478ade305d488ac',  #not ours
-         'value': 110000000}
-    ],
-                       'version': 1}
-    wallet.add_new_utxos(faketxforwallet, "aa" * 32)
-    faketxforspending = {'ins': [
-        {'outpoint': {'hash': 'aa' * 32,
-                      'index': 0}}, {'outpoint': {'hash': 'aa' * 32,
-                                                  'index': 1}}, {'outpoint':
-                                                                 {'hash':
-                                                                  'aa' * 32,
-                                                                  'index': 2}},
-        {'outpoint':
-         {'hash':
-          '3f3ea820d706e08ad8dc1d2c392c98facb1b067ae4c671043ae9461057bd2a3c',
-          'index': 1},
-         'script': '',
-         'sequence': 4294967295}
-    ]}
-    wallet.select_utxos(1, 100000)
-    with pytest.raises(Exception) as e_info:
-        wallet.select_utxos(0, 100000)
-    #ensure get_utxos_by_mixdepth can handle utxos outside of maxmixdepth
-    wallet.max_mix_depth = 2
-    mul = wallet.get_utxos_by_mixdepth()
-    assert mul[3] != {}
-    wallet.remove_old_utxos(faketxforspending)
-
 
 @pytest.fixture(scope="module")
 def setup_wallets():

scripts/README.md

@@ -18,12 +18,12 @@ Brief explanation of the function of each of the scripts:
 
 Either use the same syntax as for normal Joinmarket:
 
-    `python sendpayment.py --fast -N 3 -m 1 -P wallet.json 50000000 <address>`
+    `python sendpayment.py --fast -N 3 -m 1 -P wallet.jmdat 50000000 <address>`
 
 or use the new schedule approach. For an example, see the [sample schedule file](https://github.com/AdamISZ/joinmarket-clientserver/blob/master/scripts/sample-schedule-for-testnet).
 Do:
 
-    `python sendpayment.py --fast -S sample-schedule-for-testnet wallet.json`
+    `python sendpayment.py --fast -S sample-schedule-for-testnet wallet.jmdat`
 
 Note that the magic string `INTERNAL` in the file creates a payment to a new address
 in the next mixdepth (wrapping around to zero if you reach the maximum mixdepth).

scripts/add-utxo.py

@@ -6,20 +6,20 @@ users to retry transactions more often without getting banned by
 the anti-snooping feature employed by makers.
 """
 
-import binascii
 import sys
 import os
 import json
+import binascii
 from pprint import pformat
 
 from optparse import OptionParser
 import jmclient.btc as btc
-from jmbase import get_password
-from jmclient import (load_program_config, jm_single, get_p2pk_vbyte, get_wallet_cls,
-                      WalletError, sync_wallet, add_external_commitments,
-                      generate_podle, update_commitments, PoDLE,
-                      set_commitment_file, get_podle_commitments,
-                      get_utxo_info, validate_utxo_data, quit)
+from jmclient import (
+    load_program_config, jm_single, get_p2pk_vbyte, open_wallet, WalletError,
+    sync_wallet, add_external_commitments, generate_podle, update_commitments,
+    PoDLE, set_commitment_file, get_podle_commitments, get_utxo_info,
+    validate_utxo_data, quit, get_wallet_path)
+
 
 def add_ext_commitments(utxo_datas):
     """Persist the PoDLE commitments for this utxo
@@ -174,30 +174,17 @@ def main():
     #Three options (-w, -r, -R) for loading utxo and privkey pairs from a wallet,
     #csv file or json file.
     if options.loadwallet:
-        while True:
-            pwd = get_password("Enter wallet decryption passphrase: ")
-            try:
-                wallet = get_wallet_cls()(options.loadwallet,
-                                pwd,
-                                options.maxmixdepth,
-                                options.gaplimit)
-            except WalletError:
-                print("Wrong password, try again.")
-                continue
-            except Exception as e:
-                print("Failed to load wallet, error message: " + repr(e))
-                sys.exit(0)
-            break
-        sync_wallet(wallet, fast=options.fastsync)
-        unsp = {}
-        for u, av in wallet.unspent.iteritems():
-                    addr = av['address']
-                    key = wallet.get_key_from_addr(addr)
-                    wifkey = btc.wif_compressed_privkey(key, vbyte=get_p2pk_vbyte())
-                    unsp[u] = {'address': av['address'],
-                               'value': av['value'], 'privkey': wifkey}
-        for u, pva  in unsp.iteritems():
-            utxo_data.append((u, pva['privkey']))
+        wallet_path = get_wallet_path(options.loadwallet, None)
+        wallet = open_wallet(wallet_path, gap_limit=options.gaplimit)
+        while not jm_single().bc_interface.wallet_synced:
+            sync_wallet(wallet, fast=options.fastsync)
+
+        for md, utxos in wallet.get_utxos_by_mixdepth_().items():
+            for (txid, index), utxo in utxos.items():
+                txhex = binascii.hexlify(txid) + ':' + str(index)
+                wif = wallet.get_wif_path(utxo['path'])
+                utxo_data.append((txhex, wif))
+
     elif options.in_file:
         with open(options.in_file, "rb") as f:
             utxo_info = f.readlines()

scripts/convert_old_wallet.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python2
+
+import argparse
+import json
+import os.path
+from hashlib import sha256
+from binascii import hexlify, unhexlify
+from collections import defaultdict
+
+from jmclient import Storage, decryptData, load_program_config
+from jmclient.wallet_utils import get_password, get_wallet_cls,\
+    cli_get_wallet_passphrase_check, get_wallet_path
+from jmbitcoin import wif_compressed_privkey
+
+
+class ConvertException(Exception):
+    pass
+
+
+def get_max_mixdepth(data):
+    return max(1, len(data.get('index_cache', [1])) - 1,
+               *data.get('imported', {}).keys())
+
+
+def is_encrypted(wallet_data):
+    return 'encrypted_seed' in wallet_data or 'encrypted_entropy' in wallet_data
+
+
+def double_sha256(plaintext):
+    return sha256(sha256(plaintext).digest()).digest()
+
+
+def decrypt_entropy_extension(enc_data, key):
+    data = decryptData(key, unhexlify(enc_data))
+    if data[-9] != b'\xff':
+        raise ConvertException("Wrong password.")
+    chunks = data.split(b'\xff')
+    if len(chunks) < 3 or data[-8:] != double_sha256(chunks[1])[:8]:
+        raise ConvertException("Wrong password.")
+    return chunks[1]
+
+
+def decrypt_wallet_data(data, password):
+    key = double_sha256(password.encode('utf-8'))
+
+    enc_entropy = data.get('encrypted_seed') or data.get('encrypted_entropy')
+    enc_entropy_ext = data.get('encrypted_mnemonic_extension')
+    enc_imported = data.get('imported_keys')
+
+    entropy = decryptData(key, unhexlify(enc_entropy))
+    data['entropy'] = entropy
+    if enc_entropy_ext:
+        data['entropy_ext'] = decrypt_entropy_extension(enc_entropy_ext, key)
+
+    if enc_imported:
+        imported_keys = defaultdict(list)
+        for e in enc_imported:
+            md = int(e['mixdepth'])
+            imported_enc_key = unhexlify(e['encrypted_privkey'])
+            imported_key = decryptData(key, imported_enc_key)
+            imported_keys[md].append(imported_key)
+        data['imported'] = imported_keys
+
+
+def new_wallet_from_data(data, file_name):
+    print("Creating new wallet file.")
+    new_pw = cli_get_wallet_passphrase_check()
+    if new_pw is False:
+        return False
+
+    storage = Storage(file_name, create=True, password=new_pw)
+    wallet_cls = get_wallet_cls()
+
+    kwdata = {
+        'entropy': data['entropy'],
+        'timestamp': data.get('creation_time'),
+        'max_mixdepth': get_max_mixdepth(data)
+    }
+
+    if 'entropy_ext' in data:
+        kwdata['entropy_extension'] = data['entropy_ext']
+
+    wallet_cls.initialize(storage, data['network'], **kwdata)
+    wallet = wallet_cls(storage)
+
+    if 'index_cache' in data:
+        for md, indices in enumerate(data['index_cache']):
+            wallet.set_next_index(md, 0, indices[0], force=True)
+            wallet.set_next_index(md, 1, indices[1], force=True)
+
+    if 'imported' in data:
+        for md in data['imported']:
+            for privkey in data['imported'][md]:
+                privkey += b'\x01'
+                wif = wif_compressed_privkey(hexlify(privkey))
+                wallet.import_private_key(md, wif)
+
+    wallet.save()
+    wallet.close()
+    return True
+
+
+def parse_old_wallet(fh):
+    file_data = json.load(fh)
+
+    if is_encrypted(file_data):
+        pw = get_password("Enter password for old wallet file: ")
+        try:
+            decrypt_wallet_data(file_data, pw)
+        except ValueError:
+            print("Failed to open wallet: bad password")
+            return
+        except Exception as e:
+            print("Error: {}".format(e))
+            print("Failed to open wallet. Wrong password?")
+            return
+
+    return file_data
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Convert old joinmarket json wallet format to new jmdat "
+                    "format")
+    parser.add_argument('old_wallet_file', type=open)
+    parser.add_argument('--name', '-n', required=False, dest='name',
+                        help="Name of the new wallet file. Default: [old wallet name].jmdat")
+
+    try:
+        args = parser.parse_args()
+    except Exception as e:
+        print("Error: {}".format(e))
+        return
+
+    data = parse_old_wallet(args.old_wallet_file)
+
+    if not data:
+        return
+
+    file_name = args.name or\
+        os.path.split(args.old_wallet_file.name)[-1].rsplit('.', 1)[0] + '.jmdat'
+    wallet_path = get_wallet_path(file_name, None)
+    if new_wallet_from_data(data, wallet_path):
+        print("New wallet file created at {}".format(wallet_path))
+    else:
+        print("Failed to convert wallet.")
+
+
+if __name__ == '__main__':
+    load_program_config()
+    main()

scripts/jmtainter.py

@@ -9,17 +9,14 @@ can spread to other outputs included.
 This is a tool for Joinmarket wallets specifically.
 """
 import binascii
-import os
-import sys
-import random
 from optparse import OptionParser
 from pprint import pformat
 import jmbitcoin as btc
-from jmclient import (load_program_config, validate_address, jm_single,
-                      WalletError, sync_wallet, RegtestBitcoinCoreInterface,
-                      estimate_tx_fee, SegwitWallet, get_p2pk_vbyte,
-                      get_p2sh_vbyte, get_wallet_cls)
-from jmbase.support import get_password
+from jmclient import (
+    load_program_config, validate_address, jm_single, WalletError, sync_wallet,
+    RegtestBitcoinCoreInterface, estimate_tx_fee, get_p2pk_vbyte,
+    get_p2sh_vbyte, open_test_wallet_maybe, get_wallet_path)
+
 
 def get_parser():
     parser = OptionParser(
@@ -79,25 +76,16 @@ def is_utxo(utxo):
     return True
 
 def cli_get_wallet(wallet_name, sync=True):
-    if not os.path.exists(os.path.join('wallets', wallet_name)):
-        wallet = get_wallet_cls()(wallet_name, None, max_mix_depth=options.amtmixdepths)
-    else:
-        while True:
-            try:
-                pwd = get_password("Enter wallet decryption passphrase: ")
-                wallet = get_wallet_cls()(wallet_name, pwd, max_mix_depth=options.amtmixdepths)
-            except WalletError:
-                print("Wrong password, try again.")
-                continue
-            except Exception as e:
-                print("Failed to load wallet, error message: " + repr(e))
-                sys.exit(0)
-            break
+    wallet_path = get_wallet_path(wallet_name, None)
+    wallet = open_test_wallet_maybe(
+        wallet_path, wallet_name, options.amtmixdepths, gap_limit=options.gaplimit)
+
     if jm_single().config.get("BLOCKCHAIN",
                               "blockchain_source") == "electrum-server":
         jm_single().bc_interface.synctype = "with-script"
     if sync:
-        sync_wallet(wallet, fast=options.fastsync)
+        while not jm_single().bc_interface.wallet_synced:
+            sync_wallet(wallet, fast=options.fastsync)
     return wallet
 
 #======Electrum specific utils=========================
@@ -116,18 +104,24 @@ def serialize_derivation(roc, i):
     return x
 #=======================================================
 
-def get_privkey_amount_from_utxo(wallet, utxo):
+
+def get_script_amount_from_utxo(wallet, utxo):
     """Given a JM wallet and a utxo string, find
     the corresponding private key and amount controlled
     in satoshis.
     """
-    for k, v in wallet.unspent.iteritems():
-        if k == utxo:
-            print("Found utxo, its value is: ", v['value'])
-            return wallet.get_key_from_addr(v['address']), v['value']
-    return (None, None)
+    for md, utxos in wallet.get_utxos_by_mixdepth_().items():
+        for (txid, index), utxo in utxos.items():
+            txhex = binascii.hexlify(txid) + ':' + str(index)
+            if txhex != utxo:
+                continue
+            script = wallet.get_script_path(utxo['path'])
+            print("Found utxo, its value is: {}".format(utxo['value']))
+            return script, utxo['value']
+    return None, None
+
 
-def create_single_acp_pair(utxo_in, priv, addr_out, amount, bump, segwit=False):
+def create_single_acp_pair(wallet, utxo_in, script, addr_out, amount, bump, segwit=False):
     """Given a utxo and a signing key for it, and its amout in satoshis,
     sign a "transaction" consisting of only 1 input and one output, signed
     with single|acp sighash flags so it can be grafted into a bigger
@@ -142,10 +136,9 @@ def create_single_acp_pair(utxo_in, priv, addr_out, amount, bump, segwit=False):
     assert bump >= 0, "Output of single|acp pair must be bigger than input for safety."
     out = {"address": addr_out, "value": amount + bump}
     tx = btc.mktx([utxo_in], [out])
-    amt = amount if segwit else None
-    return btc.sign(tx, 0, priv,
-                    hashcode=btc.SIGHASH_SINGLE|btc.SIGHASH_ANYONECANPAY,
-                    amount=amt)
+    return wallet.sign_tx(tx, {0: (script, amount)},
+                          hashcode=btc.SIGHASH_SINGLE|btc.SIGHASH_ANYONECANPAY)
+
 
 def graft_onto_single_acp(wallet, txhex, amount, destaddr):
     """Given a serialized txhex which is checked to be of
@@ -200,13 +193,15 @@ def graft_onto_single_acp(wallet, txhex, amount, destaddr):
     df['ins'][0]['script'] = d['ins'][0]['script']
     if 'txinwitness' in d['ins'][0]:
         df['ins'][0]['txinwitness'] = d['ins'][0]['txinwitness']
-    fulltx = btc.serialize(df)
+
     for i, iu in enumerate(input_utxos):
-        priv, inamt = get_privkey_amount_from_utxo(wallet, iu)
-        print("Signing index: ", i+1, " with privkey: ", priv, " and amount: ", inamt, " for utxo: ", iu)
-        fulltx = btc.sign(fulltx, i+1, priv, amount=inamt)
-    return (True, fulltx)
-        
+        script, inamt = get_script_amount_from_utxo(wallet, iu)
+        print("Signing index: ", i+1, " with script: ", script, " and amount: ", inamt, " for utxo: ", iu)
+        fulltx = wallet.sign_tx(df, {i: (script, inamt)})
+
+    return True, btc.serialize(fulltx)
+
+
 if __name__ == "__main__":
     parser = get_parser()
     (options, args) = parser.parse_args()
@@ -227,22 +222,22 @@ if __name__ == "__main__":
             "Use wallet-tool.py method 'showutxos' to select one")
             exit(0)
         utxo_in = args[2]
-        priv, amount = get_privkey_amount_from_utxo(wallet, utxo_in)
-        if not priv:
+        script, amount = get_script_amount_from_utxo(wallet, utxo_in)
+        if not script:
             print("Failed to find the utxo's private key from the wallet; check "
                   "if this utxo is actually contained in the wallet using "
                   "wallet-tool.py showutxos")
             exit(0)
         #destination sourced from wallet
         addr_out = wallet.get_new_addr((options.mixdepth+1)%options.amtmixdepths, 1)
-        serialized_single_acp = create_single_acp_pair(utxo_in, priv, addr_out, amount,
-                                                       options.bump, segwit=True)
+        single_acp = create_single_acp_pair(wallet, utxo_in, script, addr_out, amount,
+                                            options.bump, segwit=True)
         print("Created the following one-in, one-out transaction, which will not "
               "be valid to broadcast itself (negative fee). Pass it to your "
               "counterparty:")
-        print(pformat(btc.deserialize(serialized_single_acp)))
+        print(pformat(single_acp))
         print("Pass the following raw hex to your counterparty:")
-        print(serialized_single_acp)
+        print(btc.serialize(single_acp))
         exit(0)
     elif args[1] == "take":
         try:

scripts/joinmarket-qt.py

@@ -52,17 +52,16 @@ JM_CORE_VERSION = '0.3.5'
 #Version of this Qt script specifically
 JM_GUI_VERSION = '7'
 
-from jmclient import (load_program_config, get_network, SegwitWallet,
-                      get_p2sh_vbyte, get_p2pk_vbyte, jm_single, validate_address,
-                      get_log, weighted_order_choose, Taker,
-                      JMClientProtocolFactory, WalletError,
-                      start_reactor, get_schedule, get_tumble_schedule,
-                      schedule_to_text, create_wallet_file,
-                      get_blockchain_interface_instance, sync_wallet, direct_send,
-                      RegtestBitcoinCoreInterface, tweak_tumble_schedule,
-                      human_readable_schedule_entry, tumbler_taker_finished_update,
-                      get_tumble_log, restart_wait, tumbler_filter_orders_callback,
-                      wallet_generate_recover_bip39, wallet_display)
+from jmclient import (
+    load_program_config, get_network, open_wallet, get_wallet_path,
+    get_p2sh_vbyte, get_p2pk_vbyte, jm_single, validate_address, get_log,
+    weighted_order_choose, Taker, JMClientProtocolFactory, WalletError,
+    start_reactor, get_schedule, get_tumble_schedule, schedule_to_text,
+    get_blockchain_interface_instance, sync_wallet,
+    direct_send, RegtestBitcoinCoreInterface, tweak_tumble_schedule,
+    human_readable_schedule_entry, tumbler_taker_finished_update,
+    get_tumble_log, restart_wait, tumbler_filter_orders_callback,
+    wallet_generate_recover_bip39, wallet_display)
 
 from qtsupport import (ScheduleWizard, TumbleRestartWizard, warnings, config_tips,
                        config_types, TaskThread, QtHandler, XStream, Buttons,
@@ -1321,7 +1320,7 @@ class JMMainWindow(QMainWindow):
 
     def recoverWallet(self):
         success = wallet_generate_recover_bip39("recover", "wallets",
-                                                "wallet.json",
+                                                "wallet.jmdat",
                                                 callbacks=(None, self.seedEntry,
                                                            self.getPassword,
                                                            self.getWalletFileName))
@@ -1373,16 +1372,14 @@ class JMMainWindow(QMainWindow):
 
     def loadWalletFromBlockchain(self, firstarg=None, pwd=None, restart_cb=None):
         if (firstarg and pwd) or (firstarg and get_network() == 'testnet'):
+            wallet_path = get_wallet_path(str(firstarg), None)
             try:
-                self.wallet = SegwitWallet(
-                    str(firstarg),
-                    pwd,
-                    max_mix_depth=jm_single().config.getint(
-                    "GUI", "max_mix_depth"),
-                    gaplimit=jm_single().config.getint("GUI", "gaplimit"))
-            except WalletError:
+                self.wallet = open_wallet(
+                    wallet_path, ask_for_password=False, password=pwd,
+                    gap_limit=jm_single().config.getint("GUI", "gaplimit"))
+            except Exception as e:
                 JMQtMessageBox(self,
-                               "Wrong password",
+                               str(e),
                                mbtype='warn',
                                title="Error")
                 return False
@@ -1473,7 +1470,7 @@ class JMMainWindow(QMainWindow):
     def getWalletFileName(self):
         walletname, ok = QInputDialog.getText(self, 'Choose wallet name',
                                               'Enter wallet file name:',
-                                              QLineEdit.Normal, "wallet.json")
+                                              QLineEdit.Normal, "wallet.jmdat")
         if not ok:
             JMQtMessageBox(self, "Create wallet aborted", mbtype='warn')
             return None
@@ -1516,7 +1513,7 @@ class JMMainWindow(QMainWindow):
         if not seed:
             success = wallet_generate_recover_bip39("generate",
                                                    "wallets",
-                                                   "wallet.json",
+                                                   "wallet.jmdat",
                                                    callbacks=(self.displayWords,
                                                               None,
                                                               self.getPassword,

scripts/sendpayment.py

@@ -17,14 +17,12 @@ import time
 import os
 import pprint
 
-from jmclient import (Taker, load_program_config, get_schedule,
-                              JMClientProtocolFactory, start_reactor,
-                              validate_address, jm_single, WalletError,
-                              choose_orders, choose_sweep_orders,
-                              cheapest_order_choose, weighted_order_choose,
-                              sync_wallet, RegtestBitcoinCoreInterface,
-                              estimate_tx_fee, direct_send, get_wallet_cls,
-                              BitcoinCoreWallet)
+from jmclient import (
+    Taker, load_program_config, get_schedule, JMClientProtocolFactory,
+    start_reactor, validate_address, jm_single, WalletError, choose_orders,
+    choose_sweep_orders, cheapest_order_choose, weighted_order_choose,
+    sync_wallet, RegtestBitcoinCoreInterface, estimate_tx_fee, direct_send,
+    open_test_wallet_maybe, get_wallet_path)
 from twisted.python.log import startLogging
 from jmbase.support import get_log, debug_dump_object, get_password
 from cli_options import get_sendpayment_parser
@@ -126,33 +124,27 @@ def main():
         #maxmixdepth in the wallet is actually the *number* of mixdepths (so misnamed);
         #to ensure we have enough, must be at least (requested index+1)
         max_mix_depth = max([mixdepth+1, options.amtmixdepths])
-        if not os.path.exists(os.path.join('wallets', wallet_name)):
-            wallet = get_wallet_cls()(wallet_name, None, max_mix_depth, options.gaplimit)
-        else:
-            while True:
-                try:
-                    pwd = get_password("Enter wallet decryption passphrase: ")
-                    wallet = get_wallet_cls()(wallet_name, pwd, max_mix_depth, options.gaplimit)
-                except WalletError:
-                    print("Wrong password, try again.")
-                    continue
-                except Exception as e:
-                    print("Failed to load wallet, error message: " + repr(e))
-                    sys.exit(0)
-                break
+
+        wallet_path = get_wallet_path(wallet_name, None)
+        wallet = open_test_wallet_maybe(
+            wallet_path, wallet_name, max_mix_depth, gap_limit=options.gaplimit)
     else:
-        wallet = BitcoinCoreWallet(fromaccount=wallet_name)
+        raise NotImplemented("Using non-joinmarket wallet is not supported.")
     if jm_single().config.get("BLOCKCHAIN",
         "blockchain_source") == "electrum-server" and options.makercount != 0:
         jm_single().bc_interface.synctype = "with-script"
     #wallet sync will now only occur on reactor start if we're joining.
-    sync_wallet(wallet, fast=options.fastsync)
+    while not jm_single().bc_interface.wallet_synced:
+        sync_wallet(wallet, fast=options.fastsync)
     if options.makercount == 0:
-        if isinstance(wallet, BitcoinCoreWallet):
-            raise NotImplementedError("Direct send only supported for JM wallets")
         direct_send(wallet, amount, mixdepth, destaddr, options.answeryes)
         return
 
+    if wallet.get_txtype() == 'p2pkh':
+        print("Only direct sends (use -N 0) are supported for "
+              "legacy (non-segwit) wallets.")
+        return
+
     def filter_orders_callback(orders_fees, cjamount):
         orders, total_cj_fee = orders_fees
         log.info("Chose these orders: " +pprint.pformat(orders))

scripts/tumbler.py

@@ -1,26 +1,19 @@
 #! /usr/bin/env python
 from __future__ import absolute_import, print_function
 
-import random
 import sys
-import threading
-from optparse import OptionParser
 from twisted.internet import reactor
-import time
 import os
 import pprint
-import copy
-import logging
 from twisted.python.log import startLogging
-from jmclient import (Taker, load_program_config, get_schedule,
-                      weighted_order_choose, JMClientProtocolFactory,
-                      start_reactor, validate_address, jm_single, WalletError,
-                      get_wallet_cls, sync_wallet, get_tumble_schedule,
-                      RegtestBitcoinCoreInterface, estimate_tx_fee,
-                      tweak_tumble_schedule, human_readable_schedule_entry,
-                      schedule_to_text, restart_waiter, get_tumble_log,
-                      tumbler_taker_finished_update, tumbler_filter_orders_callback)
-
+from jmclient import (
+    Taker, load_program_config, get_schedule,  weighted_order_choose,
+    JMClientProtocolFactory, start_reactor, validate_address, jm_single,
+    get_wallet_path, open_test_wallet_maybe, WalletError, sync_wallet,
+    get_tumble_schedule, RegtestBitcoinCoreInterface, estimate_tx_fee,
+    tweak_tumble_schedule, human_readable_schedule_entry, schedule_to_text,
+    restart_waiter, get_tumble_log, tumbler_taker_finished_update,
+    tumbler_filter_orders_callback)
 from jmbase.support import get_log, debug_dump_object, get_password
 from cli_options import get_tumbler_parser
 log = get_log()
@@ -39,24 +32,14 @@ def main():
     #Load the wallet
     wallet_name = args[0]
     max_mix_depth = options['mixdepthsrc'] + options['mixdepthcount']
-    if not os.path.exists(os.path.join('wallets', wallet_name)):
-        wallet = get_wallet_cls()(wallet_name, None, max_mix_depth)
-    else:
-        while True:
-            try:
-                pwd = get_password("Enter wallet decryption passphrase: ")
-                wallet = get_wallet_cls()(wallet_name, pwd, max_mix_depth)
-            except WalletError:
-                print("Wrong password, try again.")
-                continue
-            except Exception as e:
-                print("Failed to load wallet, error message: " + repr(e))
-                sys.exit(0)
-            break
+    wallet_path = get_wallet_path(wallet_name, None)
+    wallet = open_test_wallet_maybe(wallet_path, wallet_name, max_mix_depth)
+
     if jm_single().config.get("BLOCKCHAIN",
                               "blockchain_source") == "electrum-server":
         jm_single().bc_interface.synctype = "with-script"
-    sync_wallet(wallet, fast=options['fastsync'])
+    while not jm_single().bc_interface.wallet_synced:
+        sync_wallet(wallet, fast=options['fastsync'])
 
     #Parse options and generate schedule
     #Output information to log files

test/common.py

@@ -4,7 +4,6 @@ from __future__ import absolute_import
 
 import sys
 import os
-import time
 import binascii
 import random
 from decimal import Decimal
@@ -12,7 +11,8 @@ from decimal import Decimal
 data_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 sys.path.insert(0, os.path.join(data_dir))
 
-from jmclient import get_wallet_cls, get_log, estimate_tx_fee, jm_single
+from jmclient import open_test_wallet_maybe, BIP32Wallet, SegwitLegacyWallet, \
+    get_log, estimate_tx_fee, jm_single
 import jmbitcoin as btc
 from jmbase import chunks
 
@@ -63,7 +63,8 @@ def make_wallets(n,
                  fixed_seeds=None,
                  test_wallet=False,
                  passwords=None,
-                 walletclass=None):
+                 walletclass=SegwitLegacyWallet,
+                 mixdepths=5):
     '''n: number of wallets to be created
        wallet_structure: array of n arrays , each subarray
        specifying the number of addresses to be populated with coins
@@ -74,35 +75,36 @@ def make_wallets(n,
        Default Wallet constructor is joinmarket.Wallet, else use TestWallet,
        which takes a password parameter as in the list passwords.
        '''
+    # FIXME: this is basically the same code as jmclient/test/commontest.py
     if len(wallet_structures) != n:
         raise Exception("Number of wallets doesn't match wallet structures")
     if not fixed_seeds:
-        seeds = chunks(binascii.hexlify(os.urandom(15 * n)), 15 * 2)
+        seeds = chunks(binascii.hexlify(os.urandom(BIP32Wallet.ENTROPY_BYTES * n)),
+                       BIP32Wallet.ENTROPY_BYTES * 2)
     else:
         seeds = fixed_seeds
     wallets = {}
     for i in range(n):
-        if test_wallet:
-            w = TestWallet(seeds[i], max_mix_depth=5, pwd=passwords[i])
+        assert len(seeds[i]) == BIP32Wallet.ENTROPY_BYTES * 2
+
+        # FIXME: pwd is ignored (but do we really need this anyway?)
+        if test_wallet and passwords and i < len(passwords):
+            pwd = passwords[i]
         else:
-            if walletclass:
-                wc = walletclass
-            else:
-                wc = get_wallet_cls()
-            w = wc(seeds[i], pwd=None, max_mix_depth=5)
+            pwd = None
+
+        w = open_test_wallet_maybe(seeds[i], seeds[i], mixdepths,
+                                   test_wallet_cls=walletclass)
+
         wallets[i + start_index] = {'seed': seeds[i],
                                     'wallet': w}
-        for j in range(5):
+        for j in range(mixdepths):
             for k in range(wallet_structures[i][j]):
                 deviation = sdev_amt * random.random()
                 amt = mean_amt - sdev_amt / 2.0 + deviation
                 if amt < 0: amt = 0.001
                 amt = float(Decimal(amt).quantize(Decimal(10)**-8))
-                jm_single().bc_interface.grab_coins(
-                    wallets[i + start_index]['wallet'].get_external_addr(j),
-                    amt)
-            #reset the index so the coins can be seen if running in same script
-            wallets[i + start_index]['wallet'].index[j][0] -= wallet_structures[i][j]
+                jm_single().bc_interface.grab_coins(w.get_external_addr(j), amt)
     return wallets
 
 

test/test_segwit.py

@@ -2,17 +2,13 @@
 from __future__ import absolute_import
 '''Test creation of segwit transactions.'''
 
-import sys
-import os
-import time
 import binascii
 import json
 from common import make_wallets
 from pprint import pformat
 import jmbitcoin as btc
 import pytest
-from jmclient import (load_program_config, jm_single, get_p2pk_vbyte,
-                      get_log, get_p2sh_vbyte, Wallet)
+from jmclient import load_program_config, jm_single, get_log, LegacyWallet
 log = get_log()
 
 
@@ -30,80 +26,11 @@ def test_segwit_valid_txs(setup_segwit):
         #and compare the json values
 
 
-def get_utxo_from_txid(txid, addr):
-    """Given a txid and an address for one of the outputs,
-    return "txid:n" where n is the index of the output
-    """
-    rawtx = jm_single().bc_interface.rpc("getrawtransaction", [txid, 1])
-    ins = []
-    for u in rawtx["vout"]:
-        if u["scriptPubKey"]["addresses"][0] == addr:
-            ins.append(txid + ":" + str(u["n"]))
-    assert len(ins) == 1
-    return ins[0]
-
-
-def make_sign_and_push(ins_sw,
-                       wallet,
-                       amount,
-                       other_ins=None,
-                       output_addr=None,
-                       change_addr=None,
-                       hashcode=btc.SIGHASH_ALL):
-    """A more complicated version of the function in test_tx_creation;
-    will merge to this one once finished.
-    ins_sw have this structure:
-    {"txid:n":(amount, priv, index), "txid2:n2":(amount2, priv2, index2), ..}
-    if other_ins is not None, it has the same format, 
-    these inputs are assumed to be plain p2pkh.
-    All of these inputs in these two sets will be consumed.
-    They are ordered according to the "index" fields (to allow testing
-    of specific ordering)
-    It's assumed that they contain sufficient coins to satisy the
-    required output specified in "amount", plus some extra for fees and a
-    change output.
-    The output_addr and change_addr, if None, are taken from the wallet
-    and are ordinary p2pkh outputs.
-    All amounts are in satoshis and only converted to btc for grab_coins
-    """
-    #total value of all inputs
-    print ins_sw
-    print other_ins
-    total = sum([x[0] for x in ins_sw.values()])
-    total += sum([x[0] for x in other_ins.values()])
-    #construct the other inputs
-    ins1 = other_ins
-    ins1.update(ins_sw)
-    ins1 = sorted(ins1.keys(), key=lambda k: ins1[k][2])
-    #random output address and change addr
-    output_addr = wallet.get_new_addr(1, 1) if not output_addr else output_addr
-    change_addr = wallet.get_new_addr(1, 0) if not change_addr else change_addr
-    outs = [{'value': amount,
-             'address': output_addr}, {'value': total - amount - 10000,
-                                       'address': change_addr}]
-    tx = btc.mktx(ins1, outs)
-    de_tx = btc.deserialize(tx)
-    for index, ins in enumerate(de_tx['ins']):
-        utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
-        temp_ins = ins_sw if utxo in ins_sw.keys() else other_ins
-        amt, priv, n = temp_ins[utxo]
-        temp_amt = amt if utxo in ins_sw.keys() else None
-        #for better test code coverage
-        print "signing tx index: " + str(index) + ", priv: " + priv
-        if index % 2:
-            priv = binascii.unhexlify(priv)
-        ms = "other" if not temp_amt else "amount: " + str(temp_amt)
-        print ms
-        tx = btc.sign(tx, index, priv, hashcode=hashcode, amount=temp_amt)
-    print pformat(btc.deserialize(tx))
-    txid = jm_single().bc_interface.pushtx(tx)
-    time.sleep(3)
-    received = jm_single().bc_interface.get_received_by_addr(
-        [output_addr], None)['data'][0]['balance']
-    #check coins were transferred as expected
-    assert received == amount
-    #pushtx returns False on any error
-    return txid
+def binarize_tx(tx):
+    for o in tx['outs']:
+        o['script'] = binascii.unhexlify(o['script'])
+    for i in tx['ins']:
+        i['outpoint']['hash'] = binascii.unhexlify(i['outpoint']['hash'])
 
 
 @pytest.mark.parametrize(
@@ -111,7 +38,7 @@ def make_sign_and_push(ins_sw,
         ([[1, 0, 0, 0, 0]], 1, 1000000, 1, [0, 1, 2], []),
         ([[4, 0, 0, 0, 1]], 3, 100000000, 1, [0, 2], [1, 3]),
         ([[4, 0, 0, 0, 1]], 3, 100000000, 1, [0, 5], [1, 2, 3, 4]),
-        ([[2, 0, 0, 0, 2]], 2, 200000007, 0.3, [0, 1, 4, 5], [2, 3, 6]),
+        ([[4, 0, 0, 0, 0]], 2, 200000007, 0.3, [0, 1, 4, 5], [2, 3, 6]),
     ])
 def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount,
                                  segwit_amt, segwit_ins, o_ins):
@@ -127,38 +54,94 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount,
     segwit_ins is a list of input indices (where to place the funding segwit utxos)
     other_ins is a list of input indices (where to place the funding non-sw utxos)
     """
-    wallet = make_wallets(1, wallet_structure, in_amt, walletclass=Wallet)[0]['wallet']
-    jm_single().bc_interface.sync_wallet(wallet, fast=True)
-    other_ins = {}
-    ctr = 0
-    for k, v in wallet.unspent.iteritems():
-        #only extract as many non-segwit utxos as we need;
-        #doesn't matter which they are
-        if ctr == len(o_ins):
-            break
-        other_ins[k] = (v["value"], wallet.get_key_from_addr(v["address"]),
-                        o_ins[ctr])
-        ctr += 1
-    ins_sw = {}
-    for i in range(len(segwit_ins)):
-        #build segwit ins from "deterministic-random" keys;
-        #intended to be the same for each run with the same parameters
-        seed = json.dumps([i, wallet_structure, in_amt, amount, segwit_ins,
-                           other_ins])
-        priv = btc.sha256(seed) + "01"
-        pub = btc.privtopub(priv)
-        #magicbyte is testnet p2sh
-        addr1 = btc.pubkey_to_p2sh_p2wpkh_address(pub, magicbyte=196)
-        print "got address for p2shp2wpkh: " + addr1
-        txid = jm_single().bc_interface.grab_coins(addr1, segwit_amt)
-        #TODO - int cast, fix?
-        ins_sw[get_utxo_from_txid(txid, addr1)] = (int(segwit_amt * 100000000),
-                                                   priv, segwit_ins[i])
-    #make_sign_and_push will sanity check the received amount is correct
-    txid = make_sign_and_push(ins_sw, wallet, amount, other_ins)
-    #will always be False if it didn't push.
+    MIXDEPTH = 0
+
+    # set up wallets and inputs
+    nsw_wallet = make_wallets(1, wallet_structure, in_amt,
+                              walletclass=LegacyWallet)[0]['wallet']
+    jm_single().bc_interface.sync_wallet(nsw_wallet, fast=True)
+    sw_wallet = make_wallets(1, [[len(segwit_ins), 0, 0, 0, 0]], segwit_amt)[0]['wallet']
+    jm_single().bc_interface.sync_wallet(sw_wallet, fast=True)
+
+    nsw_utxos = nsw_wallet.get_utxos_by_mixdepth_()[MIXDEPTH]
+    sw_utxos = sw_wallet.get_utxos_by_mixdepth_()[MIXDEPTH]
+    assert len(o_ins) <= len(nsw_utxos), "sync failed"
+    assert len(segwit_ins) <= len(sw_utxos), "sync failed"
+
+    total_amt_in_sat = 0
+
+    nsw_ins = {}
+    for nsw_in_index in o_ins:
+        total_amt_in_sat += in_amt * 10**8
+        nsw_ins[nsw_in_index] = nsw_utxos.popitem()
+
+    sw_ins = {}
+    for sw_in_index in segwit_ins:
+        total_amt_in_sat += int(segwit_amt * 10**8)
+        sw_ins[sw_in_index] = sw_utxos.popitem()
+
+    all_ins = {}
+    all_ins.update(nsw_ins)
+    all_ins.update(sw_ins)
+
+    # sanity checks
+    assert len(all_ins) == len(nsw_ins) + len(sw_ins), \
+        "test broken, duplicate index"
+    for k in all_ins:
+        assert 0 <= k < len(all_ins), "test broken, missing input index"
+
+    # FIXME: encoding mess, mktx should accept binary input formats
+    tx_ins = []
+    for i, (txid, data) in sorted(all_ins.items(), key=lambda x: x[0]):
+        tx_ins.append('{}:{}'.format(binascii.hexlify(txid[0]), txid[1]))
+
+    # create outputs
+    FEE = 50000
+    assert FEE < total_amt_in_sat - amount, "test broken, not enough funds"
+
+    cj_script = nsw_wallet.get_new_script(MIXDEPTH + 1, True)
+    change_script = nsw_wallet.get_new_script(MIXDEPTH, True)
+    change_amt = total_amt_in_sat - amount - FEE
+
+    tx_outs = [
+        {'script': binascii.hexlify(cj_script),
+         'value': amount},
+        {'script': binascii.hexlify(change_script),
+         'value': change_amt}]
+    tx = btc.deserialize(btc.mktx(tx_ins, tx_outs))
+    binarize_tx(tx)
+
+    # import new addresses to bitcoind
+    jm_single().bc_interface.import_addresses(
+        [nsw_wallet.script_to_addr(x)
+         for x in [cj_script, change_script]],
+        jm_single().bc_interface.get_wallet_name(nsw_wallet))
+
+    # sign tx
+    scripts = {}
+    for nsw_in_index in o_ins:
+        inp = nsw_ins[nsw_in_index][1]
+        scripts[nsw_in_index] = (inp['script'], inp['value'])
+    nsw_wallet.sign_tx(tx, scripts)
+
+    scripts = {}
+    for sw_in_index in segwit_ins:
+        inp = sw_ins[sw_in_index][1]
+        scripts[sw_in_index] = (inp['script'], inp['value'])
+    sw_wallet.sign_tx(tx, scripts)
+
+    print(tx)
+
+    # push and verify
+    txid = jm_single().bc_interface.pushtx(binascii.hexlify(btc.serialize(tx)))
     assert txid
 
+    balances = jm_single().bc_interface.get_received_by_addr(
+        [nsw_wallet.script_to_addr(cj_script),
+         nsw_wallet.script_to_addr(change_script)], None)['data']
+    assert balances[0]['balance'] == amount
+    assert balances[1]['balance'] == change_amt
+
 
 @pytest.fixture(scope="module")
 def setup_segwit():

test/ygrunner.py

@@ -110,7 +110,7 @@ def test_start_ygs(setup_ygrunner, num_ygs, wallet_structures, mean_amt,
     print("Seed : " + wallets[num_ygs]['seed'])
     #useful to see the utxos on screen sometimes
     sync_wallet(wallet, fast=True)
-    print(wallet.unspent)
+    print(wallet.get_utxos_by_mixdepth())
     txfee = 1000
     cjfee_a = 4200
     cjfee_r = '0.001'