From ca57a14d0ada2a4b49bebd5bdcaf1d7d5cc98352 Mon Sep 17 00:00:00 2001 From: undeath Date: Tue, 7 Aug 2018 02:03:45 +0200 Subject: [PATCH 01/23] add new wallet implementation --- jmclient/jmclient/cryptoengine.py | 266 ++++ jmclient/jmclient/storage.py | 327 ++++ jmclient/jmclient/wallet.py | 1727 +++++++++++++++------ jmclient/test/test_argon2.py | 35 + jmclient/test/test_blockchaininterface.py | 149 ++ jmclient/test/test_coinjoin.py | 122 +- jmclient/test/test_storage.py | 128 ++ jmclient/test/test_utxomanager.py | 93 ++ jmclient/test/test_wallet.py | 590 +++++++ scripts/convert_old_wallet.py | 151 ++ 10 files changed, 3104 insertions(+), 484 deletions(-) create mode 100644 jmclient/jmclient/cryptoengine.py create mode 100644 jmclient/jmclient/storage.py create mode 100644 jmclient/test/test_argon2.py create mode 100644 jmclient/test/test_blockchaininterface.py create mode 100644 jmclient/test/test_storage.py create mode 100644 jmclient/test/test_utxomanager.py create mode 100644 jmclient/test/test_wallet.py create mode 100644 scripts/convert_old_wallet.py diff --git a/jmclient/jmclient/cryptoengine.py b/jmclient/jmclient/cryptoengine.py new file mode 100644 index 0000000..88de28e --- /dev/null +++ b/jmclient/jmclient/cryptoengine.py @@ -0,0 +1,266 @@ +from __future__ import print_function, absolute_import, division, unicode_literals + + +from binascii import hexlify, unhexlify +from collections import OrderedDict + + +from . import btc +from .configure import get_network + + +TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH = range(3) +NET_MAINNET, NET_TESTNET = range(2) +NET_MAP = {'mainnet': NET_MAINNET, 'testnet': NET_TESTNET} +WIF_PREFIX_MAP = {'mainnet': 0x80, 'testnet': 0xef} +BIP44_COIN_MAP = {'mainnet': 2**31, 'testnet': 2**31 + 1} + + +# +# library stuff that should be in btc/jmbitcoin +# + + +P2PKH_PRE, P2PKH_POST = b'\x76\xa9\x14', b'\x88\xac' +P2SH_P2WPKH_PRE, P2SH_P2WPKH_POST = b'\xa9\x14', b'\x87' +P2WPKH_PRE = b'\x00\x14' + + +def _pubkey_to_script(pubkey, script_pre, script_post=b''): + # sanity check for public key + # see https://github.com/bitcoin/bitcoin/blob/master/src/pubkey.h + if not ((len(pubkey) == 33 and pubkey[0] in (b'\x02', b'\x03')) or + (len(pubkey) == 65 and pubkey[0] in (b'\x04', b'\x06', b'\x07'))): + raise Exception("Invalid public key!") + h = btc.bin_hash160(pubkey) + assert len(h) == 0x14 + assert script_pre[-1] == b'\x14' + return script_pre + h + script_post + + +def pubkey_to_p2pkh_script(pubkey): + return _pubkey_to_script(pubkey, P2PKH_PRE, P2PKH_POST) + + +def pubkey_to_p2sh_p2wpkh_script(pubkey): + wscript = pubkey_to_p2wpkh_script(pubkey) + return P2SH_P2WPKH_PRE + btc.bin_hash160(wscript) + P2SH_P2WPKH_POST + + +def pubkey_to_p2wpkh_script(pubkey): + return _pubkey_to_script(pubkey, P2WPKH_PRE) + + +class classproperty(object): + """ + from https://stackoverflow.com/a/5192374 + """ + def __init__(self, f): + self.f = f + + def __get__(self, obj, owner): + return self.f(owner) + + +class SimpleLruCache(OrderedDict): + """ + note: python3.2 has a lru cache in functools + """ + def __init__(self, max_size): + OrderedDict.__init__(self) + assert max_size > 0 + self.max_size = max_size + + def __setitem__(self, key, value): + OrderedDict.__setitem__(self, key, value) + self._adjust_size() + + def _adjust_size(self): + while len(self) > self.max_size: + self.popitem(last=False) + + +# +# library stuff end +# + + +class EngineError(Exception): + pass + + +class BTCEngine(object): + # must be set by subclasses + VBYTE = None + __LRU_KEY_CACHE = SimpleLruCache(50) + + @classproperty + def BIP32_priv_vbytes(cls): + return btc.PRIVATE[NET_MAP[get_network()]] + + @classproperty + def WIF_PREFIX(cls): + return WIF_PREFIX_MAP[get_network()] + + @classproperty + def BIP44_COIN_TYPE(cls): + return BIP44_COIN_MAP[get_network()] + + @staticmethod + def privkey_to_pubkey(privkey): + return btc.privkey_to_pubkey(privkey, False) + + @staticmethod + def address_to_script(addr): + return unhexlify(btc.address_to_script(addr)) + + @classmethod + def wif_to_privkey(cls, wif): + raw = btc.b58check_to_bin(wif) + vbyte = btc.get_version_byte(wif) + + if (btc.BTC_P2PK_VBYTE[get_network()] + cls.WIF_PREFIX) & 0xff == vbyte: + key_type = TYPE_P2PKH + elif (btc.BTC_P2SH_VBYTE[get_network()] + cls.WIF_PREFIX) & 0xff == vbyte: + key_type = TYPE_P2SH_P2WPKH + else: + key_type = None + + return raw, key_type + + @classmethod + def privkey_to_wif(cls, priv): + return btc.bin_to_b58check(priv, cls.WIF_PREFIX) + + @classmethod + def derive_bip32_master_key(cls, seed): + # FIXME: slight encoding mess + return btc.bip32_deserialize( + btc.bip32_master_key(seed, vbytes=cls.BIP32_priv_vbytes)) + + @classmethod + def derive_bip32_privkey(cls, master_key, path): + assert len(path) > 1 + return cls._walk_bip32_path(master_key, path)[-1] + + @classmethod + def derive_bip32_pub_export(cls, master_key, path): + priv = cls._walk_bip32_path(master_key, path) + return btc.bip32_serialize(btc.raw_bip32_privtopub(priv)) + + @classmethod + def derive_bip32_priv_export(cls, master_key, path): + return btc.bip32_serialize(cls._walk_bip32_path(master_key, path)) + + @classmethod + def _walk_bip32_path(cls, master_key, path): + key = master_key + for lvl in path[1:]: + assert lvl >= 0 + assert lvl < 2**32 + if (key, lvl) in cls.__LRU_KEY_CACHE: + key = cls.__LRU_KEY_CACHE[(key, lvl)] + else: + cls.__LRU_KEY_CACHE[(key, lvl)] = btc.raw_bip32_ckd(key, lvl) + key = cls.__LRU_KEY_CACHE[(key, lvl)] + return key + + @classmethod + def privkey_to_script(cls, privkey): + pub = cls.privkey_to_pubkey(privkey) + return cls.pubkey_to_script(pub) + + @classmethod + def pubkey_to_script(cls, pubkey): + raise NotImplementedError() + + @classmethod + def privkey_to_address(cls, privkey): + script = cls.privkey_to_script(privkey) + return btc.script_to_address(script, cls.VBYTE) + + @classmethod + def pubkey_to_address(cls, pubkey): + script = cls.pubkey_to_script(pubkey) + return btc.script_to_address(script, cls.VBYTE) + + @classmethod + def sign_transaction(cls, tx, index, privkey, amount): + raise NotImplementedError() + + @staticmethod + def sign_message(privkey, message): + """ + args: + privkey: bytes + message: bytes + returns: + base64-encoded signature + """ + return btc.ecdsa_sign(message, privkey, True, False) + + @classmethod + def script_to_address(cls, script): + return btc.script_to_address(script, vbyte=cls.VBYTE) + + +class BTC_P2PKH(BTCEngine): + @classproperty + def VBYTE(cls): + return btc.BTC_P2PK_VBYTE[get_network()] + + @classmethod + def pubkey_to_script(cls, pubkey): + return pubkey_to_p2pkh_script(pubkey) + + @classmethod + def sign_transaction(cls, tx, index, privkey, *args, **kwargs): + hashcode = kwargs.get('hashcode') or btc.SIGHASH_ALL + + pubkey = cls.privkey_to_pubkey(privkey) + script = cls.pubkey_to_script(pubkey) + + signing_tx = btc.serialize(btc.signature_form(tx, index, script, + hashcode=hashcode)) + # FIXME: encoding mess + sig = unhexlify(btc.ecdsa_tx_sign(signing_tx, hexlify(privkey), + **kwargs)) + + tx['ins'][index]['script'] = btc.serialize_script([sig, pubkey]) + + return tx + + +class BTC_P2SH_P2WPKH(BTCEngine): + # FIXME: implement different bip32 key export prefixes like electrum? + # see http://docs.electrum.org/en/latest/seedphrase.html#list-of-reserved-numbers + + @classproperty + def VBYTE(cls): + return btc.BTC_P2SH_VBYTE[get_network()] + + @classmethod + def pubkey_to_script(cls, pubkey): + return pubkey_to_p2sh_p2wpkh_script(pubkey) + + @classmethod + def sign_transaction(cls, tx, index, privkey, amount, + hashcode=btc.SIGHASH_ALL, **kwargs): + assert amount is not None + + pubkey = cls.privkey_to_pubkey(privkey) + wpkscript = pubkey_to_p2wpkh_script(pubkey) + pkscript = pubkey_to_p2pkh_script(pubkey) + + signing_tx = btc.segwit_signature_form(tx, index, pkscript, amount, + hashcode=hashcode, + decoder_func=lambda x: x) + # FIXME: encoding mess + sig = unhexlify(btc.ecdsa_tx_sign(signing_tx, hexlify(privkey), + hashcode=hashcode, **kwargs)) + + assert len(wpkscript) == 0x16 + tx['ins'][index]['script'] = b'\x16' + wpkscript + tx['ins'][index]['txinwitness'] = [sig, pubkey] + + return tx diff --git a/jmclient/jmclient/storage.py b/jmclient/jmclient/storage.py new file mode 100644 index 0000000..573c572 --- /dev/null +++ b/jmclient/jmclient/storage.py @@ -0,0 +1,327 @@ +from __future__ import print_function, absolute_import, division, unicode_literals + +import os +import shutil +import atexit +import bencoder +import pyaes +from hashlib import sha256 +from argon2 import low_level +from .support import get_random_bytes + + +class Argon2Hash(object): + def __init__(self, password, salt=None, hash_len=32, salt_len=16, + time_cost=500, memory_cost=1000, parallelism=4, + argon2_type=low_level.Type.I, version=19): + """ + args: + password: password as bytes + salt: salt in bytes or None to create random one, must have length >= 8 + hash_len: generated hash length in bytes + salt_len: salt length in bytes, ignored if salt is not None, must be >= 8 + + Other arguments are argon2 settings. Only change those if you know what + you're doing. Optimized for slow hashing suitable for file encryption. + """ + # Default and recommended settings from argon2.PasswordHasher are for + # interactive logins. For encryption we want something much slower. + self.settings = { + 'time_cost': time_cost, + 'memory_cost': memory_cost, + 'parallelism': parallelism, + 'hash_len': hash_len, + 'type': argon2_type, + 'version': version + } + self.salt = salt if salt is not None else get_random_bytes(salt_len) + self.hash = low_level.hash_secret_raw(password, self.salt, + **self.settings) + + +class StorageError(Exception): + pass + + +class StoragePasswordError(StorageError): + pass + + +class Storage(object): + """ + Responsible for reading/writing [encrypted] data to disk. + + All data to be stored must be added to self.data which defaults to an + empty dict. + + self.data must contain serializable data (dict, list, tuple, bytes, numbers) + Having str objects anywhere in self.data will lead to undefined behaviour (py3). + All dict keys must be bytes. + + KDF: argon2, ENC: AES-256-CBC + """ + MAGIC_UNENC = b'JMWALLET' + MAGIC_ENC = b'JMENCWLT' + MAGIC_DETECT_ENC = b'JMWALLET' + + ENC_KEY_BYTES = 32 # AES-256 + SALT_LENGTH = 16 + + def __init__(self, path, password=None, create=False, read_only=False): + """ + args: + path: file path to storage + password: bytes or None for unencrypted file + create: create file if it does not exist + read_only: do not change anything on the file system + """ + self.path = path + self._lock_file = None + self._hash = None + self._data_checksum = None + self.data = None + self.changed = False + self.read_only = read_only + self.newly_created = False + + if not os.path.isfile(path): + if create and not read_only: + self._create_new(password) + self._save_file() + self.newly_created = True + else: + raise StorageError("File not found.") + else: + self._load_file(password) + + assert self.data is not None + assert self._data_checksum is not None + + self._create_lock() + + def is_encrypted(self): + return self._hash is not None + + def is_locked(self): + return self._lock_file and os.path.exists(self._lock_file) + + def was_changed(self): + """ + return True if data differs from data on disk + """ + return self._data_checksum != self._get_data_checksum() + + def change_password(self, password): + if self.read_only: + raise StorageError("Cannot change password of read-only file.") + self._set_hash(password) + self._save_file() + + def save(self): + """ + Write file to disk if data was modified + """ + #if not self.was_changed(): + # return + if self.read_only: + raise StorageError("Read-only storage cannot be saved.") + self._save_file() + + @classmethod + def is_storage_file(cls, path): + return cls._get_file_magic(path) in (cls.MAGIC_ENC, cls.MAGIC_UNENC) + + @classmethod + def is_encrypted_storage_file(cls, path): + return cls._get_file_magic(path) == cls.MAGIC_ENC + + @classmethod + def _get_file_magic(cls, path): + assert len(cls.MAGIC_ENC) == len(cls.MAGIC_UNENC) + with open(path, 'rb') as fh: + return fh.read(len(cls.MAGIC_ENC)) + + def _get_data_checksum(self): + if self.data is None: #pragma: no cover + return None + return sha256(self._serialize(self.data)).digest() + + def _update_data_hash(self): + self._data_checksum = self._get_data_checksum() + + def _create_new(self, password): + self.data = {} + self._set_hash(password) + + def _set_hash(self, password): + if password is None: + self._hash = None + else: + self._hash = self._hash_password(password) + + def _save_file(self): + assert self.read_only == False + data = self._serialize(self.data) + enc_data = self._encrypt_file(data) + + magic = self.MAGIC_UNENC if data is enc_data else self.MAGIC_ENC + self._write_file(magic + enc_data) + self._update_data_hash() + + def _load_file(self, password): + data = self._read_file() + assert len(self.MAGIC_ENC) == len(self.MAGIC_UNENC) == 8 + magic = data[:8] + + if magic not in (self.MAGIC_ENC, self.MAGIC_UNENC): + raise StorageError("File does not appear to be a joinmarket wallet.") + + data = data[8:] + + if magic == self.MAGIC_ENC: + if password is None: + raise StorageError("Password required to open wallet.") + data = self._decrypt_file(password, data) + else: + assert magic == self.MAGIC_UNENC + + self.data = self._deserialize(data) + self._update_data_hash() + + def _write_file(self, data): + assert self.read_only is False + + if not os.path.exists(self.path): + # newly created storage + with open(self.path, 'wb') as fh: + fh.write(data) + return + + # using a tmpfile ensures the write is atomic + tmpfile = '{}.tmp'.format(self.path) + + with open(tmpfile, 'wb') as fh: + shutil.copystat(self.path, tmpfile) + fh.write(data) + + #FIXME: behaviour with symlinks might be weird + shutil.move(tmpfile, self.path) + + def _read_file(self): + # this method mainly exists for easier mocking + with open(self.path, 'rb') as fh: + return fh.read() + + @staticmethod + def _serialize(data): + return bencoder.bencode(data) + + @staticmethod + def _deserialize(data): + return bencoder.bdecode(data) + + def _encrypt_file(self, data): + if not self.is_encrypted(): + return data + + iv = get_random_bytes(16) + container = { + b'enc': {b'salt': self._hash.salt, b'iv': iv}, + b'data': self._encrypt(data, iv) + } + return self._serialize(container) + + def _decrypt_file(self, password, data): + assert password is not None + + container = self._deserialize(data) + assert b'enc' in container + assert b'data' in container + + self._hash = self._hash_password(password, container[b'enc'][b'salt']) + + return self._decrypt(container[b'data'], container[b'enc'][b'iv']) + + def _encrypt(self, data, iv): + encrypter = pyaes.Encrypter( + pyaes.AESModeOfOperationCBC(self._hash.hash, iv=iv)) + enc_data = encrypter.feed(self.MAGIC_DETECT_ENC + data) + enc_data += encrypter.feed() + + return enc_data + + def _decrypt(self, data, iv): + decrypter = pyaes.Decrypter( + pyaes.AESModeOfOperationCBC(self._hash.hash, iv=iv)) + try: + dec_data = decrypter.feed(data) + dec_data += decrypter.feed() + except ValueError: + # in most "wrong password" cases the pkcs7 padding will be wrong + raise StoragePasswordError("Wrong password.") + + if not dec_data.startswith(self.MAGIC_DETECT_ENC): + raise StoragePasswordError("Wrong password.") + return dec_data[len(self.MAGIC_DETECT_ENC):] + + @classmethod + def _hash_password(cls, password, salt=None): + return Argon2Hash(password, salt, + hash_len=cls.ENC_KEY_BYTES, salt_len=cls.SALT_LENGTH) + + def _create_lock(self): + if self.read_only: + return + self._lock_file = '{}.lock'.format(self.path) + if os.path.exists(self._lock_file): + self._lock_file = None + raise StorageError("File is currently in use. If this is a " + "leftover from a crashed instance you need to " + "remove the lock file manually") + #FIXME: in python >=3.3 use mode xb + with open(self._lock_file, 'wb'): + pass + + atexit.register(self.close) + + def _remove_lock(self): + if self._lock_file: + os.remove(self._lock_file) + self._lock_file = None + + def close(self): + if not self.read_only and self.was_changed(): + self._save_file() + self._remove_lock() + self.read_only = True + + def __del__(self): + self.close() + + +class VolatileStorage(Storage): + """ + Storage that is never actually written to disk and only kept in memory. + + This exists for easier testing. + """ + + def __init__(self, password=None, data=None): + self.file_data = None + super(VolatileStorage, self).__init__('VOLATILE', password, + create=True) + if data: + self.file_data = data + self._load_file(password) + + def _create_lock(self): + pass + + def _remove_lock(self): + pass + + def _write_file(self, data): + self.file_data = data + + def _read_file(self): + return self.file_data diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index cb220d8..d0330da 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -1,29 +1,62 @@ -from __future__ import print_function -import json -import os -import pprint -import sys -import datetime -from decimal import Decimal +from __future__ import print_function, absolute_import, division, unicode_literals + +from ConfigParser import NoOptionError +import warnings +import functools +import collections +import numbers +from binascii import hexlify, unhexlify +from datetime import datetime +from copy import deepcopy from mnemonic import Mnemonic -from ConfigParser import NoSectionError -from getpass import getpass +from hashlib import sha256 +from itertools import chain +from decimal import Decimal + + +from .configure import jm_single +from .support import select_gradual, select_greedy, select_greediest, \ + select +from .cryptoengine import BTC_P2PKH, BTC_P2SH_P2WPKH, TYPE_P2PKH, \ + TYPE_P2SH_P2WPKH +from .support import get_random_bytes +from . import mn_encode, mn_decode, btc + -import btc -from jmclient.slowaes import encryptData, decryptData -from jmclient.blockchaininterface import BitcoinCoreInterface, RegtestBitcoinCoreInterface -from jmclient.configure import jm_single, get_network, get_p2pk_vbyte, get_p2sh_vbyte -from jmbase.support import get_log -from jmclient.support import select_gradual, select_greedy,select_greediest, select +""" +transaction dict format: + { + 'version': int, + 'locktime': int, + 'ins': [ + { + 'outpoint': { + 'hash': bytes, + 'index': int + }, + 'script': bytes, + 'sequence': int, + 'txinwitness': [bytes] + } + ], + 'outs': [ + { + 'script': bytes, + 'value': int + } + ] + } +""" -log = get_log() -JM_WALLET_P2PKH = "00" -JM_WALLET_SW_P2SH_P2WPKH = "01" +def _int_to_bytestr(i): + return str(i).encode('ascii') + class WalletError(Exception): pass + def estimate_tx_fee(ins, outs, txtype='p2pkh'): '''Returns an estimate of the number of satoshis required for a transaction with the given number of inputs and outputs, @@ -38,7 +71,6 @@ def estimate_tx_fee(ins, outs, txtype='p2pkh'): str(absurd_fee) + ", quitting.") if txtype in ['p2pkh', 'p2shMofN']: tx_estimated_bytes = btc.estimate_tx_size(ins, outs, txtype) - log.debug("Estimated transaction size: "+str(tx_estimated_bytes)) return int((tx_estimated_bytes * fee_per_kb)/Decimal(1000.0)) elif txtype=='p2sh-p2wpkh': witness_estimate, non_witness_estimate = btc.estimate_tx_size( @@ -48,526 +80,1261 @@ def estimate_tx_fee(ins, outs, txtype='p2pkh'): else: raise NotImplementedError("Txtype: " + txtype + " not implemented.") -def create_wallet_file(pwd, seed): - password_key = btc.bin_dbl_sha256(pwd) - encrypted_seed = encryptData(password_key, seed.decode('hex')) - timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") - return json.dumps({'creator': 'joinmarket project', - 'creation_time': timestamp, - 'encrypted_seed': encrypted_seed.encode('hex'), - 'network': get_network()}) -class AbstractWallet(object): - """ - Abstract wallet for use with JoinMarket - Mostly written with Wallet in mind, the default JoinMarket HD wallet - """ +#FIXME: move this to a utilities file? +def deprecated(func): + @functools.wraps(func) + def wrapped(*args, **kwargs): + warnings.warn("Call to deprecated function {}.".format(func.__name__), + category=DeprecationWarning, stacklevel=2) + return func(*args, **kwargs) - def __init__(self): - self.max_mix_depth = 0 - self.unspent = None - self.utxo_selector = select - try: - config = jm_single().config - if config.get("POLICY", "merge_algorithm") == "gradual": - self.utxo_selector = select_gradual - elif config.get("POLICY", "merge_algorithm") == "greedy": - self.utxo_selector = select_greedy - elif config.get("POLICY", "merge_algorithm") == "greediest": - self.utxo_selector = select_greediest - elif config.get("POLICY", "merge_algorithm") != "default": - raise Exception("Unknown merge algorithm") - except NoSectionError: - pass + return wrapped - def get_key_from_addr(self, addr): - return None + +class UTXOManager(object): + STORAGE_KEY = b'utxo' + TXID_LEN = 32 + + def __init__(self, storage, merge_func): + self.storage = storage + self.selector = merge_func + # {mixdexpth: {(txid, index): (path, value)}} + self._utxo = None + self._load_storage() + assert self._utxo is not None + + @classmethod + def initialize(cls, storage): + storage.data[cls.STORAGE_KEY] = {} + + def _load_storage(self): + assert isinstance(self.storage.data[self.STORAGE_KEY], dict) + + self._utxo = collections.defaultdict(dict) + for md, data in self.storage.data[self.STORAGE_KEY].items(): + md = int(md) + md_data = self._utxo[md] + for utxo, value in data.items(): + txid = utxo[:self.TXID_LEN] + index = int(utxo[self.TXID_LEN:]) + md_data[(txid, index)] = value + + def save(self, write=True): + new_data = {} + self.storage.data[self.STORAGE_KEY] = new_data + for md, data in self._utxo.items(): + md = _int_to_bytestr(md) + new_data[md] = {} + # storage keys must be bytes() + for (txid, index), value in data.items(): + new_data[md][txid + _int_to_bytestr(index)] = value + if write: + self.storage.save() + + def reset(self): + self._utxo = collections.defaultdict(dict) + + def have_utxo(self, txid, index): + for md in self._utxo: + if (txid, index) in self._utxo[md]: + return md + return False + + def remove_utxo(self, txid, index, mixdepth): + assert isinstance(txid, bytes) + assert len(txid) == self.TXID_LEN + assert isinstance(index, numbers.Integral) + assert isinstance(mixdepth, numbers.Integral) + + return self._utxo[mixdepth].pop((txid, index)) + + def add_utxo(self, txid, index, path, value, mixdepth): + assert isinstance(txid, bytes) + assert len(txid) == self.TXID_LEN + assert isinstance(index, numbers.Integral) + assert isinstance(value, numbers.Integral) + assert isinstance(mixdepth, numbers.Integral) + + self._utxo[mixdepth][(txid, index)] = (path, value) + + def select_utxos(self, mixdepth, amount, utxo_filter=()): + assert isinstance(mixdepth, numbers.Integral) + utxos = self._utxo[mixdepth] + available = [{'utxo': utxo, 'value': val} + for utxo, (addr, val) in utxos.items() if utxo not in utxo_filter] + selected = self.selector(available, amount) + return {s['utxo']: {'path': utxos[s['utxo']][0], + 'value': utxos[s['utxo']][1]} + for s in selected} + + def get_balance_by_mixdepth(self): + balance_dict = collections.defaultdict(int) + for mixdepth, utxomap in self._utxo.items(): + value = sum(x[1] for x in utxomap.values()) + balance_dict[mixdepth] = value + return balance_dict def get_utxos_by_mixdepth(self): - return None + return deepcopy(self._utxo) + + def __eq__(self, o): + return self._utxo == o._utxo and \ + self.selector is o.selector + + +class BaseWallet(object): + TYPE = None + + MERGE_ALGORITHMS = { + 'default': select, + 'gradual': select_gradual, + 'greedy': select_greedy, + 'greediest': select_greediest + } + + _ENGINES = { + TYPE_P2PKH: BTC_P2PKH, + TYPE_P2SH_P2WPKH: BTC_P2SH_P2WPKH + } + + _ENGINE = None + + def __init__(self, storage, gap_limit=6, merge_algorithm_name=None): + # to be defined by inheriting classes + assert self.TYPE is not None + assert self._ENGINE is not None + + self.merge_algorithm = self._get_merge_algorithm(merge_algorithm_name) + self.gap_limit = gap_limit + self._storage = storage + self._utxos = None + self.max_mixdepth = None + self.network = None + + # {script: path}, should always hold mappings for all "known" keys + self._script_map = {} + + self._load_storage() + + assert self._utxos is not None + assert self.max_mixdepth is not None + assert self.max_mixdepth >= 0 + assert self.network in ('mainnet', 'testnet') + + @property + @deprecated + def max_mix_depth(self): + return self.max_mixdepth + + @property + @deprecated + def gaplimit(self): + return self.gap_limit + + def _load_storage(self): + """ + load data from storage + """ + if self._storage.data[b'wallet_type'] != self.TYPE: + raise Exception("Wrong class to initialize wallet of type {}." + .format(self.TYPE)) + self.network = self._storage.data[b'network'].decode('ascii') + self.max_mixdepth = self._storage.data[b'max_mixdepth'] + self._utxos = UTXOManager(self._storage, self.merge_algorithm) + + def save(self): + """ + Write data to associated storage object and trigger persistent update. + """ + self._storage.data[b'max_mixdepth'] = self.max_mixdepth + self._storage.save() + + @classmethod + def initialize(cls, storage, network, max_mixdepth=2, timestamp=None, + write=True): + """ + Initialize wallet in an empty storage. Must be used on a storage object + before creating a wallet object with it. + + args: + storage: a Storage object + network: str, network we are on, 'mainnet' or 'testnet' + max_mixdepth: int, number of the highest mixdepth + timestamp: bytes or None, defaults to the current time + write: execute storage.save() + """ + assert network in ('mainnet', 'testnet') + assert max_mixdepth >= 0 + + if storage.data != {}: + # prevent accidentally overwriting existing wallet + raise WalletError("Refusing to initialize wallet in non-empty " + "storage.") + + if not timestamp: + timestamp = datetime.now().strftime('%Y/%m/%d %H:%M:%S') + + storage.data[b'network'] = network.encode('ascii') + storage.data[b'max_mixdepth'] = max_mixdepth + storage.data[b'created'] = timestamp.encode('ascii') + storage.data[b'wallet_type'] = cls.TYPE + + UTXOManager.initialize(storage) + + if write: + storage.save() + + def get_txtype(self): + """ + use TYPE constant instead if possible + """ + if self.TYPE == TYPE_P2PKH: + return 'p2pkh' + elif self.TYPE == TYPE_P2SH_P2WPKH: + return 'p2sh-p2wpkh' + assert False + + def sign_tx(self, tx, scripts, **kwargs): + """ + Add signatures to transaction for inputs referenced by scripts. + + args: + tx: transaction dict + scripts: {input_index: (output_script, amount)} + kwargs: additional arguments for engine.sign_transaction + returns: + input transaction dict with added signatures + """ + for index, (script, amount) in scripts.items(): + assert amount > 0 + path = self.script_to_path(script) + privkey, engine = self._get_priv_from_path(path) + engine.sign_transaction(tx, index, privkey, amount, **kwargs) + return tx + + @deprecated + def get_key_from_addr(self, addr): + """ + There should be no reason for code outside the wallet to need a privkey. + """ + script = self._ENGINE.address_to_script(addr) + path = self.script_to_path(script) + privkey = self._get_priv_from_path(path)[0] + return hexlify(privkey) - def get_external_addr(self, mixing_depth): + def get_external_addr(self, mixdepth): """ Return an address suitable for external distribution, including funding the wallet from other sources, or receiving payments or donations. JoinMarket will never generate these addresses for internal use. """ - return None + script = self.get_external_script(mixdepth) + return self.script_to_addr(script) - def get_internal_addr(self, mixing_depth): + def get_internal_addr(self, mixdepth): """ Return an address for internal usage, as change addresses and when participating in transactions initiated by other parties. """ - return None + script = self.get_internal_script(mixdepth) + return self.script_to_addr(script) + + def get_external_script(self, mixdepth): + return self.get_new_script(mixdepth, False) + + def get_internal_script(self, mixdepth): + return self.get_new_script(mixdepth, True) @classmethod - def pubkey_to_address(cls, pubkey): - return None + def addr_to_script(cls, addr): + return cls._ENGINE.address_to_script(addr) - def update_cache_index(self): - pass + @classmethod + def pubkey_to_addr(cls, pubkey): + return cls._ENGINE.pubkey_to_address(pubkey) - def remove_old_utxos(self, tx): - pass + def script_to_addr(self, script): + assert self.is_known_script(script) + path = self.script_to_path(script) + engine = self._get_priv_from_path(path)[1] + return engine.script_to_address(script) - def add_new_utxos(self, tx, txid): - pass + @deprecated + def get_key(self, mixdepth, internal, index): + raise NotImplementedError() - def select_utxos(self, mixdepth, amount, utxo_filter=None): - if utxo_filter is None: - utxo_filter = [] - utxo_list = self.get_utxos_by_mixdepth()[mixdepth] - unspent = [{'utxo': utxo, - 'value': addrval['value']} - for utxo, addrval in utxo_list.iteritems() if utxo not in utxo_filter] - inputs = self.utxo_selector(unspent, amount) - log.debug('for mixdepth={} amount={} selected:'.format( - mixdepth, amount)) - log.debug(pprint.pformat(inputs)) - return dict([(i['utxo'], {'value': i['value'], - 'address': utxo_list[i['utxo']]['address']}) - for i in inputs]) + def get_addr(self, mixdepth, internal, index): + script = self.get_script(mixdepth, internal, index) + return self.script_to_addr(script) - def get_balance_by_mixdepth(self, verbose=True): - mix_balance = {} - for m in range(self.max_mix_depth): - mix_balance[m] = 0 - for mixdepth, utxos in self.get_utxos_by_mixdepth(verbose).iteritems(): - mix_balance[mixdepth] = sum( - [addrval['value'] for addrval in utxos.values()]) - return mix_balance - -class Wallet(AbstractWallet): - def __init__(self, - seedarg, - pwd, - max_mix_depth=2, - gaplimit=6, - extend_mixdepth=False, - storepassword=False, - wallet_dir=None): - super(Wallet, self).__init__() - self.vflag = JM_WALLET_P2PKH - self.max_mix_depth = max_mix_depth - self.storepassword = storepassword - # key is address, value is (mixdepth, forchange, index) if mixdepth = - # -1 it's an imported key and index refers to imported_privkeys - self.addr_cache = {} - self.unspent = {} - self.spent_utxos = [] - self.imported_privkeys = {} - self.seed = self.wallet_data_to_seed( - self.read_wallet_file_data(seedarg, pwd, wallet_dir=wallet_dir)) - if not self.seed: - raise WalletError("Failed to decrypt wallet") - if extend_mixdepth and len(self.index_cache) > max_mix_depth: - self.max_mix_depth = len(self.index_cache) - self.gaplimit = gaplimit - mixing_depth_keys = self.get_mixing_depth_keys(self.get_master_key()) - self.keys = [(btc.bip32_ckd(m, 0), btc.bip32_ckd(m, 1)) - for m in mixing_depth_keys] - - # self.index = [[0, 0]]*max_mix_depth - self.index = [] - for i in range(self.max_mix_depth): - self.index.append([0, 0]) - - def get_master_key(self): - if not self.seed: - raise Exception("Cannot extract master key of wallet, no seed.") - #Legacy used the seed in hex - if not isinstance(self, SegwitWallet): - bip32seed = self.seed - else: - bip32seed = self.seed.decode('hex') - return btc.bip32_master_key(bip32seed, (btc.MAINNET_PRIVATE if get_network( - ) == 'mainnet' else btc.TESTNET_PRIVATE)) + def get_addr_path(self, path): + script = self.get_script_path(path) + return self.script_to_addr(script) - def get_mixing_depth_keys(self, master): - """legacy path is m/0/n for n 0..N mixing depths + def get_new_addr(self, mixdepth, internal): + """ + use get_external_addr/get_internal_addr """ - m_0 = btc.bip32_ckd(master, 0) - return [btc.bip32_ckd(m_0, c) for c in range(self.max_mix_depth)] + script = self.get_new_script(mixdepth, internal) + return self.script_to_addr(script) - def get_root_path(self): - return "m/0" + def get_new_script(self, mixdepth, internal): + raise NotImplementedError() - def wallet_data_to_seed(self, entropy): - """for base/legacy wallet type, this is a passthrough. - for bip39 style wallets, this will convert from one to the other - """ - if entropy is None: - return None - #Feature for testnet testing: if we are using direct command line - #brainwallets (as we do for regtest), strip the flag. - if entropy.startswith("FAKESEED"): - entropy = entropy[8:] - return entropy + def get_wif(self, mixdepth, internal, index): + return self.get_wif_path(self.get_path(mixdepth, internal, index)) - def get_txtype(self): - """Return string defining wallet type - for purposes of transaction size estimates + def get_wif_path(self, path): + priv, engine = self._get_priv_from_path(path) + return engine.privkey_to_wif(priv) + + def get_path(self, mixdepth=None, internal=None, index=None): + raise NotImplementedError() + + def get_details(self, path): """ - return 'p2pkh' + Return mixdepth, internal, index for a given path + + args: + path: wallet path + returns: + tuple (mixdepth, type, index) - def sign(self, tx, i, priv, amount): - """Sign a transaction for pushing - onto the network. The amount field - is not used in this case (p2pkh) + type is one of 0, 1, 'imported' """ - return btc.sign(tx, i, priv) + raise NotImplementedError() - @classmethod - def script_to_address(cls, script): - """Return the address for a given output script, - which will be p2pkh for the default Wallet object, - and reading the correct network byte from the config. + @deprecated + def update_cache_index(self): + """ + Deprecated alias for save() """ - return btc.script_to_address(script, cls.get_vbyte()) + self.save() - @classmethod - def pubkey_to_address(cls, pubkey): - return btc.pubkey_to_address(pubkey, cls.get_vbyte()) - - def read_wallet_file_data(self, filename, pwd=None, wallet_dir=None): - self.path = None - wallet_dir = wallet_dir if wallet_dir else 'wallets' - self.index_cache = [[0, 0]] * self.max_mix_depth - path = os.path.join(wallet_dir, filename) - if not os.path.isfile(path): - if get_network() == 'testnet': - log.debug('filename interpreted as seed, only available in ' - 'testnet because this probably has lower entropy') - return "FAKESEED" + filename - else: - raise IOError('wallet file not found') - if not pwd: - log.info("Password required for non-testnet seed wallet") - return None - self.path = path - fd = open(path, 'r') - walletfile = fd.read() - fd.close() - walletdata = json.loads(walletfile) - if walletdata['network'] != get_network(): - raise ValueError('wallet network(%s) does not match ' - 'joinmarket configured network(%s)' % ( - walletdata['network'], get_network())) - if 'index_cache' in walletdata: - self.index_cache = walletdata['index_cache'] - if self.max_mix_depth > len(self.index_cache): - #This can happen e.g. in tumbler when we need more mixdepths - #than currently exist. Since we have no info for those extra - #depths, we must default to (0,0) (but sync should find used - #adddresses). - self.index_cache += [[0,0]] * ( - self.max_mix_depth - len(self.index_cache)) - password_key = btc.bin_dbl_sha256(pwd) - if 'encrypted_seed' in walletdata: #accept old field name - encrypted_entropy = walletdata['encrypted_seed'] - elif 'encrypted_entropy' in walletdata: - encrypted_entropy = walletdata['encrypted_entropy'] - try: - decrypted_entropy = decryptData( - password_key, - encrypted_entropy.decode('hex')).encode('hex') - # there is a small probability of getting a valid PKCS7 - # padding by chance from a wrong password; sanity check the - # seed length - if len(decrypted_entropy) != 32: - raise ValueError - except ValueError: - log.info('Incorrect password') - return None - - if 'encrypted_mnemonic_extension' in walletdata: - try: - cleartext = decryptData(password_key, - walletdata['encrypted_mnemonic_extension'].decode('hex')) - #theres a small chance of not getting a ValueError from the wrong - # password so also check the sum - if cleartext[-9] != '\xff': - raise ValueError - chunks = cleartext.split('\xff') - if len(chunks) < 3 or cleartext[-8:] != btc.dbl_sha256(chunks[1])[:8]: - raise ValueError - mnemonic_extension = chunks[1] - except ValueError: - log.info('incorrect password') - return None - else: - mnemonic_extension = None - - if self.storepassword: - self.password_key = password_key - self.walletdata = walletdata - if 'imported_keys' in walletdata: - for epk_m in walletdata['imported_keys']: - privkey = decryptData( - password_key, - epk_m['encrypted_privkey'].decode( 'hex')).encode('hex') - #Imported keys are stored as 32 byte strings only, so the - #second version below is sufficient, really. - if len(privkey) != 64: - raise Exception( - "Unexpected privkey format; already compressed?:" + privkey) - privkey += "01" - if epk_m['mixdepth'] not in self.imported_privkeys: - self.imported_privkeys[epk_m['mixdepth']] = [] - self.addr_cache[btc.privtoaddr( - privkey, magicbyte=get_p2pk_vbyte())] = (epk_m['mixdepth'], -1, - len(self.imported_privkeys[epk_m['mixdepth']])) - self.imported_privkeys[epk_m['mixdepth']].append(privkey) - - if mnemonic_extension: - return (decrypted_entropy, mnemonic_extension) - else: - return decrypted_entropy + @deprecated + def remove_old_utxos(self, tx): + tx = deepcopy(tx) + for inp in tx['ins']: + inp['outpoint']['hash'] = unhexlify(inp['outpoint']['hash']) - def update_cache_index(self): - if not self.path: - return - if not os.path.isfile(self.path): - return - fd = open(self.path, 'r') - walletfile = fd.read() - fd.close() - walletdata = json.loads(walletfile) - walletdata['index_cache'] = self.index - walletfile = json.dumps(walletdata) - fd = open(self.path, 'w') - fd.write(walletfile) - fd.close() - - def get_key(self, mixing_depth, forchange, i): - return btc.bip32_extract_key(btc.bip32_ckd( - self.keys[mixing_depth][forchange], i)) - - def get_addr(self, mixing_depth, forchange, i): - return btc.privtoaddr( - self.get_key(mixing_depth, forchange, i), magicbyte=get_p2pk_vbyte()) - - def get_new_addr(self, mixing_depth, forchange, import_required=False): - index = self.index[mixing_depth] - addr = self.get_addr(mixing_depth, forchange, index[forchange]) - self.addr_cache[addr] = (mixing_depth, forchange, index[forchange]) - index[forchange] += 1 - # self.update_cache_index() - bc_interface = jm_single().bc_interface - if isinstance(bc_interface, BitcoinCoreInterface) or isinstance( - bc_interface, RegtestBitcoinCoreInterface) or import_required: - # do not import in the middle of sync_wallet() - if bc_interface.wallet_synced: - if bc_interface.rpc('getaccount', [addr]) == '': - log.debug('importing address ' + addr + ' to bitcoin core') - bc_interface.rpc( - 'importaddress', - [addr, bc_interface.get_wallet_name(self), False]) - return addr - - def get_external_addr(self, mixing_depth): - return self.get_new_addr(mixing_depth, 0) - - def get_internal_addr(self, mixing_depth): - return self.get_new_addr(mixing_depth, 1) + ret = self.remove_old_utxos_(tx) - def get_key_from_addr(self, addr): - if addr not in self.addr_cache: - return None - ac = self.addr_cache[addr] - if ac[1] >= 0: - return self.get_key(*ac) - else: - return self.imported_privkeys[ac[0]][ac[2]] + removed_utxos = {} + for (txid, index), val in ret.items(): + val['address'] = self.get_addr_path(val['path']) + removed_utxos[hexlify(txid) + ':' + str(index)] = val + return removed_utxos - def remove_old_utxos(self, tx): + def remove_old_utxos_(self, tx): + """ + Remove all own inputs of tx from internal utxo list. + + args: + tx: transaction dict + returns: + {(txid, index): {'script': bytes, 'value': int} for all removed utxos + """ removed_utxos = {} - for ins in tx['ins']: - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - if utxo not in self.unspent: + for inp in tx['ins']: + txid, index = inp['outpoint']['hash'], inp['outpoint']['index'] + md = self._utxos.have_utxo(txid, index) + if md is False: continue - removed_utxos[utxo] = self.unspent[utxo] - del self.unspent[utxo] - log.debug('removed utxos, wallet now is \n' + pprint.pformat( - self.get_utxos_by_mixdepth(verbose=False))) - self.spent_utxos += removed_utxos.keys() + path, value = self._utxos.remove_utxo(txid, index, md) + script = self.get_script_path(path) + removed_utxos[(txid, index)] = {'script': script, + 'path': path, + 'value': value} return removed_utxos - @staticmethod - def get_vbyte(): - return get_p2pk_vbyte() - + @deprecated def add_new_utxos(self, tx, txid): + tx = deepcopy(tx) + for out in tx['outs']: + out['script'] = unhexlify(out['script']) + + ret = self.add_new_utxos_(tx, unhexlify(txid)) + + added_utxos = {} + for (txid_bin, index), val in ret.items(): + addr = self.get_addr_path(val['path']) + val['address'] = addr + added_utxos[txid + ':' + str(index)] = val + return added_utxos + + def add_new_utxos_(self, tx, txid): + """ + Add all outputs of tx for this wallet to internal utxo list. + + args: + tx: transaction dict + returns: + {(txid, index): {'script': bytes, 'path': tuple, 'value': int} + for all added utxos + """ + assert isinstance(txid, bytes) and len(txid) == self._utxos.TXID_LEN added_utxos = {} for index, outs in enumerate(tx['outs']): - addr = btc.script_to_address(outs['script'], self.get_vbyte()) - if addr not in self.addr_cache: + try: + self.add_utxo(txid, index, outs['script'], outs['value']) + except WalletError: continue - addrdict = {'address': addr, 'value': outs['value']} - utxo = txid + ':' + str(index) - added_utxos[utxo] = addrdict - self.unspent[utxo] = addrdict - log.debug('added utxos, wallet now is \n' + pprint.pformat( - self.get_utxos_by_mixdepth())) + + path = self.script_to_path(outs['script']) + added_utxos[(txid, index)] = {'script': outs['script'], + 'path': path, + 'value': outs['value']} return added_utxos + def add_utxo(self, txid, index, script, value): + assert isinstance(txid, bytes) + assert isinstance(index, int) + assert isinstance(script, bytes) + assert isinstance(value, int) + + if script not in self._script_map: + raise WalletError("Tried to add UTXO for unknown key to wallet.") + + path = self.script_to_path(script) + mixdepth = self._get_mixdepth_from_path(path) + self._utxos.add_utxo(txid, index, path, value, mixdepth) + + @deprecated + def select_utxos(self, mixdepth, amount, utxo_filter=None): + utxo_filter_new = None + if utxo_filter: + utxo_filter_new = [(unhexlify(utxo[:64]), int(utxo[65:])) + for utxo in utxo_filter] + ret = self.select_utxos_(mixdepth, amount, utxo_filter_new) + ret_conv = {} + for utxo, data in ret.items(): + addr = self.get_addr_path(data['path']) + utxo_txt = hexlify(utxo[0]) + ':' + str(utxo[1]) + ret_conv[utxo_txt] = {'address': addr, 'value': data['value']} + return ret_conv + + def select_utxos_(self, mixdepth, amount, utxo_filter=None): + """ + args: + mixdepth: int, mixdepth to select utxos from + amount: int, total minimum amount of all selected utxos + utxo_filter: list of (txid, index), utxos not to select + + returns: + {(txid, index): {'script': bytes, 'path': tuple, 'value': int}} + """ + assert isinstance(mixdepth, numbers.Integral) + assert isinstance(amount, numbers.Integral) + + if not utxo_filter: + utxo_filter = () + for i in utxo_filter: + assert len(i) == 2 + assert isinstance(i[0], bytes) + assert isinstance(i[1], numbers.Integral) + ret = self._utxos.select_utxos(mixdepth, amount, utxo_filter) + + for data in ret.values(): + data['script'] = self.get_script_path(data['path']) + + return ret + + def reset_utxos(self): + self._utxos.reset() + + def get_balance_by_mixdepth(self, verbose=True): + # TODO: verbose + return self._utxos.get_balance_by_mixdepth() + + @deprecated def get_utxos_by_mixdepth(self, verbose=True): + # TODO: verbose + ret = self.get_utxos_by_mixdepth_() + + utxos_conv = collections.defaultdict(dict) + for md, utxos in ret.items(): + for utxo, data in utxos.items(): + utxo_str = hexlify(utxo[0]) + ':' + str(utxo[1]) + addr = self.get_addr_path(data['path']) + data['address'] = addr + utxos_conv[md][utxo_str] = data + return utxos_conv + + def get_utxos_by_mixdepth_(self): + """ + returns: + {mixdepth: {(txid, index): + {'script': bytes, 'path': tuple, 'value': int}}} """ - returns a list of utxos sorted by different mix levels - """ - mix_utxo_list = {} - for m in range(self.max_mix_depth): - mix_utxo_list[m] = {} - for utxo, addrvalue in self.unspent.iteritems(): - mixdepth = self.addr_cache[addrvalue['address']][0] - if mixdepth not in mix_utxo_list: - mix_utxo_list[mixdepth] = {} - mix_utxo_list[mixdepth][utxo] = addrvalue - if verbose: - log.debug('get_utxos_by_mixdepth = \n' + pprint.pformat(mix_utxo_list)) - return mix_utxo_list - -class Bip39Wallet(Wallet): - """Using python module `mnemonic` to implement - BIP39, English only: - https://github.com/bitcoin/bips/blob/master/bip-0039.mediawiki + mix_utxos = self._utxos.get_utxos_by_mixdepth() + + script_utxos = collections.defaultdict(dict) + for md, data in mix_utxos.items(): + for utxo, (path, value) in data.items(): + script = self.get_script_path(path) + script_utxos[md][utxo] = {'script': script, + 'path': path, + 'value': value} + return script_utxos + + @classmethod + def _get_merge_algorithm(cls, algorithm_name=None): + if not algorithm_name: + try: + algorithm_name = jm_single().config.get('POLICY', + 'merge_algorithm') + except NoOptionError: + algorithm_name = 'default' + + alg = cls.MERGE_ALGORITHMS.get(algorithm_name) + if alg is None: + raise Exception("Unknown merge algorithm: '{}'." + "".format(algorithm_name)) + return alg + + def _get_mixdepth_from_path(self, path): + raise NotImplementedError() + + def get_script_path(self, path): + """ + internal note: This is the final sink for all operations that somehow + need to derive a script. If anything goes wrong when deriving a + script this is the place to look at. + + args: + path: wallet path + returns: + script + """ + raise NotImplementedError() + + def get_script(self, mixdepth, internal, index): + path = self.get_path(mixdepth, internal, index) + return self.get_script_path(path) + + def _get_priv_from_path(self, path): + raise NotImplementedError() + + def get_path_repr(self, path): + """ + Get a human-readable representation of the wallet path. + + args: + path: path tuple + returns: + str + """ + raise NotImplementedError() + + def path_repr_to_path(self, pathstr): + """ + Convert a human-readable path representation to internal representation. + + args: + pathstr: str + returns: + path tuple + """ + raise NotImplementedError() + + def get_next_unused_index(self, mixdepth, internal): + """ + Get the next index for public scripts/addresses not yet handed out. + + returns: + int >= 0 + """ + raise NotImplementedError() + + def get_mnemonic_words(self): + """ + Get mnemonic seed words for master key. + + returns: + mnemonic_seed, seed_extension + mnemonic_seed is a space-separated str of words + seed_extension is a str or None + """ + raise NotImplementedError() + + def sign_message(self, message, path): + """ + Sign the message using the key referenced by path. + + args: + message: bytes + path: path tuple + returns: + signature as base64-encoded string + """ + priv, engine = self._get_priv_from_path(path) + return engine.sign_message(priv, message) + + def get_wallet_id(self): + """ + Get a human-readable identifier for the wallet. + + returns: + str + """ + raise NotImplementedError() + + def yield_imported_paths(self, mixdepth): + """ + Get an iterator for all imported keys in given mixdepth. + + params: + mixdepth: int + returns: + iterator of wallet paths + """ + return iter([]) + + def is_known_addr(self, addr): + """ + Check if address is known to belong to this wallet. + + params: + addr: str + returns: + bool + """ + script = self.addr_to_script(addr) + return script in self._script_map + + def is_known_script(self, script): + """ + Check if script is known to belong to this wallet. + + params: + script: bytes + returns: + bool + """ + assert isinstance(script, bytes) + return script in self._script_map + + def get_addr_mixdepth(self, addr): + script = self.addr_to_script(addr) + return self.get_script_mixdepth(script) + + def get_script_mixdepth(self, script): + path = self.script_to_path(script) + return self._get_mixdepth_from_path(path) + + def yield_known_paths(self): + """ + Generator for all paths currently known to the wallet + + returns: + path generator + """ + for s in self._script_map.values(): + yield s + + def addr_to_path(self, addr): + script = self.addr_to_script(addr) + return self.script_to_path(script) + + def script_to_path(self, script): + assert script in self._script_map + return self._script_map[script] + + def set_next_index(self, mixdepth, internal, index, force=False): + """ + Set the next index to use when generating a new key pair. + + params: + mixdepth: int + internal: 0/False or 1/True + index: int + force: True if you know the wallet already knows all scripts + up to (excluding) the given index + + Warning: improper use of 'force' will cause undefined behavior! + """ + raise NotImplementedError() + + def close(self): + self._storage.close() + + def __del__(self): + self.close() + + +class ImportWalletMixin(object): """ - def wallet_data_to_seed(self, data): - if data is None: - return None - self.mnemonic_extension = None - if isinstance(data, tuple): - entropy, self.mnemonic_extension = data - else: - entropy = data - if get_network() == "testnet": - if entropy.startswith("FAKESEED"): - return entropy[8:] - self.entropy = entropy.decode('hex') - m = Mnemonic("english") - return m.to_seed(m.to_mnemonic(self.entropy), - '' if not self.mnemonic_extension else self.mnemonic_extension).encode('hex') - -class SegwitWallet(Bip39Wallet): - - """This implements an HD wallet (BIP32), - with address type P2SH/P2WPKH of segwit (BIP141), - using BIP39 mnemonics (see BIP39Wallet), - and the structure is intended as an implementation of BIP49, - which is a derivative of BIP44: - https://github.com/bitcoin/bips/blob/master/bip-0049.mediawiki + Mixin for BaseWallet to support importing keys. """ - def __init__(self, seedarg, pwd, max_mix_depth=2, gaplimit=6, - extend_mixdepth=False, storepassword=False, wallet_dir=None): - self.entropy = None - super(SegwitWallet, self).__init__(seedarg, pwd, max_mix_depth, gaplimit, - extend_mixdepth, storepassword, - wallet_dir=wallet_dir) - self.vflag = JM_WALLET_SW_P2SH_P2WPKH - - def get_root_path(self): - testflag = "1'" if get_network() == "testnet" else "0'" - return "m/49'/" + testflag - - def get_mixing_depth_keys(self, master): - pre_root = btc.bip32_ckd(master, 49 + 2**31) - testnet_flag = 1 if get_network() == "testnet" else 0 - root = btc.bip32_ckd(pre_root, testnet_flag + 2**31) - return [btc.bip32_ckd(root, c + 2**31) for c in range(self.max_mix_depth)] - - @staticmethod - def get_vbyte(): - return get_p2sh_vbyte() + _IMPORTED_STORAGE_KEY = b'imported_keys' + _IMPORTED_ROOT_PATH = b'imported' - def get_txtype(self): - """Return string defining wallet type - for purposes of transaction size estimates + def __init__(self, storage, gap_limit=6, merge_algorithm_name=None): + # {mixdepth: [(privkey, type)]} + self._imported = None + # path is (_IMPORTED_ROOT_PATH, mixdepth, key_index) + super(ImportWalletMixin, self).__init__(storage, gap_limit, + merge_algorithm_name) + + def _load_storage(self): + super(ImportWalletMixin, self)._load_storage() + self._imported = collections.defaultdict(list) + for md, keys in self._storage.data[self._IMPORTED_STORAGE_KEY].items(): + md = int(md) + self._imported[md] = keys + for index, (key, key_type) in enumerate(keys): + if not key: + # imported key was removed + continue + assert key_type in self._ENGINES + self._cache_imported_key(md, key, key_type, index) + + def save(self): + import_data = {} + for md in self._imported: + import_data[_int_to_bytestr(md)] = self._imported[md] + self._storage.data[self._IMPORTED_STORAGE_KEY] = import_data + super(ImportWalletMixin, self).save() + + @classmethod + def initialize(cls, storage, network, max_mixdepth=2, timestamp=None, + write=True, **kwargs): + super(ImportWalletMixin, cls).initialize( + storage, network, max_mixdepth, timestamp, write=False, **kwargs) + storage.data[cls._IMPORTED_STORAGE_KEY] = {} + + if write: + storage.save() + + def import_private_key(self, mixdepth, wif, key_type=None): """ - return 'p2sh-p2wpkh' + Import a private key in WIF format. - def get_addr(self, mixing_depth, forchange, i): - """Construct a p2sh-p2wpkh style address for the - keypair corresponding to mixing depth mixing_depth, - branch forchange and index i + args: + mixdepth: int, mixdepth to import key into + wif: str, private key in WIF format + key_type: int, must match a TYPE_* constant of this module, + used to verify against the key type extracted from WIF + + raises: + WalletError: if key's network does not match wallet network + WalletError: if key is not compressed and type is not P2PKH + WalletError: if key_type does not match data from WIF + + returns: + path of imported key """ - pub = btc.privtopub(self.get_key(mixing_depth, forchange, i)) - return btc.pubkey_to_p2sh_p2wpkh_address(pub, magicbyte=self.get_vbyte()) + if not 0 <= mixdepth <= self.max_mixdepth: + raise WalletError("Mixdepth must be positive and at most {}." + "".format(self.max_mixdepth)) + if key_type is not None and key_type not in self._ENGINES: + raise WalletError("Unsupported key type for imported keys.") - @classmethod - def script_to_address(cls, script): - """Return the address for a given output script, - which will be p2sh-p2wpkh for the segwit (currently). - The underlying witness is however invisible at this layer; - so it's just a p2sh address. + privkey, key_type_wif = self._ENGINE.wif_to_privkey(wif) + + # FIXME: there is no established standard for encoding key type in wif + #if key_type is not None and key_type_wif is not None and \ + # key_type != key_type_wif: + # raise WalletError("Expected key type does not match WIF type.") + + # default to wallet key type if not told otherwise + if key_type is None: + key_type = self.TYPE + #key_type = key_type_wif if key_type_wif is not None else self.TYPE + engine = self._ENGINES[key_type] + + if engine.privkey_to_script(privkey) in self._script_map: + raise WalletError("Cannot import key, already in wallet: {}" + "".format(wif)) + + self._imported[mixdepth].append((privkey, key_type)) + return self._cache_imported_key(mixdepth, privkey, key_type, + len(self._imported[mixdepth]) - 1) + + def remove_imported_key(self, script=None, address=None, path=None): """ - return btc.script_to_address(script, cls.get_vbyte()) + Remove an imported key. Arguments are exclusive. + + args: + script: bytes + address: str + path: path + """ + if sum((bool(script), bool(address), bool(path))) != 1: + raise Exception("Only one of script|address|path may be given.") + + if address: + script = self.addr_to_script(address) + if script: + path = self.script_to_path(script) + + if not path: + raise WalletError("Cannot find key in wallet.") + + if not self._is_imported_path(path): + raise WalletError("Cannot remove non-imported key.") + + assert len(path) == 3 + + if not script: + script = self.get_script_path(path) + + # we need to retain indices + self._imported[path[1]][path[2]] = (b'', -1) + + del self._script_map[script] + + def _cache_imported_key(self, mixdepth, privkey, key_type, index): + engine = self._ENGINES[key_type] + path = (self._IMPORTED_ROOT_PATH, mixdepth, index) + + self._script_map[engine.privkey_to_script(privkey)] = path + + return path + + def _get_mixdepth_from_path(self, path): + if not self._is_imported_path(path): + return super(ImportWalletMixin, self)._get_mixdepth_from_path(path) + + assert len(path) == 3 + return path[1] + + def _get_priv_from_path(self, path): + if not self._is_imported_path(path): + return super(ImportWalletMixin, self)._get_priv_from_path(path) + + assert len(path) == 3 + md, i = path[1], path[2] + assert 0 <= md <= self.max_mixdepth + + if len(self._imported[md]) <= i: + raise WalletError("unknown imported key at {}" + "".format(self.get_path_repr(path))) + + key, key_type = self._imported[md][i] + + if key_type == -1: + raise WalletError("imported key was removed") + + return key, self._ENGINES[key_type] @classmethod - def pubkey_to_address(cls, pubkey): - return btc.pubkey_to_p2sh_p2wpkh_address(pubkey, cls.get_vbyte()) - - def sign(self, tx, i, priv, amount): - """Sign a transaction; the amount field - triggers the segwit style signing. - """ - log.debug("About to sign for this amount: " + str(amount)) - return btc.sign(tx, i, priv, amount=amount) - -class BitcoinCoreWallet(AbstractWallet): #pragma: no cover - def __init__(self, fromaccount): - super(BitcoinCoreWallet, self).__init__() - if not isinstance(jm_single().bc_interface, - BitcoinCoreInterface): - raise RuntimeError('Bitcoin Core wallet can only be used when ' - 'blockchain interface is BitcoinCoreInterface') - self.fromaccount = fromaccount - self.max_mix_depth = 1 + def _is_imported_path(cls, path): + return len(path) == 3 and path[0] == cls._IMPORTED_ROOT_PATH - def get_key_from_addr(self, addr): - self.ensure_wallet_unlocked() - wifkey = jm_single().bc_interface.rpc('dumpprivkey', [addr]) - return btc.from_wif_privkey(wifkey, vbyte=get_p2pk_vbyte()) + def path_repr_to_path(self, pathstr): + spath = pathstr.encode('ascii').split(b'/') + if not self._is_imported_path(spath): + return super(ImportWalletMixin, self).path_repr_to_path(pathstr) - def get_utxos_by_mixdepth(self): - unspent_list = jm_single().bc_interface.rpc('listunspent', []) - result = {0: {}} - for u in unspent_list: - if not u['spendable']: - continue - if self.fromaccount and ( - ('account' not in u) or u['account'] != - self.fromaccount): + return self._IMPORTED_ROOT_PATH, int(spath[1]), int(spath[2]) + + def get_path_repr(self, path): + if not self._is_imported_path(path): + return super(ImportWalletMixin, self).get_path_repr(path) + + assert len(path) == 3 + return 'imported/{}/{}'.format(*path[1:]) + + def yield_imported_paths(self, mixdepth): + assert 0 <= mixdepth <= self.max_mixdepth + + for index in range(len(self._imported[mixdepth])): + if self._imported[mixdepth][index][1] == -1: continue - result[0][u['txid'] + ':' + str(u['vout'])] = { - 'address': u['address'], - 'value': int(Decimal(str(u['amount'])) * Decimal('1e8'))} - return result - - def get_internal_addr(self, mixing_depth): - return jm_single().bc_interface.rpc('getrawchangeaddress', []) - - @staticmethod - def ensure_wallet_unlocked(): - wallet_info = jm_single().bc_interface.rpc('getwalletinfo', []) - if 'unlocked_until' in wallet_info and wallet_info[ - 'unlocked_until'] <= 0: - while True: - password = getpass( - 'Enter passphrase to unlock wallet: ') - if password == '': - raise RuntimeError('Aborting wallet unlock') - try: - # TODO cleanly unlock wallet after use, not with arbitrary timeout - jm_single().bc_interface.rpc( - 'walletpassphrase', [password, 10]) - break - except jm_single().JsonRpcError as exc: - if exc.code != -14: - raise exc - # Wrong passphrase, try again. - -def get_wallet_cls(): - if jm_single().config.get("POLICY", "segwit") == "true": - return SegwitWallet - return Wallet + yield (self._IMPORTED_ROOT_PATH, mixdepth, index) + + def get_details(self, path): + if not self._is_imported_path(path): + return super(ImportWalletMixin, self).get_details(path) + return path[1], 'imported', path[2] + + def get_script_path(self, path): + if not self._is_imported_path(path): + return super(ImportWalletMixin, self).get_script_path(path) + + priv, engine = self._get_priv_from_path(path) + return engine.privkey_to_script(priv) + + +class BIP39WalletMixin(object): + """ + Mixin to use BIP-39 mnemonic seed with BIP32Wallet + """ + _BIP39_EXTENSION_KEY = b'seed_extension' + MNEMONIC_LANG = 'english' + + def _load_storage(self): + super(BIP39WalletMixin, self)._load_storage() + self._entropy_extension = self._storage.data.get(self._BIP39_EXTENSION_KEY) + + @classmethod + def initialize(cls, storage, network, max_mixdepth=2, timestamp=None, + entropy=None, entropy_extension=None, write=True, **kwargs): + super(BIP39WalletMixin, cls).initialize( + storage, network, max_mixdepth, timestamp, entropy, + write=False, **kwargs) + if entropy_extension: + storage.data[cls._BIP39_EXTENSION_KEY] = entropy_extension + + if write: + storage.save() + + def _create_master_key(self): + ent, ext = self.get_mnemonic_words() + m = Mnemonic(self.MNEMONIC_LANG) + return m.to_seed(ent, ext or b'') + + @classmethod + def _verify_entropy(cls, ent): + # every 4-bytestream is a valid entropy for BIP-39 + return ent and len(ent) % 4 == 0 + + def get_mnemonic_words(self): + entropy = super(BIP39WalletMixin, self)._create_master_key() + m = Mnemonic(self.MNEMONIC_LANG) + return m.to_mnemonic(entropy), self._entropy_extension + + @classmethod + def entropy_from_mnemonic(cls, seed): + m = Mnemonic(cls.MNEMONIC_LANG) + seed = seed.lower() + if not m.check(seed): + raise WalletError("Invalid mnemonic seed.") + + ent = m.to_entropy(seed) + + if not cls._verify_entropy(ent): + raise WalletError("Seed entropy is too low.") + + return bytes(ent) + + +class BIP32Wallet(BaseWallet): + _STORAGE_ENTROPY_KEY = b'entropy' + _STORAGE_INDEX_CACHE = b'index_cache' + BIP32_MAX_PATH_LEVEL = 2**31 + BIP32_EXT_ID = 0 + BIP32_INT_ID = 1 + ENTROPY_BYTES = 16 + + def __init__(self, storage, gap_limit=6, merge_algorithm_name=None): + self._entropy = None + # {mixdepth: {type: index}} with type being 0/1 for [non]-internal + self._index_cache = None + # path is a tuple of BIP32 levels, + # m is the master key's fingerprint + # other levels are ints + super(BIP32Wallet, self).__init__(storage, gap_limit, + merge_algorithm_name) + assert self._index_cache is not None + assert self._verify_entropy(self._entropy) + + _master_entropy = self._create_master_key() + assert _master_entropy + self._master_key = self._derive_bip32_master_key(_master_entropy) + + # used to verify paths for sanity checking and for wallet id creation + self._key_ident = b'' # otherwise get_bip32_* won't work + self._key_ident = sha256(sha256( + self.get_bip32_priv_export(0, 0).encode('ascii')).digest())\ + .digest()[:3] + self._populate_script_map() + + @classmethod + def initialize(cls, storage, network, max_mixdepth=2, timestamp=None, + entropy=None, write=True): + """ + args: + entropy: ENTROPY_BYTES bytes or None to have wallet generate some + """ + if entropy and not cls._verify_entropy(entropy): + raise WalletError("Invalid entropy.") + + super(BIP32Wallet, cls).initialize(storage, network, max_mixdepth, + timestamp, write=False) + + if not entropy: + entropy = get_random_bytes(cls.ENTROPY_BYTES, True) + + storage.data[cls._STORAGE_ENTROPY_KEY] = entropy + storage.data[cls._STORAGE_INDEX_CACHE] = {} + + if write: + storage.save() + + def _load_storage(self): + super(BIP32Wallet, self)._load_storage() + self._entropy = self._storage.data[self._STORAGE_ENTROPY_KEY] + + self._index_cache = collections.defaultdict( + lambda: collections.defaultdict(int)) + + for md, data in self._storage.data[self._STORAGE_INDEX_CACHE].items(): + md = int(md) + md_map = self._index_cache[md] + for t, k in data.items(): + md_map[int(t)] = k + + def _populate_script_map(self): + for md in self._index_cache: + for int_type in (self.BIP32_EXT_ID, self.BIP32_INT_ID): + for i in range(self._index_cache[md][int_type]): + path = self.get_path(md, int_type, i) + script = self.get_script_path(path) + self._script_map[script] = path + + def save(self): + for md, data in self._index_cache.items(): + str_data = {} + str_md = _int_to_bytestr(md) + + for t, k in data.items(): + str_data[_int_to_bytestr(t)] = k + + self._storage.data[self._STORAGE_INDEX_CACHE][str_md] = str_data + + super(BIP32Wallet, self).save() + + def _create_master_key(self): + """ + for base/legacy wallet type, this is a passthrough. + for bip39 style wallets, this will convert from one to the other + """ + return self._entropy + + @classmethod + def _verify_entropy(cls, ent): + # This is not very useful but true for BIP32. Subclasses may have + # stricter requirements. + return bool(ent) + + @classmethod + def _derive_bip32_master_key(cls, seed): + return cls._ENGINE.derive_bip32_master_key(seed) + + def get_script_path(self, path): + if not self._is_my_bip32_path(path): + raise WalletError("unable to get script for unknown key path") + + md, int_type, index = self.get_details(path) + + if not 0 <= md <= self.max_mixdepth: + raise WalletError("Mixdepth outside of wallet's range.") + assert int_type in (self.BIP32_EXT_ID, self.BIP32_INT_ID) + + current_index = self._index_cache[md][int_type] + + if index == current_index: + return self.get_new_script(md, int_type) + + priv, engine = self._get_priv_from_path(path) + script = engine.privkey_to_script(priv) + + return script + + def get_path(self, mixdepth=None, internal=None, index=None): + if mixdepth is not None: + assert isinstance(mixdepth, int) + if not 0 <= mixdepth <= self.max_mixdepth: + raise WalletError("Mixdepth outside of wallet's range.") + + if internal is not None: + if mixdepth is None: + raise Exception("mixdepth must be set if internal is set") + int_type = self._get_internal_type(internal) + + if index is not None: + assert isinstance(index, int) + if internal is None: + raise Exception("internal must be set if index is set") + assert index <= self._index_cache[mixdepth][int_type] + assert index < self.BIP32_MAX_PATH_LEVEL + return tuple(chain(self._get_bip32_export_path(mixdepth, internal), + (index,))) + + return tuple(self._get_bip32_export_path(mixdepth, internal)) + + def get_path_repr(self, path): + path = list(path) + assert self._is_my_bip32_path(path) + path.pop(0) + return 'm' + '/' + '/'.join(map(self._path_level_to_repr, path)) + + @classmethod + def _harden_path_level(cls, lvl): + assert isinstance(lvl, int) + if not 0 <= lvl < cls.BIP32_MAX_PATH_LEVEL: + raise WalletError("Unable to derive hardened path level from {}." + "".format(lvl)) + return lvl + cls.BIP32_MAX_PATH_LEVEL + + @classmethod + def _path_level_to_repr(cls, lvl): + assert isinstance(lvl, int) + if not 0 <= lvl < cls.BIP32_MAX_PATH_LEVEL * 2: + raise WalletError("Invalid path level {}.".format(lvl)) + if lvl < cls.BIP32_MAX_PATH_LEVEL: + return str(lvl) + return str(lvl - cls.BIP32_MAX_PATH_LEVEL) + "'" + + def path_repr_to_path(self, pathstr): + spath = pathstr.split('/') + assert len(spath) > 0 + if spath[0] != 'm': + raise WalletError("Not a valid wallet path: {}".format(pathstr)) + + def conv_level(lvl): + if lvl[-1] == "'": + return self._harden_path_level(int(lvl[:-1])) + return int(lvl) + + return tuple(chain((self._key_ident,), map(conv_level, spath[1:]))) + + def _get_mixdepth_from_path(self, path): + if not self._is_my_bip32_path(path): + raise WalletError("Invalid path, unknown root: {}".format(path)) + + return path[len(self._get_bip32_base_path())] + + def _get_priv_from_path(self, path): + if not self._is_my_bip32_path(path): + raise WalletError("Invalid path, unknown root: {}".format(path)) + + return self._ENGINE.derive_bip32_privkey(self._master_key, path), \ + self._ENGINE + + def _is_my_bip32_path(self, path): + return path[0] == self._key_ident + + def get_new_script(self, mixdepth, internal): + # This is called by get_script_path and calls back there. We need to + # ensure all conditions match to avoid endless recursion. + int_type = self._get_internal_type(internal) + index = self._index_cache[mixdepth][int_type] + self._index_cache[mixdepth][int_type] += 1 + path = self.get_path(mixdepth, int_type, index) + script = self.get_script_path(path) + self._script_map[script] = path + return script + + def get_script(self, mixdepth, internal, index): + path = self.get_path(mixdepth, internal, index) + return self.get_script_path(path) + + @deprecated + def get_key(self, mixdepth, internal, index): + int_type = self._get_internal_type(internal) + path = self.get_path(mixdepth, int_type, index) + priv = self._ENGINE.derive_bip32_privkey(self._master_key, path) + return hexlify(priv) + + def get_bip32_priv_export(self, mixdepth=None, internal=None): + path = self._get_bip32_export_path(mixdepth, internal) + return self._ENGINE.derive_bip32_priv_export(self._master_key, path) + + def get_bip32_pub_export(self, mixdepth=None, internal=None): + path = self._get_bip32_export_path(mixdepth, internal) + return self._ENGINE.derive_bip32_pub_export(self._master_key, path) + + def _get_bip32_export_path(self, mixdepth=None, internal=None): + if mixdepth is None: + assert internal is None + path = tuple() + else: + assert 0 <= mixdepth <= self.max_mixdepth + if internal is None: + path = (self._get_bip32_mixdepth_path_level(mixdepth),) + else: + int_type = self._get_internal_type(internal) + path = (self._get_bip32_mixdepth_path_level(mixdepth), int_type) + + return tuple(chain(self._get_bip32_base_path(), path)) + + def _get_bip32_base_path(self): + return self._key_ident, + + @classmethod + def _get_bip32_mixdepth_path_level(cls, mixdepth): + return mixdepth + + def _get_internal_type(self, is_internal): + return self.BIP32_INT_ID if is_internal else self.BIP32_EXT_ID + + def get_next_unused_index(self, mixdepth, internal): + assert 0 <= mixdepth <= self.max_mixdepth + int_type = self._get_internal_type(internal) + + if self._index_cache[mixdepth][int_type] >= self.BIP32_MAX_PATH_LEVEL: + # FIXME: theoretically this should work for up to + # self.BIP32_MAX_PATH_LEVEL * 2, no? + raise WalletError("All addresses used up, cannot generate new ones.") + + return self._index_cache[mixdepth][int_type] + + def get_mnemonic_words(self): + return ' '.join(mn_encode(hexlify(self._entropy))), None + + @classmethod + def entropy_from_mnemonic(cls, seed): + words = seed.split() + if len(words) != 12: + raise WalletError("Seed phrase must consist of exactly 12 words.") + + return unhexlify(mn_decode(words)) + + def get_wallet_id(self): + return hexlify(self._key_ident) + + def set_next_index(self, mixdepth, internal, index, force=False): + int_type = self._get_internal_type(internal) + if not (force or index <= self._index_cache[mixdepth][int_type]): + raise Exception("cannot advance index without force=True") + self._index_cache[mixdepth][int_type] = index + + def get_details(self, path): + if not self._is_my_bip32_path(path): + raise Exception("path does not belong to wallet") + return self._get_mixdepth_from_path(path), path[-2], path[-1] + + +class LegacyWallet(ImportWalletMixin, BIP32Wallet): + TYPE = TYPE_P2PKH + _ENGINE = BTC_P2PKH + + def _create_master_key(self): + return hexlify(self._entropy).encode('ascii') + + def _get_bip32_base_path(self): + return self._key_ident, 0 + + +class BIP49Wallet(BIP32Wallet): + _BIP49_PURPOSE = 2**31 + 49 + _ENGINE = BTC_P2SH_P2WPKH + + def _get_bip32_base_path(self): + return self._key_ident, self._BIP49_PURPOSE,\ + self._ENGINE.BIP44_COIN_TYPE + + @classmethod + def _get_bip32_mixdepth_path_level(cls, mixdepth): + assert 0 <= mixdepth < 2**31 + return cls._harden_path_level(mixdepth) + + def _get_mixdepth_from_path(self, path): + if not self._is_my_bip32_path(path): + raise WalletError("Invalid path, unknown root: {}".format(path)) + + return path[len(self._get_bip32_base_path())] - 2**31 + + +class SegwitLegacyWallet(ImportWalletMixin, BIP39WalletMixin, BIP49Wallet): + TYPE = TYPE_P2SH_P2WPKH + + +WALLET_IMPLEMENTATIONS = { + LegacyWallet.TYPE: LegacyWallet, + SegwitLegacyWallet.TYPE: SegwitLegacyWallet +} diff --git a/jmclient/test/test_argon2.py b/jmclient/test/test_argon2.py new file mode 100644 index 0000000..7de4b78 --- /dev/null +++ b/jmclient/test/test_argon2.py @@ -0,0 +1,35 @@ +from __future__ import print_function, absolute_import, division, unicode_literals + +from jmclient import Argon2Hash, get_random_bytes +import pytest + + +def test_argon2_sanity(): + pwd = b'password' + salt = b'saltsalt' + + h = Argon2Hash(pwd, salt, 16) + + assert len(h.hash) == 16 + assert h.salt == salt + assert h.hash == b'\x05;V\xd7fy\xdfI\xa4\xe7F$_\\3\xcb' + + +def test_get_random_bytes(): + assert len(get_random_bytes(16)) == 16 + assert get_random_bytes(16) != get_random_bytes(16) + + +def test_argon2(): + pwd = b'testpass' + h = Argon2Hash(pwd, hash_len=16, salt_len=22) + + assert len(h.hash) == 16 + assert len(h.salt) == 22 + + h2 = Argon2Hash(pwd, h.salt, hash_len=16) + + assert h.settings == h2.settings + assert h.hash == h2.hash + assert h.salt == h2.salt + diff --git a/jmclient/test/test_blockchaininterface.py b/jmclient/test/test_blockchaininterface.py new file mode 100644 index 0000000..58fe400 --- /dev/null +++ b/jmclient/test/test_blockchaininterface.py @@ -0,0 +1,149 @@ +from __future__ import absolute_import, print_function + +"""Blockchaininterface functionality tests.""" + +import binascii +from commontest import create_wallet_for_sync, make_sign_and_push + +import pytest +from jmclient import load_program_config, jm_single, sync_wallet, get_log + +log = get_log() + + +def sync_test_wallet(fast, wallet): + sync_count = 0 + jm_single().bc_interface.wallet_synced = False + while not jm_single().bc_interface.wallet_synced: + sync_wallet(wallet, fast=fast) + sync_count += 1 + # avoid infinite loop + assert sync_count < 10 + log.debug("Tried " + str(sync_count) + " times") + + +@pytest.mark.parametrize('fast', (False, True)) +def test_empty_wallet_sync(setup_wallets, fast): + wallet = create_wallet_for_sync([0, 0, 0, 0, 0], ['test_empty_wallet_sync']) + + sync_test_wallet(fast, wallet) + + broken = True + for md in range(wallet.max_mixdepth + 1): + for internal in (True, False): + broken = False + assert 0 == wallet.get_next_unused_index(md, internal) + assert not broken + + +@pytest.mark.parametrize('fast,internal', ( + (False, False), (False, True), + (True, False), (True, True))) +def test_sequentially_used_wallet_sync(setup_wallets, fast, internal): + used_count = [1, 3, 6, 2, 23] + wallet = create_wallet_for_sync( + used_count, ['test_sequentially_used_wallet_sync'], + populate_internal=internal) + + sync_test_wallet(fast, wallet) + + broken = True + for md in range(len(used_count)): + broken = False + assert used_count[md] == wallet.get_next_unused_index(md, internal) + assert not broken + + +@pytest.mark.parametrize('fast', (False, True)) +def test_gap_used_wallet_sync(setup_wallets, fast): + used_count = [1, 3, 6, 2, 23] + wallet = create_wallet_for_sync(used_count, ['test_gap_used_wallet_sync']) + wallet.gap_limit = 20 + + for md in range(len(used_count)): + x = -1 + for x in range(md): + assert x <= wallet.gap_limit, "test broken" + # create some unused addresses + wallet.get_new_script(md, True) + wallet.get_new_script(md, False) + used_count[md] += x + 2 + jm_single().bc_interface.grab_coins(wallet.get_new_addr(md, True), 1) + jm_single().bc_interface.grab_coins(wallet.get_new_addr(md, False), 1) + + # reset indices to simulate completely unsynced wallet + for md in range(wallet.max_mixdepth + 1): + wallet.set_next_index(md, True, 0) + wallet.set_next_index(md, False, 0) + + sync_test_wallet(fast, wallet) + + broken = True + for md in range(len(used_count)): + broken = False + assert md + 1 == wallet.get_next_unused_index(md, True) + assert used_count[md] == wallet.get_next_unused_index(md, False) + assert not broken + + +@pytest.mark.parametrize('fast', (False, True)) +def test_multigap_used_wallet_sync(setup_wallets, fast): + start_index = 5 + used_count = [start_index, 0, 0, 0, 0] + wallet = create_wallet_for_sync(used_count, ['test_multigap_used_wallet_sync']) + wallet.gap_limit = 5 + + mixdepth = 0 + for w in range(5): + for x in range(int(wallet.gap_limit * 0.6)): + assert x <= wallet.gap_limit, "test broken" + # create some unused addresses + wallet.get_new_script(mixdepth, True) + wallet.get_new_script(mixdepth, False) + used_count[mixdepth] += x + 2 + jm_single().bc_interface.grab_coins(wallet.get_new_addr(mixdepth, True), 1) + jm_single().bc_interface.grab_coins(wallet.get_new_addr(mixdepth, False), 1) + + # reset indices to simulate completely unsynced wallet + for md in range(wallet.max_mixdepth + 1): + wallet.set_next_index(md, True, 0) + wallet.set_next_index(md, False, 0) + + sync_test_wallet(fast, wallet) + + assert used_count[mixdepth] - start_index == wallet.get_next_unused_index(mixdepth, True) + assert used_count[mixdepth] == wallet.get_next_unused_index(mixdepth, False) + + +@pytest.mark.parametrize('fast', (False, True)) +def test_retain_unused_indices_wallet_sync(setup_wallets, fast): + used_count = [0, 0, 0, 0, 0] + wallet = create_wallet_for_sync(used_count, ['test_retain_unused_indices_wallet_sync']) + + for x in range(9): + wallet.get_new_script(0, 1) + + sync_test_wallet(fast, wallet) + + assert wallet.get_next_unused_index(0, 1) == 9 + + +@pytest.mark.parametrize('fast', (False, True)) +def test_imported_wallet_sync(setup_wallets, fast): + used_count = [0, 0, 0, 0, 0] + wallet = create_wallet_for_sync(used_count, ['test_imported_wallet_sync']) + source_wallet = create_wallet_for_sync(used_count, ['test_imported_wallet_sync_origin']) + + address = source_wallet.get_new_addr(0, 1) + wallet.import_private_key(0, source_wallet.get_wif(0, 1, 0)) + txid = binascii.unhexlify(jm_single().bc_interface.grab_coins(address, 1)) + + sync_test_wallet(fast, wallet) + + assert wallet._utxos.have_utxo(txid, 0) == 0 + + +@pytest.fixture(scope='module') +def setup_wallets(): + load_program_config() + jm_single().bc_interface.tick_forward_chain_interval = 1 diff --git a/jmclient/test/test_coinjoin.py b/jmclient/test/test_coinjoin.py index 3137048..5498fac 100644 --- a/jmclient/test/test_coinjoin.py +++ b/jmclient/test/test_coinjoin.py @@ -10,10 +10,11 @@ import pytest from twisted.internet import reactor from jmclient import load_program_config, jm_single, get_log,\ - YieldGeneratorBasic, Taker, sync_wallet + YieldGeneratorBasic, Taker, sync_wallet, LegacyWallet, SegwitLegacyWallet from jmclient.podle import set_commitment_file -from commontest import make_wallets +from commontest import make_wallets, binarize_tx from test_taker import dummy_filter_orderbook +import jmbitcoin as btc testdir = os.path.dirname(os.path.realpath(__file__)) log = get_log() @@ -51,8 +52,11 @@ def create_orderbook(makers): def create_taker(wallet, schedule, monkeypatch): def on_finished_callback(*args, **kwargs): + log.debug("on finished called with: {}, {}".format(args, kwargs)) + on_finished_callback.status = args[0] on_finished_callback.called = True on_finished_callback.called = False + on_finished_callback.status = None taker = Taker(wallet, schedule, callbacks=(dummy_filter_orderbook, None, on_finished_callback)) @@ -104,7 +108,8 @@ def do_tx_signing(taker, makers, active_orders, txdata): return taker_final_result -def test_simple_coinjoin(monkeypatch, tmpdir, setup_cj): +@pytest.mark.parametrize('wallet_cls', (LegacyWallet, SegwitLegacyWallet)) +def test_simple_coinjoin(monkeypatch, tmpdir, setup_cj, wallet_cls): def raise_exit(i): raise Exception("sys.exit called") monkeypatch.setattr(sys, 'exit', raise_exit) @@ -113,7 +118,7 @@ def test_simple_coinjoin(monkeypatch, tmpdir, setup_cj): MAKER_NUM = 3 wallets = make_wallets_to_list(make_wallets( MAKER_NUM + 1, wallet_structures=[[4, 0, 0, 0, 0]] * (MAKER_NUM + 1), - mean_amt=1)) + mean_amt=1, wallet_cls=wallet_cls)) jm_single().bc_interface.tickchain() sync_wallets(wallets) @@ -138,6 +143,115 @@ def test_simple_coinjoin(monkeypatch, tmpdir, setup_cj): taker_final_result = do_tx_signing(taker, makers, active_orders, txdata) assert taker_final_result is not False + assert taker.on_finished_callback.status is not False + + +def test_coinjoin_mixdepth_wrap_taker(monkeypatch, tmpdir, setup_cj): + def raise_exit(i): + raise Exception("sys.exit called") + monkeypatch.setattr(sys, 'exit', raise_exit) + set_commitment_file(str(tmpdir.join('commitments.json'))) + + MAKER_NUM = 3 + wallets = make_wallets_to_list(make_wallets( + MAKER_NUM + 1, + wallet_structures=[[4, 0, 0, 0, 0]] * MAKER_NUM + [[0, 0, 0, 0, 3]], + mean_amt=1)) + + for w in wallets: + assert w.max_mixdepth == 4 + + jm_single().bc_interface.tickchain() + jm_single().bc_interface.tickchain() + sync_wallets(wallets) + + cj_fee = 2000 + makers = [YieldGeneratorBasic( + wallets[i], + [0, cj_fee, 0, 'swabsoffer', 10**7]) for i in range(MAKER_NUM)] + + orderbook = create_orderbook(makers) + assert len(orderbook) == MAKER_NUM + + cj_amount = int(1.1 * 10**8) + # mixdepth, amount, counterparties, dest_addr, waittime + schedule = [(4, cj_amount, MAKER_NUM, 'INTERNAL', 0)] + taker = create_taker(wallets[-1], schedule, monkeypatch) + + active_orders, maker_data = init_coinjoin(taker, makers, + orderbook, cj_amount) + + txdata = taker.receive_utxos(maker_data) + assert txdata[0], "taker.receive_utxos error" + + taker_final_result = do_tx_signing(taker, makers, active_orders, txdata) + assert taker_final_result is not False + + tx = btc.deserialize(txdata[2]) + binarize_tx(tx) + + w = wallets[-1] + w.remove_old_utxos_(tx) + w.add_new_utxos_(tx, b'\x00' * 32) # fake txid + + balances = w.get_balance_by_mixdepth() + assert balances[0] == cj_amount + # <= because of tx fee + assert balances[4] <= 3 * 10**8 - cj_amount - (cj_fee * MAKER_NUM) + + +def test_coinjoin_mixdepth_wrap_maker(monkeypatch, tmpdir, setup_cj): + def raise_exit(i): + raise Exception("sys.exit called") + monkeypatch.setattr(sys, 'exit', raise_exit) + set_commitment_file(str(tmpdir.join('commitments.json'))) + + MAKER_NUM = 2 + wallets = make_wallets_to_list(make_wallets( + MAKER_NUM + 1, + wallet_structures=[[0, 0, 0, 0, 4]] * MAKER_NUM + [[3, 0, 0, 0, 0]], + mean_amt=1)) + + for w in wallets: + assert w.max_mixdepth == 4 + + jm_single().bc_interface.tickchain() + jm_single().bc_interface.tickchain() + sync_wallets(wallets) + + cj_fee = 2000 + makers = [YieldGeneratorBasic( + wallets[i], + [0, cj_fee, 0, 'swabsoffer', 10**7]) for i in range(MAKER_NUM)] + + orderbook = create_orderbook(makers) + assert len(orderbook) == MAKER_NUM + + cj_amount = int(1.1 * 10**8) + # mixdepth, amount, counterparties, dest_addr, waittime + schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0)] + taker = create_taker(wallets[-1], schedule, monkeypatch) + + active_orders, maker_data = init_coinjoin(taker, makers, + orderbook, cj_amount) + + txdata = taker.receive_utxos(maker_data) + assert txdata[0], "taker.receive_utxos error" + + taker_final_result = do_tx_signing(taker, makers, active_orders, txdata) + assert taker_final_result is not False + + tx = btc.deserialize(txdata[2]) + binarize_tx(tx) + + for i in range(MAKER_NUM): + w = wallets[i] + w.remove_old_utxos_(tx) + w.add_new_utxos_(tx, b'\x00' * 32) # fake txid + + balances = w.get_balance_by_mixdepth() + assert balances[0] == cj_amount + assert balances[4] == 4 * 10**8 - cj_amount + cj_fee @pytest.fixture(scope='module') diff --git a/jmclient/test/test_storage.py b/jmclient/test/test_storage.py new file mode 100644 index 0000000..1c54304 --- /dev/null +++ b/jmclient/test/test_storage.py @@ -0,0 +1,128 @@ +from __future__ import print_function, absolute_import, division, unicode_literals + +from jmclient import storage +import pytest + + +class MockStorage(storage.Storage): + def __init__(self, data, *args, **kwargs): + self.file_data = data + self.locked = False + super(type(self), self).__init__(*args, **kwargs) + + def _read_file(self): + if hasattr(self, 'file_data'): + return self.file_data + return b'' + + def _write_file(self, data): + self.file_data = data + + def _create_lock(self): + self.locked = not self.read_only + + def _remove_lock(self): + self.locked = False + + +def test_storage(): + s = MockStorage(None, 'nonexistant', b'password', create=True) + assert s.file_data.startswith(s.MAGIC_ENC) + assert s.locked + assert s.is_encrypted() + assert not s.was_changed() + + old_data = s.file_data + + s.data[b'mydata'] = b'test' + assert s.was_changed() + s.save() + assert s.file_data != old_data + enc_data = s.file_data + + old_data = s.file_data + s.change_password(b'newpass') + assert s.is_encrypted() + assert not s.was_changed() + assert s.file_data != old_data + + old_data = s.file_data + s.change_password(None) + assert not s.is_encrypted() + assert not s.was_changed() + assert s.file_data != old_data + assert s.file_data.startswith(s.MAGIC_UNENC) + + s2 = MockStorage(enc_data, __file__, b'password') + assert s2.locked + assert s2.is_encrypted() + assert not s2.was_changed() + assert s2.data[b'mydata'] == b'test' + + +def test_storage_invalid(): + with pytest.raises(storage.StorageError, message="File does not exist"): + MockStorage(None, 'nonexistant', b'password') + + s = MockStorage(None, 'nonexistant', b'password', create=True) + with pytest.raises(storage.StorageError, message="Wrong password"): + MockStorage(s.file_data, __file__, b'wrongpass') + + with pytest.raises(storage.StorageError, message="No password"): + MockStorage(s.file_data, __file__) + + with pytest.raises(storage.StorageError, message="Non-wallet file, unencrypted"): + MockStorage(b'garbagefile', __file__) + + with pytest.raises(storage.StorageError, message="Non-wallet file, encrypted"): + MockStorage(b'garbagefile', __file__, b'password') + + +def test_storage_readonly(): + s = MockStorage(None, 'nonexistant', b'password', create=True) + s = MockStorage(s.file_data, __file__, b'password', read_only=True) + s.data[b'mydata'] = b'test' + + assert not s.locked + assert s.was_changed() + + with pytest.raises(storage.StorageError): + s.save() + + with pytest.raises(storage.StorageError): + s.change_password(b'newpass') + + +def test_storage_lock(tmpdir): + p = str(tmpdir.join('test.jmdat')) + pw = None + + with pytest.raises(storage.StorageError, message="File does not exist"): + storage.Storage(p, pw) + + s = storage.Storage(p, pw, create=True) + assert s.is_locked() + assert not s.is_encrypted() + assert s.data == {} + + with pytest.raises(storage.StorageError, message="File is locked"): + storage.Storage(p, pw) + + assert storage.Storage.is_storage_file(p) + assert not storage.Storage.is_encrypted_storage_file(p) + + s.data[b'test'] = b'value' + s.save() + s.close() + del s + + s = storage.Storage(p, pw, read_only=True) + assert not s.is_locked() + assert s.data == {b'test': b'value'} + s.close() + del s + + s = storage.Storage(p, pw) + assert s.is_locked() + assert s.data == {b'test': b'value'} + diff --git a/jmclient/test/test_utxomanager.py b/jmclient/test/test_utxomanager.py new file mode 100644 index 0000000..2c64e8e --- /dev/null +++ b/jmclient/test/test_utxomanager.py @@ -0,0 +1,93 @@ +from __future__ import print_function, absolute_import, division, unicode_literals + +from jmclient.wallet import UTXOManager +from test_storage import MockStorage +import pytest + +from jmclient import load_program_config +import jmclient +from commontest import DummyBlockchainInterface + + +def select(unspent, value): + return unspent + + +def test_utxomanager_persist(setup_env_nodeps): + storage = MockStorage(None, 'wallet.jmdat', None, create=True) + UTXOManager.initialize(storage) + um = UTXOManager(storage, select) + + txid = b'\x00' * UTXOManager.TXID_LEN + index = 0 + path = (0,) + mixdepth = 0 + value = 500 + + um.add_utxo(txid, index, path, value, mixdepth) + um.add_utxo(txid, index+1, path, value, mixdepth+1) + + um.save() + del um + + um = UTXOManager(storage, select) + + assert um.have_utxo(txid, index) is mixdepth + assert um.have_utxo(txid, index+1) is mixdepth + 1 + assert um.have_utxo(txid, index+2) is False + + utxos = um.get_utxos_by_mixdepth() + assert len(utxos[mixdepth]) == 1 + assert len(utxos[mixdepth+1]) == 1 + assert len(utxos[mixdepth+2]) == 0 + + balances = um.get_balance_by_mixdepth() + assert balances[mixdepth] == value + assert balances[mixdepth+1] == value + + um.remove_utxo(txid, index, mixdepth) + assert um.have_utxo(txid, index) is False + + um.save() + del um + + um = UTXOManager(storage, select) + + assert um.have_utxo(txid, index) is False + assert um.have_utxo(txid, index+1) is mixdepth + 1 + + utxos = um.get_utxos_by_mixdepth() + assert len(utxos[mixdepth]) == 0 + assert len(utxos[mixdepth+1]) == 1 + + balances = um.get_balance_by_mixdepth() + assert balances[mixdepth] == 0 + assert balances[mixdepth+1] == value + assert balances[mixdepth+2] == 0 + + +def test_utxomanager_select(setup_env_nodeps): + storage = MockStorage(None, 'wallet.jmdat', None, create=True) + UTXOManager.initialize(storage) + um = UTXOManager(storage, select) + + txid = b'\x00' * UTXOManager.TXID_LEN + index = 0 + path = (0,) + mixdepth = 0 + value = 500 + + um.add_utxo(txid, index, path, value, mixdepth) + + assert len(um.select_utxos(mixdepth, value)) is 1 + assert len(um.select_utxos(mixdepth+1, value)) is 0 + + um.add_utxo(txid, index+1, path, value, mixdepth) + assert len(um.select_utxos(mixdepth, value)) is 2 + + +@pytest.fixture +def setup_env_nodeps(monkeypatch): + monkeypatch.setattr(jmclient.configure, 'get_blockchain_interface_instance', + lambda x: DummyBlockchainInterface()) + load_program_config() diff --git a/jmclient/test/test_wallet.py b/jmclient/test/test_wallet.py new file mode 100644 index 0000000..7031101 --- /dev/null +++ b/jmclient/test/test_wallet.py @@ -0,0 +1,590 @@ +from __future__ import print_function, absolute_import, division, unicode_literals +'''Wallet functionality tests.''' + +import os +import json +from binascii import hexlify, unhexlify + +import pytest +import jmbitcoin as btc +from commontest import binarize_tx +from jmclient import load_program_config, jm_single, get_log,\ + SegwitLegacyWallet,BIP32Wallet, BIP49Wallet, LegacyWallet,\ + VolatileStorage, get_network, cryptoengine, WalletError + +testdir = os.path.dirname(os.path.realpath(__file__)) +log = get_log() + + +def signed_tx_is_segwit(tx): + for inp in tx['ins']: + if 'txinwitness' not in inp: + return False + return True + + +def assert_segwit(tx): + assert signed_tx_is_segwit(tx) + + +def assert_not_segwit(tx): + assert not signed_tx_is_segwit(tx) + + +def get_populated_wallet(amount=10**8, num=3): + storage = VolatileStorage() + SegwitLegacyWallet.initialize(storage, get_network()) + wallet = SegwitLegacyWallet(storage) + + # fund three wallet addresses at mixdepth 0 + for i in range(num): + fund_wallet_addr(wallet, wallet.get_internal_addr(0), amount / 10**8) + + return wallet + + +def fund_wallet_addr(wallet, addr, value_btc=1): + txin_id = jm_single().bc_interface.grab_coins(addr, value_btc) + txinfo = jm_single().bc_interface.rpc('gettransaction', [txin_id]) + txin = btc.deserialize(unhexlify(txinfo['hex'])) + utxo_in = wallet.add_new_utxos_(txin, unhexlify(txin_id)) + assert len(utxo_in) == 1 + return list(utxo_in.keys())[0] + + +def get_bip39_vectors(): + fh = open(os.path.join(testdir, 'bip39vectors.json')) + data = json.load(fh)['english'] + fh.close() + return data + + +@pytest.mark.parametrize('entropy,mnemonic,key,xpriv', get_bip39_vectors()) +def test_bip39_seeds(monkeypatch, setup_wallet, entropy, mnemonic, key, xpriv): + jm_single().config.set('BLOCKCHAIN', 'network', 'mainnet') + created_entropy = SegwitLegacyWallet.entropy_from_mnemonic(mnemonic) + assert entropy == hexlify(created_entropy) + storage = VolatileStorage() + SegwitLegacyWallet.initialize( + storage, get_network(), entropy=created_entropy, + entropy_extension=b'TREZOR', max_mixdepth=4) + wallet = SegwitLegacyWallet(storage) + assert (mnemonic, b'TREZOR') == wallet.get_mnemonic_words() + assert key == hexlify(wallet._create_master_key()) + + # need to monkeypatch this, else we'll default to the BIP-49 path + monkeypatch.setattr(SegwitLegacyWallet, '_get_bip32_base_path', + BIP32Wallet._get_bip32_base_path) + assert xpriv == wallet.get_bip32_priv_export() + + +def test_bip49_seed(monkeypatch, setup_wallet): + jm_single().config.set('BLOCKCHAIN', 'network', 'testnet') + mnemonic = 'abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about' + master_xpriv = 'tprv8ZgxMBicQKsPe5YMU9gHen4Ez3ApihUfykaqUorj9t6FDqy3nP6eoXiAo2ssvpAjoLroQxHqr3R5nE3a5dU3DHTjTgJDd7zrbniJr6nrCzd' + account0_xpriv = 'tprv8gRrNu65W2Msef2BdBSUgFdRTGzC8EwVXnV7UGS3faeXtuMVtGfEdidVeGbThs4ELEoayCAzZQ4uUji9DUiAs7erdVskqju7hrBcDvDsdbY' + addr0_script_hash = '336caa13e08b96080a32b5d818d59b4ab3b36742' + + entropy = SegwitLegacyWallet.entropy_from_mnemonic(mnemonic) + storage = VolatileStorage() + SegwitLegacyWallet.initialize( + storage, get_network(), entropy=entropy, max_mixdepth=0) + wallet = SegwitLegacyWallet(storage) + assert (mnemonic, None) == wallet.get_mnemonic_words() + assert account0_xpriv == wallet.get_bip32_priv_export(0) + assert addr0_script_hash == hexlify(wallet.get_external_script(0)[2:-1]) + + # FIXME: is this desired behaviour? BIP49 wallet will not return xpriv for + # the root key but only for key after base path + monkeypatch.setattr(SegwitLegacyWallet, '_get_bip32_base_path', + BIP32Wallet._get_bip32_base_path) + assert master_xpriv == wallet.get_bip32_priv_export() + + +def test_bip32_test_vector_1(monkeypatch, setup_wallet): + jm_single().config.set('BLOCKCHAIN', 'network', 'mainnet') + + entropy = unhexlify('000102030405060708090a0b0c0d0e0f') + storage = VolatileStorage() + LegacyWallet.initialize( + storage, get_network(), entropy=entropy, max_mixdepth=0) + + # test vector 1 is using hardened derivation for the account/mixdepth level + monkeypatch.setattr(LegacyWallet, '_get_mixdepth_from_path', + BIP49Wallet._get_mixdepth_from_path) + monkeypatch.setattr(LegacyWallet, '_get_bip32_mixdepth_path_level', + BIP49Wallet._get_bip32_mixdepth_path_level) + monkeypatch.setattr(LegacyWallet, '_get_bip32_base_path', + BIP32Wallet._get_bip32_base_path) + monkeypatch.setattr(LegacyWallet, '_create_master_key', + BIP32Wallet._create_master_key) + + wallet = LegacyWallet(storage) + + assert wallet.get_bip32_priv_export() == 'xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi' + assert wallet.get_bip32_pub_export() == 'xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8' + assert wallet.get_bip32_priv_export(0) == 'xprv9uHRZZhk6KAJC1avXpDAp4MDc3sQKNxDiPvvkX8Br5ngLNv1TxvUxt4cV1rGL5hj6KCesnDYUhd7oWgT11eZG7XnxHrnYeSvkzY7d2bhkJ7' + assert wallet.get_bip32_pub_export(0) == 'xpub68Gmy5EdvgibQVfPdqkBBCHxA5htiqg55crXYuXoQRKfDBFA1WEjWgP6LHhwBZeNK1VTsfTFUHCdrfp1bgwQ9xv5ski8PX9rL2dZXvgGDnw' + assert wallet.get_bip32_priv_export(0, 1) == 'xprv9wTYmMFdV23N2TdNG573QoEsfRrWKQgWeibmLntzniatZvR9BmLnvSxqu53Kw1UmYPxLgboyZQaXwTCg8MSY3H2EU4pWcQDnRnrVA1xe8fs' + assert wallet.get_bip32_pub_export(0, 1) == 'xpub6ASuArnXKPbfEwhqN6e3mwBcDTgzisQN1wXN9BJcM47sSikHjJf3UFHKkNAWbWMiGj7Wf5uMash7SyYq527Hqck2AxYysAA7xmALppuCkwQ' + # there are more test vectors but those don't match joinmarket's wallet + # structure, hence they make litte sense to test here + + +def test_bip32_test_vector_2(monkeypatch, setup_wallet): + jm_single().config.set('BLOCKCHAIN', 'network', 'mainnet') + + entropy = unhexlify('fffcf9f6f3f0edeae7e4e1dedbd8d5d2cfccc9c6c3c0bdbab7b4b1aeaba8a5a29f9c999693908d8a8784817e7b7875726f6c696663605d5a5754514e4b484542') + storage = VolatileStorage() + LegacyWallet.initialize( + storage, get_network(), entropy=entropy, max_mixdepth=0) + + monkeypatch.setattr(LegacyWallet, '_get_bip32_base_path', + BIP32Wallet._get_bip32_base_path) + monkeypatch.setattr(LegacyWallet, '_create_master_key', + BIP32Wallet._create_master_key) + + wallet = LegacyWallet(storage) + + assert wallet.get_bip32_priv_export() == 'xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U' + assert wallet.get_bip32_pub_export() == 'xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB' + assert wallet.get_bip32_priv_export(0) == 'xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt' + assert wallet.get_bip32_pub_export(0) == 'xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH' + # there are more test vectors but those don't match joinmarket's wallet + # structure, hence they make litte sense to test here + + +def test_bip32_test_vector_3(monkeypatch, setup_wallet): + jm_single().config.set('BLOCKCHAIN', 'network', 'mainnet') + + entropy = unhexlify('4b381541583be4423346c643850da4b320e46a87ae3d2a4e6da11eba819cd4acba45d239319ac14f863b8d5ab5a0d0c64d2e8a1e7d1457df2e5a3c51c73235be') + storage = VolatileStorage() + LegacyWallet.initialize( + storage, get_network(), entropy=entropy, max_mixdepth=0) + + # test vector 3 is using hardened derivation for the account/mixdepth level + monkeypatch.setattr(LegacyWallet, '_get_mixdepth_from_path', + BIP49Wallet._get_mixdepth_from_path) + monkeypatch.setattr(LegacyWallet, '_get_bip32_mixdepth_path_level', + BIP49Wallet._get_bip32_mixdepth_path_level) + monkeypatch.setattr(LegacyWallet, '_get_bip32_base_path', + BIP32Wallet._get_bip32_base_path) + monkeypatch.setattr(LegacyWallet, '_create_master_key', + BIP32Wallet._create_master_key) + + wallet = LegacyWallet(storage) + + assert wallet.get_bip32_priv_export() == 'xprv9s21ZrQH143K25QhxbucbDDuQ4naNntJRi4KUfWT7xo4EKsHt2QJDu7KXp1A3u7Bi1j8ph3EGsZ9Xvz9dGuVrtHHs7pXeTzjuxBrCmmhgC6' + assert wallet.get_bip32_pub_export() == 'xpub661MyMwAqRbcEZVB4dScxMAdx6d4nFc9nvyvH3v4gJL378CSRZiYmhRoP7mBy6gSPSCYk6SzXPTf3ND1cZAceL7SfJ1Z3GC8vBgp2epUt13' + assert wallet.get_bip32_priv_export(0) == 'xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L' + assert wallet.get_bip32_pub_export(0) == 'xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y' + + +@pytest.mark.parametrize('mixdepth,internal,index,address,wif', [ + [0, 0, 0, 'mpCX9EbdXpcrKMtjEe1fqFhvzctkfzMYTX', 'cVqtSSoVxFyPqTRGfeESi31uCYfgTF4tGWRtGeVs84fzybiX5TPk'], + [0, 0, 5, 'mtj85a3pFppRhrxNcFig1k7ECshrZjJ9XC', 'cMsFXc4TRw9PTcCTv7x9mr88rDeGXBTLEV67mKaw2cxCkjkhL32G'], + [0, 1, 3, 'n1EaQuqvTRm719hsSJ7yRsj49JfoG1C86q', 'cUgSTqnAtvYoQRXCYy4wCFfaks2Zrz1d55m6mVhFyVhQbkDi7JGJ'], + [2, 1, 2, 'mfxkBk7uDhmF5PJGS9d1NonGiAxPwJqQP4', 'cPcZXSiXPuS5eiT4oDrDKi1mFumw5D1RcWzK2gkGdEHjEz99eyXn'] +]) +def test_bip32_addresses_p2pkh(monkeypatch, setup_wallet, mixdepth, internal, index, address, wif): + """ + Test with a random but fixed entropy + """ + jm_single().config.set('BLOCKCHAIN', 'network', 'testnet') + + entropy = unhexlify('2e0339ba89b4a1272cdf78b27ee62669ee01992a59e836e2807051be128ca817') + storage = VolatileStorage() + LegacyWallet.initialize( + storage, get_network(), entropy=entropy, max_mixdepth=3) + + monkeypatch.setattr(LegacyWallet, '_get_bip32_base_path', + BIP32Wallet._get_bip32_base_path) + monkeypatch.setattr(LegacyWallet, '_create_master_key', + BIP32Wallet._create_master_key) + + wallet = LegacyWallet(storage) + + # wallet needs to know about all intermediate keys + for i in range(index + 1): + wallet.get_new_script(mixdepth, internal) + + assert wif == wallet.get_wif(mixdepth, internal, index) + assert address == wallet.get_addr(mixdepth, internal, index) + + +@pytest.mark.parametrize('mixdepth,internal,index,address,wif', [ + [0, 0, 0, '2MzY5yyonUY7zpHspg7jB7WQs1uJxKafQe4', 'cRAGLvPmhpzJNgdMT4W2gVwEW3fusfaDqdQWM2vnWLgXKzCWKtcM'], + [0, 0, 5, '2MsKvqPGStp3yXT8UivuAaGwfPzT7xYwSWk', 'cSo3h7nRuV4fwhVPXeTDJx6cBCkjAzS9VM8APXViyjoSaMq85ZKn'], + [0, 1, 3, '2N7k6wiQqkuMaApwGhk3HKrifprUSDydqUv', 'cTwq3UsZa8STVmwZR94dDphgqgdLFeuaRFD1Ea44qjbjFfKEb1n5'], + [2, 1, 2, '2MtE6gzHgmEXeWzKsmCJFEqkrpNuBDvoRnz', 'cPV8FZuCvrRpk4RhmhpjnSucHhaQZUan4Vbyo1NVQtuAxurW9grb'] +]) +def test_bip32_addresses_p2sh_p2wpkh(setup_wallet, mixdepth, internal, index, address, wif): + """ + Test with a random but fixed entropy + """ + jm_single().config.set('BLOCKCHAIN', 'network', 'testnet') + + entropy = unhexlify('2e0339ba89b4a1272cdf78b27ee62669ee01992a59e836e2807051be128ca817') + storage = VolatileStorage() + SegwitLegacyWallet.initialize( + storage, get_network(), entropy=entropy, max_mixdepth=3) + wallet = SegwitLegacyWallet(storage) + + # wallet needs to know about all intermediate keys + for i in range(index + 1): + wallet.get_new_script(mixdepth, internal) + + assert wif == wallet.get_wif(mixdepth, internal, index) + assert address == wallet.get_addr(mixdepth, internal, index) + + +def test_import_key(setup_wallet): + jm_single().config.set('BLOCKCHAIN', 'network', 'testnet') + storage = VolatileStorage() + SegwitLegacyWallet.initialize(storage, get_network()) + wallet = SegwitLegacyWallet(storage) + + wallet.import_private_key( + 0, 'cRAGLvPmhpzJNgdMT4W2gVwEW3fusfaDqdQWM2vnWLgXKzCWKtcM', + cryptoengine.TYPE_P2SH_P2WPKH) + wallet.import_private_key( + 1, 'cVqtSSoVxFyPqTRGfeESi31uCYfgTF4tGWRtGeVs84fzybiX5TPk', + cryptoengine.TYPE_P2PKH) + + with pytest.raises(WalletError): + wallet.import_private_key( + 1, 'cRAGLvPmhpzJNgdMT4W2gVwEW3fusfaDqdQWM2vnWLgXKzCWKtcM', + cryptoengine.TYPE_P2SH_P2WPKH) + + # test persist imported keys + wallet.save() + data = storage.file_data + + del wallet + del storage + + storage = VolatileStorage(data=data) + wallet = SegwitLegacyWallet(storage) + + imported_paths_md0 = list(wallet.yield_imported_paths(0)) + imported_paths_md1 = list(wallet.yield_imported_paths(1)) + assert len(imported_paths_md0) == 1 + assert len(imported_paths_md1) == 1 + + # verify imported addresses + assert wallet.get_addr_path(imported_paths_md0[0]) == '2MzY5yyonUY7zpHspg7jB7WQs1uJxKafQe4' + assert wallet.get_addr_path(imported_paths_md1[0]) == 'mpCX9EbdXpcrKMtjEe1fqFhvzctkfzMYTX' + + # test remove key + wallet.remove_imported_key(path=imported_paths_md0[0]) + assert not list(wallet.yield_imported_paths(0)) + + assert wallet.get_details(imported_paths_md1[0]) == (1, 'imported', 0) + + +@pytest.mark.parametrize('wif,keytype,type_check', [ + ['cVqtSSoVxFyPqTRGfeESi31uCYfgTF4tGWRtGeVs84fzybiX5TPk', + cryptoengine.TYPE_P2PKH, assert_not_segwit], + ['cRAGLvPmhpzJNgdMT4W2gVwEW3fusfaDqdQWM2vnWLgXKzCWKtcM', + cryptoengine.TYPE_P2SH_P2WPKH, assert_segwit] +]) +def test_signing_imported(setup_wallet, wif, keytype, type_check): + jm_single().config.set('BLOCKCHAIN', 'network', 'testnet') + storage = VolatileStorage() + SegwitLegacyWallet.initialize(storage, get_network()) + wallet = SegwitLegacyWallet(storage) + + MIXDEPTH = 0 + path = wallet.import_private_key(MIXDEPTH, wif, keytype) + utxo = fund_wallet_addr(wallet, wallet.get_addr_path(path)) + tx = btc.deserialize(btc.mktx(['{}:{}'.format(hexlify(utxo[0]), utxo[1])], + ['00'*17 + ':' + str(10**8 - 9000)])) + binarize_tx(tx) + script = wallet.get_script_path(path) + wallet.sign_tx(tx, {0: (script, 10**8)}) + type_check(tx) + txout = jm_single().bc_interface.pushtx(hexlify(btc.serialize(tx))) + assert txout + + +@pytest.mark.parametrize('wallet_cls,type_check', [ + [LegacyWallet, assert_not_segwit], + [SegwitLegacyWallet, assert_segwit] +]) +def test_signing_simple(setup_wallet, wallet_cls, type_check): + jm_single().config.set('BLOCKCHAIN', 'network', 'testnet') + storage = VolatileStorage() + wallet_cls.initialize(storage, get_network()) + wallet = wallet_cls(storage) + utxo = fund_wallet_addr(wallet, wallet.get_internal_addr(0)) + tx = btc.deserialize(btc.mktx(['{}:{}'.format(hexlify(utxo[0]), utxo[1])], + ['00'*17 + ':' + str(10**8 - 9000)])) + binarize_tx(tx) + script = wallet.get_script(0, 1, 0) + wallet.sign_tx(tx, {0: (script, 10**8)}) + type_check(tx) + txout = jm_single().bc_interface.pushtx(hexlify(btc.serialize(tx))) + assert txout + + +def test_add_utxos(setup_wallet): + jm_single().config.set('BLOCKCHAIN', 'network', 'testnet') + amount = 10**8 + num_tx = 3 + + wallet = get_populated_wallet(amount, num_tx) + + balances = wallet.get_balance_by_mixdepth() + assert balances[0] == num_tx * amount + for md in range(1, wallet.max_mixdepth + 1): + assert balances[md] == 0 + + utxos = wallet.get_utxos_by_mixdepth_() + assert len(utxos[0]) == num_tx + for md in range(1, wallet.max_mixdepth + 1): + assert not utxos[md] + + with pytest.raises(Exception): + # no funds in mixdepth + wallet.select_utxos_(1, amount) + + with pytest.raises(Exception): + # not enough funds + wallet.select_utxos_(0, amount * (num_tx + 1)) + + wallet.reset_utxos() + assert wallet.get_balance_by_mixdepth()[0] == 0 + + +def test_select_utxos(setup_wallet): + jm_single().config.set('BLOCKCHAIN', 'network', 'testnet') + amount = 10**8 + + wallet = get_populated_wallet(amount) + utxos = wallet.select_utxos_(0, amount // 2) + + assert len(utxos) == 1 + utxos = list(utxos.keys()) + + more_utxos = wallet.select_utxos_(0, int(amount * 1.5), utxo_filter=utxos) + assert len(more_utxos) == 2 + assert utxos[0] not in more_utxos + + +def test_add_new_utxos(setup_wallet): + jm_single().config.set('BLOCKCHAIN', 'network', 'testnet') + wallet = get_populated_wallet(num=1) + + scripts = [wallet.get_new_script(x, True) for x in range(3)] + tx_scripts = list(scripts) + tx_scripts.append(b'\x22'*17) + + tx = btc.deserialize(btc.mktx( + ['0'*64 + ':2'], [{'script': hexlify(s), 'value': 10**8} + for s in tx_scripts])) + binarize_tx(tx) + txid = b'\x01' * 32 + added = wallet.add_new_utxos_(tx, txid) + assert len(added) == len(scripts) + + added_scripts = {x['script'] for x in added.values()} + for s in scripts: + assert s in added_scripts + + balances = wallet.get_balance_by_mixdepth() + assert balances[0] == 2 * 10**8 + assert balances[1] == 10**8 + assert balances[2] == 10**8 + assert len(balances) == wallet.max_mixdepth + 1 + + +def test_remove_old_utxos(setup_wallet): + jm_single().config.set('BLOCKCHAIN', 'network', 'testnet') + wallet = get_populated_wallet() + + # add some more utxos to mixdepth 1 + for i in range(3): + txin = jm_single().bc_interface.grab_coins( + wallet.get_internal_addr(1), 1) + wallet.add_utxo(unhexlify(txin), 0, wallet.get_script(1, 1, i), 10**8) + + inputs = wallet.select_utxos_(0, 10**8) + inputs.update(wallet.select_utxos_(1, 2 * 10**8)) + assert len(inputs) == 3 + + tx_inputs = list(inputs.keys()) + tx_inputs.append((b'\x12'*32, 6)) + + tx = btc.deserialize(btc.mktx( + ['{}:{}'.format(hexlify(txid), i) for txid, i in tx_inputs], + ['0' * 36 + ':' + str(3 * 10**8 - 1000)])) + binarize_tx(tx) + + removed = wallet.remove_old_utxos_(tx) + assert len(removed) == len(inputs) + + for txid in removed: + assert txid in inputs + + balances = wallet.get_balance_by_mixdepth() + assert balances[0] == 2 * 10**8 + assert balances[1] == 10**8 + assert balances[2] == 0 + assert len(balances) == wallet.max_mixdepth + 1 + + +def test_initialize_twice(setup_wallet): + wallet = get_populated_wallet(num=0) + storage = wallet._storage + with pytest.raises(WalletError): + SegwitLegacyWallet.initialize(storage, get_network()) + + +def test_is_known(setup_wallet): + wallet = get_populated_wallet(num=0) + script = wallet.get_new_script(1, True) + addr = wallet.get_new_addr(2, False) + + assert wallet.is_known_script(script) + assert wallet.is_known_addr(addr) + assert wallet.is_known_addr(wallet.script_to_addr(script)) + assert wallet.is_known_script(wallet.addr_to_script(addr)) + + assert not wallet.is_known_script(b'\x12' * len(script)) + assert not wallet.is_known_addr('2MzY5yyonUY7zpHspg7jB7WQs1uJxKafQe4') + + +def test_wallet_save(setup_wallet): + wallet = get_populated_wallet() + + script = wallet.get_external_script(1) + + wallet.save() + storage = wallet._storage + data = storage.file_data + + del wallet + del storage + + storage = VolatileStorage(data=data) + wallet = SegwitLegacyWallet(storage) + + assert wallet.get_next_unused_index(0, True) == 3 + assert wallet.get_next_unused_index(0, False) == 0 + assert wallet.get_next_unused_index(1, True) == 0 + assert wallet.get_next_unused_index(1, False) == 1 + assert wallet.is_known_script(script) + + +def test_set_next_index(setup_wallet): + wallet = get_populated_wallet() + + assert wallet.get_next_unused_index(0, True) == 3 + + with pytest.raises(Exception): + # cannot advance index without force=True + wallet.set_next_index(0, True, 5) + + wallet.set_next_index(0, True, 1) + assert wallet.get_next_unused_index(0, True) == 1 + + wallet.set_next_index(0, True, 20, force=True) + assert wallet.get_next_unused_index(0, True) == 20 + + script = wallet.get_new_script(0, True) + path = wallet.script_to_path(script) + index = wallet.get_details(path)[2] + assert index == 20 + + +def test_path_repr(setup_wallet): + wallet = get_populated_wallet() + path = wallet.get_path(2, False, 0) + path_repr = wallet.get_path_repr(path) + path_new = wallet.path_repr_to_path(path_repr) + + assert path_new == path + + +def test_path_repr_imported(setup_wallet): + wallet = get_populated_wallet(num=0) + path = wallet.import_private_key( + 0, 'cRAGLvPmhpzJNgdMT4W2gVwEW3fusfaDqdQWM2vnWLgXKzCWKtcM', + cryptoengine.TYPE_P2SH_P2WPKH) + path_repr = wallet.get_path_repr(path) + path_new = wallet.path_repr_to_path(path_repr) + + assert path_new == path + + +def test_wrong_wallet_cls(setup_wallet): + storage = VolatileStorage() + SegwitLegacyWallet.initialize(storage, get_network()) + wallet = SegwitLegacyWallet(storage) + + wallet.save() + data = storage.file_data + + del wallet + del storage + + storage = VolatileStorage(data=data) + + with pytest.raises(Exception): + LegacyWallet(storage) + + +def test_wallet_id(setup_wallet): + storage1 = VolatileStorage() + SegwitLegacyWallet.initialize(storage1, get_network()) + wallet1 = SegwitLegacyWallet(storage1) + + storage2 = VolatileStorage() + LegacyWallet.initialize(storage2, get_network(), entropy=wallet1._entropy) + wallet2 = LegacyWallet(storage2) + + assert wallet1.get_wallet_id() != wallet2.get_wallet_id() + + storage2 = VolatileStorage() + SegwitLegacyWallet.initialize(storage2, get_network(), + entropy=wallet1._entropy) + wallet2 = SegwitLegacyWallet(storage2) + + assert wallet1.get_wallet_id() == wallet2.get_wallet_id() + + +def test_addr_script_conversion(setup_wallet): + wallet = get_populated_wallet(num=1) + + path = wallet.get_path(0, True, 0) + script = wallet.get_script_path(path) + addr = wallet.script_to_addr(script) + + assert script == wallet.addr_to_script(addr) + addr_path = wallet.addr_to_path(addr) + assert path == addr_path + + +def test_imported_key_removed(setup_wallet): + wif = 'cRAGLvPmhpzJNgdMT4W2gVwEW3fusfaDqdQWM2vnWLgXKzCWKtcM' + key_type = cryptoengine.TYPE_P2SH_P2WPKH + + storage = VolatileStorage() + SegwitLegacyWallet.initialize(storage, get_network()) + wallet = SegwitLegacyWallet(storage) + + path = wallet.import_private_key(1, wif, key_type) + script = wallet.get_script_path(path) + assert wallet.is_known_script(script) + + wallet.remove_imported_key(path=path) + assert not wallet.is_known_script(script) + + with pytest.raises(WalletError): + wallet.get_script_path(path) + + +@pytest.fixture(scope='module') +def setup_wallet(): + load_program_config() + jm_single().bc_interface.tick_forward_chain_interval = 2 diff --git a/scripts/convert_old_wallet.py b/scripts/convert_old_wallet.py new file mode 100644 index 0000000..390418d --- /dev/null +++ b/scripts/convert_old_wallet.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python2 + +import argparse +import json +import os.path +from hashlib import sha256 +from binascii import hexlify, unhexlify +from collections import defaultdict + +from jmclient import Storage, decryptData, load_program_config +from jmclient.wallet_utils import get_password, get_wallet_cls,\ + cli_get_wallet_passphrase_check, get_wallet_path +from jmbitcoin import wif_compressed_privkey + + +class ConvertException(Exception): + pass + + +def get_max_mixdepth(data): + return max(1, len(data.get('index_cache', [1])) - 1, + *data.get('imported', {}).keys()) + + +def is_encrypted(wallet_data): + return 'encrypted_seed' in wallet_data or 'encrypted_entropy' in wallet_data + + +def double_sha256(plaintext): + return sha256(sha256(plaintext).digest()).digest() + + +def decrypt_entropy_extension(enc_data, key): + data = decryptData(key, unhexlify(enc_data)) + if data[-9] != b'\xff': + raise ConvertException("Wrong password.") + chunks = data.split(b'\xff') + if len(chunks) < 3 or data[-8:] != double_sha256(chunks[1])[:8]: + raise ConvertException("Wrong password.") + return chunks[1] + + +def decrypt_wallet_data(data, password): + key = double_sha256(password.encode('utf-8')) + + enc_entropy = data.get('encrypted_seed') or data.get('encrypted_entropy') + enc_entropy_ext = data.get('encrypted_mnemonic_extension') + enc_imported = data.get('imported_keys') + + entropy = decryptData(key, unhexlify(enc_entropy)) + data['entropy'] = entropy + if enc_entropy_ext: + data['entropy_ext'] = decrypt_entropy_extension(enc_entropy_ext, key) + + if enc_imported: + imported_keys = defaultdict(list) + for e in enc_imported: + md = int(e['mixdepth']) + imported_enc_key = unhexlify(e['encrypted_privkey']) + imported_key = decryptData(key, imported_enc_key) + imported_keys[md].append(imported_key) + data['imported'] = imported_keys + + +def new_wallet_from_data(data, file_name): + print("Creating new wallet file.") + new_pw = cli_get_wallet_passphrase_check() + if new_pw is False: + return False + + storage = Storage(file_name, create=True, password=new_pw) + wallet_cls = get_wallet_cls() + + kwdata = { + 'entropy': data['entropy'], + 'timestamp': data.get('creation_time'), + 'max_mixdepth': get_max_mixdepth(data) + } + + if 'entropy_ext' in data: + kwdata['entropy_extension'] = data['entropy_ext'] + + wallet_cls.initialize(storage, data['network'], **kwdata) + wallet = wallet_cls(storage) + + if 'index_cache' in data: + for md, indices in enumerate(data['index_cache']): + wallet.set_next_index(md, 0, indices[0], force=True) + wallet.set_next_index(md, 1, indices[1], force=True) + + if 'imported' in data: + for md in data['imported']: + for privkey in data['imported'][md]: + privkey += b'\x01' + wif = wif_compressed_privkey(hexlify(privkey)) + wallet.import_private_key(md, wif) + + wallet.save() + wallet.close() + return True + + +def parse_old_wallet(fh): + file_data = json.load(fh) + + if is_encrypted(file_data): + pw = get_password("Enter password for old wallet file: ") + try: + decrypt_wallet_data(file_data, pw) + except ValueError: + print("Failed to open wallet: bad password") + return + except Exception as e: + print("Error: {}".format(e)) + print("Failed to open wallet. Wrong password?") + return + + return file_data + + +def main(): + parser = argparse.ArgumentParser( + description="Convert old joinmarket json wallet format to new jmdat " + "format") + parser.add_argument('old_wallet_file', type=open) + parser.add_argument('--name', '-n', required=False, dest='name', + help="Name of the new wallet file. Default: [old wallet name].jmdat") + + try: + args = parser.parse_args() + except Exception as e: + print("Error: {}".format(e)) + return + + data = parse_old_wallet(args.old_wallet_file) + + if not data: + return + + file_name = args.name or\ + os.path.split(args.old_wallet_file.name)[-1].rsplit('.', 1)[0] + '.jmdat' + wallet_path = get_wallet_path(file_name, None) + if new_wallet_from_data(data, wallet_path): + print("New wallet file created at {}".format(wallet_path)) + else: + print("Failed to convert wallet.") + + +if __name__ == '__main__': + load_program_config() + main() From 474a77dd2026ad44825a1804f6993ff8bece2381 Mon Sep 17 00:00:00 2001 From: undeath Date: Fri, 3 Nov 2017 22:04:37 +0100 Subject: [PATCH 02/23] add setup.py dependencies --- jmclient/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jmclient/setup.py b/jmclient/setup.py index 946e9b1..3b2615d 100644 --- a/jmclient/setup.py +++ b/jmclient/setup.py @@ -9,5 +9,5 @@ setup(name='joinmarketclient', author_email='', license='GPL', packages=['jmclient'], - install_requires=['joinmarketbase==0.3.5', 'mnemonic', 'qt4reactor'], + install_requires=['joinmarketbase==0.3.5', 'mnemonic', 'qt4reactor', 'argon2_cffi', 'bencoder.pyx', 'pyaes'], zip_safe=False) From 995c123eecbbbf84b6676cfec5578fa6ae6dffdd Mon Sep 17 00:00:00 2001 From: undeath Date: Thu, 9 Nov 2017 18:58:41 +0100 Subject: [PATCH 03/23] replace old wallet implementation with new one --- jmbitcoin/jmbitcoin/secp256k1_transaction.py | 17 ++++---- jmclient/jmclient/__init__.py | 12 ++++-- jmclient/jmclient/client_protocol.py | 5 +-- jmclient/jmclient/support.py | 13 +++++-- jmclient/jmclient/taker_utils.py | 11 +++--- scripts/sendpayment.py | 41 ++++++++------------ 6 files changed, 49 insertions(+), 50 deletions(-) diff --git a/jmbitcoin/jmbitcoin/secp256k1_transaction.py b/jmbitcoin/jmbitcoin/secp256k1_transaction.py index 31a4451..c234d34 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_transaction.py +++ b/jmbitcoin/jmbitcoin/secp256k1_transaction.py @@ -178,7 +178,8 @@ SIGHASH_NONE = 2 SIGHASH_SINGLE = 3 SIGHASH_ANYONECANPAY = 0x80 -def segwit_signature_form(txobj, i, script, amount, hashcode=SIGHASH_ALL): +def segwit_signature_form(txobj, i, script, amount, hashcode=SIGHASH_ALL, + decoder_func=binascii.unhexlify): """Given a deserialized transaction txobj, an input index i, which spends from a witness, a script for redemption and an amount in satoshis, prepare @@ -187,7 +188,7 @@ def segwit_signature_form(txobj, i, script, amount, hashcode=SIGHASH_ALL): #if isinstance(txobj, string_or_bytes_types): # return serialize(segwit_signature_form(deserialize(txobj), i, script, # amount, hashcode)) - script = binascii.unhexlify(script) + script = decoder_func(script) nVersion = encode(txobj["version"], 256, 4)[::-1] #create hashPrevouts if hashcode & SIGHASH_ANYONECANPAY: @@ -195,7 +196,7 @@ def segwit_signature_form(txobj, i, script, amount, hashcode=SIGHASH_ALL): else: pi = "" for inp in txobj["ins"]: - pi += binascii.unhexlify(inp["outpoint"]["hash"])[::-1] + pi += decoder_func(inp["outpoint"]["hash"])[::-1] pi += encode(inp["outpoint"]["index"], 256, 4)[::-1] hashPrevouts = bin_dbl_sha256(pi) #create hashSequence @@ -208,7 +209,7 @@ def segwit_signature_form(txobj, i, script, amount, hashcode=SIGHASH_ALL): else: hashSequence = "\x00"*32 #add this input's outpoint - thisOut = binascii.unhexlify(txobj["ins"][i]["outpoint"]["hash"])[::-1] + thisOut = decoder_func(txobj["ins"][i]["outpoint"]["hash"])[::-1] thisOut += encode(txobj["ins"][i]["outpoint"]["index"], 256, 4)[::-1] scriptCode = num_to_var_int(len(script)) + script amt = encode(amount, 256, 8)[::-1] @@ -218,13 +219,13 @@ def segwit_signature_form(txobj, i, script, amount, hashcode=SIGHASH_ALL): pi = "" for out in txobj["outs"]: pi += encode(out["value"], 256, 8)[::-1] - pi += (num_to_var_int(len(binascii.unhexlify(out["script"]))) + \ - binascii.unhexlify(out["script"])) + pi += (num_to_var_int(len(decoder_func(out["script"]))) + \ + decoder_func(out["script"])) hashOutputs = bin_dbl_sha256(pi) elif hashcode & 0x1f == SIGHASH_SINGLE and i < len(txobj['outs']): pi = encode(txobj["outs"][i]["value"], 256, 8)[::-1] - pi += (num_to_var_int(len(binascii.unhexlify(txobj["outs"][i]["script"]))) + - binascii.unhexlify(txobj["outs"][i]["script"])) + pi += (num_to_var_int(len(decoder_func(txobj["outs"][i]["script"]))) + + decoder_func(txobj["outs"][i]["script"])) hashOutputs = bin_dbl_sha256(pi) else: hashOutputs = "\x00"*32 diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index e285696..11f3507 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -11,14 +11,18 @@ from btc import * from .support import (calc_cj_fee, choose_sweep_orders, choose_orders, cheapest_order_choose, weighted_order_choose, rand_norm_array, rand_pow_array, rand_exp_array, select, - select_gradual, select_greedy, select_greediest) + select_gradual, select_greedy, select_greediest, + get_random_bytes) from .jsonrpc import JsonRpcError, JsonRpcConnectionError, JsonRpc from .old_mnemonic import mn_decode, mn_encode from .slowaes import decryptData, encryptData from .taker import Taker -from .wallet import (AbstractWallet, BitcoinCoreInterface, Wallet, - BitcoinCoreWallet, estimate_tx_fee, WalletError, - create_wallet_file, SegwitWallet, Bip39Wallet, get_wallet_cls) +from .wallet import (estimate_tx_fee, WalletError, BaseWallet, ImportWalletMixin, + BIP39WalletMixin, BIP32Wallet, BIP49Wallet, LegacyWallet, + SegwitLegacyWallet, UTXOManager, WALLET_IMPLEMENTATIONS) +from .storage import (Argon2Hash, Storage, StorageError, + StoragePasswordError, VolatileStorage) +from .cryptoengine import BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, EngineError from .configure import (load_program_config, get_p2pk_vbyte, jm_single, get_network, validate_address, get_irc_mchannels, get_blockchain_interface_instance, get_p2sh_vbyte, set_config) diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index 5a9b2d8..32abe7c 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -16,9 +16,8 @@ import hashlib import os import sys import pprint -from jmclient import (Taker, Wallet, jm_single, get_irc_mchannels, - load_program_config, get_log, get_p2sh_vbyte, - RegtestBitcoinCoreInterface) +from jmclient import (jm_single, get_irc_mchannels, get_log, get_p2sh_vbyte, + RegtestBitcoinCoreInterface) from jmbase import _byteify import btc diff --git a/jmclient/jmclient/support.py b/jmclient/jmclient/support.py index 7020d4f..b62d514 100644 --- a/jmclient/jmclient/support.py +++ b/jmclient/jmclient/support.py @@ -1,9 +1,5 @@ from __future__ import absolute_import, print_function -import sys - -import logging -import pprint import random from jmbase.support import get_log from decimal import Decimal @@ -23,6 +19,15 @@ Only for sampling purposes """ +def get_random_bytes(num_bytes, cryptographically_secure=False): + if cryptographically_secure: + # uses os.urandom if available + generator = random.SystemRandom() + else: + generator = random + return bytes(bytearray((generator.randrange(256) for b in xrange(num_bytes)))) + + def rand_norm_array(mu, sigma, n): # use normalvariate instead of gauss for thread safety return [random.normalvariate(mu, sigma) for _ in range(n)] diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index 83b5afe..a91b108 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -7,7 +7,7 @@ import time import numbers from .configure import get_log, jm_single, validate_address from .schedule import human_readable_schedule_entry, tweak_tumble_schedule -from .wallet import Wallet, SegwitWallet, estimate_tx_fee +from .wallet import BaseWallet, estimate_tx_fee from jmclient import mktx, deserialize, sign, txhash log = get_log() @@ -42,10 +42,10 @@ def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False, assert mixdepth >= 0 assert isinstance(amount, numbers.Integral) assert amount >=0 - assert isinstance(wallet, Wallet) or isinstance(wallet, SegwitWallet) + assert isinstance(wallet, BaseWallet) from pprint import pformat - txtype = 'p2sh-p2wpkh' if isinstance(wallet, SegwitWallet) else 'p2pkh' + txtype = wallet.get_txtype() if amount == 0: utxos = wallet.get_utxos_by_mixdepth()[mixdepth] if utxos == {}: @@ -79,9 +79,8 @@ def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False, utxo = ins['outpoint']['hash'] + ':' + str( ins['outpoint']['index']) addr = utxos[utxo]['address'] - signing_amount = utxos[utxo]['value'] - amt = signing_amount if isinstance(wallet, SegwitWallet) else None - tx = sign(tx, index, wallet.get_key_from_addr(addr), amount=amt) + amount = utxos[utxo]['value'] + tx = sign(tx, index, wallet.get_key_from_addr(addr), amount=amount) txsigned = deserialize(tx) log.info("Got signed transaction:\n") log.info(tx + "\n") diff --git a/scripts/sendpayment.py b/scripts/sendpayment.py index 3c4e9a5..ff6598e 100644 --- a/scripts/sendpayment.py +++ b/scripts/sendpayment.py @@ -17,14 +17,12 @@ import time import os import pprint -from jmclient import (Taker, load_program_config, get_schedule, - JMClientProtocolFactory, start_reactor, - validate_address, jm_single, WalletError, - choose_orders, choose_sweep_orders, - cheapest_order_choose, weighted_order_choose, - sync_wallet, RegtestBitcoinCoreInterface, - estimate_tx_fee, direct_send, get_wallet_cls, - BitcoinCoreWallet) +from jmclient import ( + Taker, load_program_config, get_schedule, JMClientProtocolFactory, + start_reactor, validate_address, jm_single, WalletError, choose_orders, + choose_sweep_orders, cheapest_order_choose, weighted_order_choose, + sync_wallet, RegtestBitcoinCoreInterface, estimate_tx_fee, direct_send, + open_test_wallet_maybe, get_wallet_path) from twisted.python.log import startLogging from jmbase.support import get_log, debug_dump_object, get_password from cli_options import get_sendpayment_parser @@ -126,33 +124,26 @@ def main(): #maxmixdepth in the wallet is actually the *number* of mixdepths (so misnamed); #to ensure we have enough, must be at least (requested index+1) max_mix_depth = max([mixdepth+1, options.amtmixdepths]) - if not os.path.exists(os.path.join('wallets', wallet_name)): - wallet = get_wallet_cls()(wallet_name, None, max_mix_depth, options.gaplimit) - else: - while True: - try: - pwd = get_password("Enter wallet decryption passphrase: ") - wallet = get_wallet_cls()(wallet_name, pwd, max_mix_depth, options.gaplimit) - except WalletError: - print("Wrong password, try again.") - continue - except Exception as e: - print("Failed to load wallet, error message: " + repr(e)) - sys.exit(0) - break + + wallet_path = get_wallet_path(wallet_name, None) + wallet = open_test_wallet_maybe( + wallet_path, wallet_name, max_mix_depth, gap_limit=options.gaplimit) else: - wallet = BitcoinCoreWallet(fromaccount=wallet_name) + raise NotImplemented("Using non-joinmarket wallet is not supported.") if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == "electrum-server" and options.makercount != 0: jm_single().bc_interface.synctype = "with-script" #wallet sync will now only occur on reactor start if we're joining. sync_wallet(wallet, fast=options.fastsync) if options.makercount == 0: - if isinstance(wallet, BitcoinCoreWallet): - raise NotImplementedError("Direct send only supported for JM wallets") direct_send(wallet, amount, mixdepth, destaddr, options.answeryes) return + if wallet.get_txtype() == 'p2pkh': + print("Only direct sends (use -N 0) are supported for " + "legacy (non-segwit) wallets.") + return + def filter_orders_callback(orders_fees, cjamount): orders, total_cj_fee = orders_fees log.info("Chose these orders: " +pprint.pformat(orders)) From 6aaabb2f306bd6e9e8f4a24de23ad6d69480cc3c Mon Sep 17 00:00:00 2001 From: undeath Date: Sun, 24 Dec 2017 23:06:20 +0100 Subject: [PATCH 04/23] change yieldgenerator using new wallet implementation, start porting wallet_utils --- jmclient/jmclient/wallet_utils.py | 276 ++++++++++++++++++---------- jmclient/jmclient/yieldgenerator.py | 25 +-- 2 files changed, 186 insertions(+), 115 deletions(-) diff --git a/jmclient/jmclient/wallet_utils.py b/jmclient/jmclient/wallet_utils.py index 08e4e4a..c9b1ab6 100644 --- a/jmclient/jmclient/wallet_utils.py +++ b/jmclient/jmclient/wallet_utils.py @@ -9,10 +9,10 @@ from datetime import datetime from mnemonic import Mnemonic from optparse import OptionParser import getpass -from jmclient import (get_network, get_wallet_cls, Bip39Wallet, podle, - encryptData, get_p2sh_vbyte, get_p2pk_vbyte, jm_single, - mn_decode, mn_encode, BitcoinCoreInterface, - JsonRpcError, sync_wallet, WalletError) +from jmclient import (get_network, WALLET_IMPLEMENTATIONS, Storage, podle, + encryptData, get_p2sh_vbyte, get_p2pk_vbyte, jm_single, mn_decode, + mn_encode, BitcoinCoreInterface, JsonRpcError, sync_wallet, WalletError, + BIP49Wallet, ImportWalletMixin, VolatileStorage, StoragePasswordError) from jmbase.support import get_password import jmclient.btc as btc @@ -330,25 +330,26 @@ def wallet_display(wallet, gaplimit, showprivkey, displayall=False, """ acctlist = [] rootpath = wallet.get_root_path() - for m in range(wallet.max_mix_depth): + for m in xrange(wallet.max_mixdepth): branchlist = [] for forchange in [0, 1]: entrylist = [] + # FIXME: why does this if/else exist? if forchange == 0: - xpub_key = btc.bip32_privtopub(wallet.keys[m][forchange]) + xpub_key = wallet.get_bip32_pub_export(m, forchange) else: xpub_key = "" - for k in range(wallet.index[m][forchange] + gaplimit): - addr = wallet.get_addr(m, forchange, k) + for k in xrange(wallet.get_next_unused_index(m, forchange) + gaplimit): + path = wallet.get_path(m, forchange, k) + addr = wallet.get_addr_path(path) balance = 0 - for addrvalue in wallet.unspent.values(): - if addr == addrvalue['address']: - balance += addrvalue['value'] - used = 'used' if k < wallet.index[m][forchange] else 'new' + for utxodata in wallet.get_utxos_by_mixdepth_()[m].values(): + if path == utxodata['path']: + balance += utxodata['value'] + used = 'used' if k < wallet.get_next_unused_index(m, forchange) else 'new' if showprivkey: - privkey = btc.wif_compressed_privkey( - wallet.get_key(m, forchange, k), get_p2pk_vbyte()) + privkey = wallet.get_wif_path(path) else: privkey = '' if (displayall or balance > 0 or @@ -362,8 +363,7 @@ def wallet_display(wallet, gaplimit, showprivkey, displayall=False, if ipb: branchlist.append(ipb) #get the xpub key of the whole account - xpub_account = btc.bip32_privtopub( - wallet.get_mixing_depth_keys(wallet.get_master_key())[m]) + xpub_account = wallet.get_bip32_pub_export(mixdepth=m) acctlist.append(WalletViewAccount(rootpath, m, branchlist, xpub=xpub_account)) walletview = WalletView(rootpath, acctlist) @@ -397,12 +397,14 @@ def cli_user_mnemonic_entry(): return (mnemonic_phrase, mnemonic_extension) def cli_get_mnemonic_extension(): - uin = raw_input('Would you like to use a two-factor mnemonic recovery' - + ' phrase? write \'n\' if you don\'t know what this is (y/n): ') + uin = raw_input("Would you like to use a two-factor mnemonic recovery " + "phrase? write 'n' if you don't know what this is (y/n): ") if len(uin) == 0 or uin[0] != 'y': - print('Not using mnemonic extension') + print("Not using mnemonic extension") return None #no mnemonic extension - return raw_input('Enter mnemonic extension: ') + print("Note: This will be stored in a reversible way. Do not reuse!") + return raw_input("Enter mnemonic extension: ") + def persist_walletfile(walletspath, default_wallet_name, encrypted_entropy, encrypted_mnemonic_extension=None, @@ -761,71 +763,170 @@ def wallet_fetch_history(wallet, options): def wallet_showseed(wallet): - if isinstance(wallet, Bip39Wallet): - if not wallet.entropy: - return "Entropy is not initialized." - m = Mnemonic("english") - text = "Wallet mnemonic recovery phrase:\n\n" + m.to_mnemonic(wallet.entropy) + "\n" - if wallet.mnemonic_extension: - text += '\nWallet mnemonic extension: ' + wallet.mnemonic_extension + '\n' - return text - hexseed = wallet.seed - print("hexseed = " + hexseed) - words = mn_encode(hexseed) - return "Wallet mnemonic seed phrase:\n\n" + " ".join(words) + "\n" + seed, extension = wallet.get_mnemonic_words() + text = "Wallet mnemonic recovery phrase:\n\n{}\n".format(seed) + if extension: + text += "\nWallet mnemonic extension: {}\n".format(extension) + return text + def wallet_importprivkey(wallet, mixdepth): - print('WARNING: This imported key will not be recoverable with your 12 ' + - 'word mnemonic phrase. Make sure you have backups.') - print('WARNING: Handling of raw ECDSA bitcoin private keys can lead to ' - 'non-intuitive behaviour and loss of funds.\n Recommended instead ' - 'is to use the \'sweep\' feature of sendpayment.py ') - privkeys = raw_input('Enter private key(s) to import: ') + print("WARNING: This imported key will not be recoverable with your 12 " + "word mnemonic phrase. Make sure you have backups.") + print("WARNING: Handling of raw ECDSA bitcoin private keys can lead to " + "non-intuitive behaviour and loss of funds.\n Recommended instead " + "is to use the \'sweep\' feature of sendpayment.py.") + privkeys = raw_input("Enter private key(s) to import: ") privkeys = privkeys.split(',') if ',' in privkeys else privkeys.split() + imported_addr = [] # TODO read also one key for each line - for privkey in privkeys: + for wif in privkeys: # TODO is there any point in only accepting wif format? check what # other wallets do - privkey_bin = btc.from_wif_privkey(privkey, - vbyte=get_p2pk_vbyte()).decode('hex')[:-1] - encrypted_privkey = encryptData(wallet.password_key, privkey_bin) - if 'imported_keys' not in wallet.walletdata: - wallet.walletdata['imported_keys'] = [] - wallet.walletdata['imported_keys'].append( - {'encrypted_privkey': encrypted_privkey.encode('hex'), - 'mixdepth': mixdepth}) - if wallet.walletdata['imported_keys']: - fd = open(wallet.path, 'w') - fd.write(json.dumps(wallet.walletdata)) - fd.close() - print('Private key(s) successfully imported') + imported_addr.append(wallet.import_private_key(mixdepth, wif)) + wallet.save() + + if not imported_addr: + print("Warning: No keys imported!") + return + + # show addresses to user so they can verify everything went as expected + print("Imported keys for addresses:") + for addr in imported_addr: + print(addr) + def wallet_dumpprivkey(wallet, hdpath): - pathlist = bip32pathparse(hdpath) - print('got pathlist: ' + str(pathlist)) - if pathlist and len(pathlist) in [5, 4]: - #note here we assume the path conforms to Wallet or SegwitWallet(BIP49) standard - m, forchange, k = pathlist[-3:] - key = wallet.get_key(m, forchange, k) - wifkey = btc.wif_compressed_privkey(key, vbyte=get_p2pk_vbyte()) - return wifkey - else: - return hdpath + " is not a valid hd wallet path" + path = wallet.path_repr_to_path(hdpath) + return wallet.get_wif_path(path) # will raise exception on invalid path + def wallet_signmessage(wallet, hdpath, message): - if hdpath.startswith(wallet.get_root_path()): - hp = bip32pathparse(hdpath) - m, forchange, k = hp[-3:] - key = wallet.get_key(m, forchange, k) - addr = wallet.pubkey_to_address(btc.privkey_to_pubkey(key)) - print('Using address: ' + addr) + msg = message.encode('utf-8') + + path = wallet.path_repr_to_path(hdpath) + sig = wallet.sign_message(msg, path) + return ("Signature: {}\n" + "To verify this in Bitcoin Core use the RPC command 'verifymessage'" + "".format(sig)) + + +def get_wallet_type(): + if jm_single().config.get('POLICY', 'segwit') == 'true': + return 'p2sh-p2wpkh' + return 'p2pkh' + + +def get_wallet_cls(wtype=None): + if wtype is None: + wtype = get_wallet_type() + + cls = WALLET_IMPLEMENTATIONS.get(wtype) + + if not cls: + raise WalletError("No wallet implementation found for type {}." + "".format(wtype)) + return cls + + +def create_wallet(path, password, max_mixdepth, **kwargs): + storage = Storage(path, password, create=True) + wallet_cls = get_wallet_cls() + wallet_cls.initialize(storage, get_network(), max_mixdepth=max_mixdepth, + **kwargs) + + +def open_test_wallet_maybe(path, seed, max_mixdepth, **kwargs): + """ + Create a volatile test wallet if path is a hex-encoded string of length 64, + otherwise run open_wallet(). + + params: + path: path to wallet file, ignored for test wallets + seed: hex-encoded test seed + max_mixdepth: see create_wallet(), ignored when calling open_wallet() + kwargs: see open_wallet() + + returns: + wallet object + """ + class SewgitTestWallet(ImportWalletMixin, BIP49Wallet): + TYPE = 'p2sh-p2wpkh' + + if len(seed) == SewgitTestWallet.ENTROPY_BYTES * 2: + try: + seed = binascii.unhexlify(seed) + except binascii.Error: + pass + else: + storage = VolatileStorage() + SewgitTestWallet.initialize( + storage, get_network(), max_mixdepth=max_mixdepth, + entropy=seed) + assert 'ask_for_password' not in kwargs + assert 'read_only' not in kwargs + return SewgitTestWallet(storage, **kwargs) + + return open_wallet(path, **kwargs) + + +def open_wallet(path, ask_for_password=True, read_only=False, **kwargs): + """ + Open the wallet file at path and return the corresponding wallet object. + + params: + path: str, full path to wallet file + ask_for_password: bool, if False password is assumed unset and user + will not be asked to type it + read_only: bool, if True, open wallet in read-only mode + kwargs: additional options to pass to wallet's init method + + returns: + wallet object + """ + if ask_for_password: + while True: + try: + # do not try empty password, assume unencrypted on empty password + pwd = get_password("Enter wallet decryption passphrase: ") or None + storage = Storage(path, password=pwd, read_only=read_only) + except StoragePasswordError: + print("Wrong password, try again.") + continue + except Exception as e: + print("Failed to load wallet, error message: " + repr(e)) + raise e + break else: - print('%s is not a valid hd wallet path' % hdpath) - return None - sig = btc.ecdsa_sign(message, key, formsg=True) - retval = "Signature: " + str(sig) + "\n" - retval += "To verify this in Bitcoin Core use the RPC command 'verifymessage'" - return retval + storage = Storage(path, read_only=read_only) + + wallet_cls = get_wallet_cls(storage) + wallet = wallet_cls(storage, **kwargs) + wallet_sanity_check(wallet) + return wallet + + +def get_wallet_cls_from_storage(storage): + wtype = storage.data.get([b'wallet_type']) + + if not wtype: + raise WalletError("File {} is not a valid wallet.".format(storage.path)) + + wtype = wtype.decode('ascii') + return get_wallet_cls(wtype) + + +def wallet_sanity_check(wallet): + if wallet.network != get_network(): + raise Exception("Wallet network mismatch: we are on {} but wallet is " + "on {}".format(get_network(), wallet.network)) + + +def get_wallet_path(file_name, wallet_dir): + # TODO: move default wallet path to ~/.joinmarket + wallet_dir = wallet_dir or 'wallets' + return os.path.join(wallet_dir, file_name) + def wallet_tool_main(wallet_root_path): """Main wallet tool script function; returned is a string (output or error) @@ -853,29 +954,12 @@ def wallet_tool_main(wallet_root_path): method = args[0] else: seed = args[0] + wallet_path = get_wallet_path(seed, wallet_root_path) method = ('display' if len(args) == 1 else args[1].lower()) - if not os.path.exists(os.path.join(wallet_root_path, seed)): - wallet = get_wallet_cls()(seed, None, options.maxmixdepth, - options.gaplimit, extend_mixdepth= not maxmixdepth_configured, - storepassword=(method == 'importprivkey'), - wallet_dir=wallet_root_path) - else: - while True: - try: - pwd = get_password("Enter wallet decryption passphrase: ") - wallet = get_wallet_cls()(seed, pwd, - options.maxmixdepth, - options.gaplimit, - extend_mixdepth=not maxmixdepth_configured, - storepassword=(method == 'importprivkey'), - wallet_dir=wallet_root_path) - except WalletError: - print("Wrong password, try again.") - continue - except Exception as e: - print("Failed to load wallet, error message: " + repr(e)) - sys.exit(0) - break + + wallet = open_test_wallet_maybe( + wallet_path, seed, options.maxmixdepth, gap_limit=options.gaplimit) + if method not in noscan_methods: # if nothing was configured, we override bitcoind's options so that # unconfirmed balance is included in the wallet display by default diff --git a/jmclient/jmclient/yieldgenerator.py b/jmclient/jmclient/yieldgenerator.py index 72dc585..564933d 100644 --- a/jmclient/jmclient/yieldgenerator.py +++ b/jmclient/jmclient/yieldgenerator.py @@ -7,10 +7,10 @@ import time import abc from twisted.python.log import startLogging from optparse import OptionParser -from jmbase import get_password from jmclient import (Maker, jm_single, get_network, load_program_config, get_log, get_wallet_cls, sync_wallet, JMClientProtocolFactory, start_reactor, calc_cj_fee, WalletError) +from .wallet_utils import open_test_wallet_maybe, get_wallet_path jlog = get_log() @@ -232,23 +232,11 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffe nickserv_password = options.password load_program_config() - if not os.path.exists(os.path.join('wallets', wallet_name)): - wallet = get_wallet_cls()(wallet_name, None, max_mix_depth=MAX_MIX_DEPTH, - gaplimit=options.gaplimit) - else: - while True: - try: - pwd = get_password("Enter wallet decryption passphrase: ") - wallet = get_wallet_cls()(wallet_name, pwd, - max_mix_depth=MAX_MIX_DEPTH, - gaplimit=options.gaplimit) - except WalletError: - print("Wrong password, try again.") - continue - except Exception as e: - print("Failed to load wallet, error message: " + repr(e)) - sys.exit(0) - break + + wallet_path = get_wallet_path(wallet_name, 'wallets') + wallet = open_test_wallet_maybe( + wallet_path, wallet_name, 4, gap_limit=options.gaplimit) + if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == "electrum-server": jm_single().bc_interface.synctype = "with-script" sync_wallet(wallet, fast=options.fastsync) @@ -265,4 +253,3 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffe start_reactor(jm_single().config.get("DAEMON", "daemon_host"), jm_single().config.getint("DAEMON", "daemon_port"), clientfactory, daemon=daemon) - From 2a0757c4bc76c1e7f92049d8685f60f0d36295b1 Mon Sep 17 00:00:00 2001 From: undeath Date: Sun, 24 Dec 2017 23:11:36 +0100 Subject: [PATCH 05/23] remove BitcoinCoreWallet --- jmclient/jmclient/blockchaininterface.py | 19 +------------------ jmclient/test/test_wallets.py | 15 +-------------- 2 files changed, 2 insertions(+), 32 deletions(-) diff --git a/jmclient/jmclient/blockchaininterface.py b/jmclient/jmclient/blockchaininterface.py index bc0ccdb..67b8648 100644 --- a/jmclient/jmclient/blockchaininterface.py +++ b/jmclient/jmclient/blockchaininterface.py @@ -1,16 +1,10 @@ from __future__ import print_function -import BaseHTTPServer import abc import ast -import json -import os -import pprint import random -import re import sys import time -import traceback import binascii from decimal import Decimal from twisted.internet import reactor, task @@ -19,7 +13,7 @@ import btc from jmclient.jsonrpc import JsonRpcConnectionError, JsonRpcError from jmclient.configure import get_p2pk_vbyte, jm_single -from jmbase.support import get_log, chunks +from jmbase.support import get_log log = get_log() @@ -425,9 +419,6 @@ class BitcoinCoreInterface(BlockchainInterface): Bitcoin Core instance, in which case "fast" should have been specifically disabled by the user. """ - from jmclient.wallet import BitcoinCoreWallet - if isinstance(wallet, BitcoinCoreWallet): - return wallet_name = self.get_wallet_name(wallet) agd = self.rpc('listaddressgroupings', []) #flatten all groups into a single list; then, remove duplicates @@ -516,10 +507,6 @@ class BitcoinCoreInterface(BlockchainInterface): def sync_addresses(self, wallet, restart_cb=None): - from jmclient.wallet import BitcoinCoreWallet - - if isinstance(wallet, BitcoinCoreWallet): - return log.debug('requesting detailed wallet history') wallet_name = self.get_wallet_name(wallet) #TODO It is worth considering making this user configurable: @@ -653,10 +640,6 @@ class BitcoinCoreInterface(BlockchainInterface): self.unspent_monitoring_loop.stop() def sync_unspent(self, wallet): - from jmclient.wallet import BitcoinCoreWallet - - if isinstance(wallet, BitcoinCoreWallet): - return st = time.time() wallet_name = self.get_wallet_name(wallet) wallet.unspent = {} diff --git a/jmclient/test/test_wallets.py b/jmclient/test/test_wallets.py index 98d944a..d16147e 100644 --- a/jmclient/test/test_wallets.py +++ b/jmclient/test/test_wallets.py @@ -23,7 +23,7 @@ from jmclient import (load_program_config, jm_single, sync_wallet, AbstractWallet, get_p2pk_vbyte, get_log, Wallet, select, select_gradual, select_greedy, select_greediest, estimate_tx_fee, encryptData, get_network, WalletError, - BitcoinCoreWallet, BitcoinCoreInterface, SegwitWallet, + BitcoinCoreInterface, SegwitWallet, wallet_generate_recover_bip39, decryptData, encryptData) from jmbase.support import chunks from taker_test_data import t_obtained_tx, t_raw_signed_tx @@ -265,19 +265,6 @@ def test_index_ahead_cache(setup_wallets): assert is_index_ahead_of_cache(wallet, 3, 1) -def test_core_wallet_no_sync(setup_wallets): - """Ensure BitcoinCoreWallet sync attempt does nothing - """ - wallet = BitcoinCoreWallet("") - #this will not trigger sync due to absence of non-zero index_cache, usually. - wallet.index_cache = [[1, 1]] - jm_single().bc_interface.wallet_synced = False - jm_single().bc_interface.sync_wallet(wallet, fast=True) - assert not jm_single().bc_interface.wallet_synced - jm_single().bc_interface.sync_wallet(wallet) - assert not jm_single().bc_interface.wallet_synced - - def test_wrong_network_bci(setup_wallets): rpc = jm_single().bc_interface.jsonRpc with pytest.raises(Exception) as e_info: From 3cf99266b5a4a521ed4b24667bdc1637e7843596 Mon Sep 17 00:00:00 2001 From: undeath Date: Mon, 25 Dec 2017 00:28:53 +0100 Subject: [PATCH 06/23] remove references to old wallet classes --- jmclient/jmclient/__init__.py | 13 ++++++---- jmclient/jmclient/wallet_utils.py | 6 +++-- jmclient/jmclient/yieldgenerator.py | 2 +- scripts/add-utxo.py | 31 ++++++++-------------- scripts/jmtainter.py | 31 +++++++--------------- scripts/joinmarket-qt.py | 35 ++++++++++++------------- scripts/tumbler.py | 40 ++++++++--------------------- 7 files changed, 59 insertions(+), 99 deletions(-) diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index 11f3507..8174bd9 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -23,9 +23,10 @@ from .wallet import (estimate_tx_fee, WalletError, BaseWallet, ImportWalletMixin from .storage import (Argon2Hash, Storage, StorageError, StoragePasswordError, VolatileStorage) from .cryptoengine import BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, EngineError -from .configure import (load_program_config, get_p2pk_vbyte, - jm_single, get_network, validate_address, get_irc_mchannels, - get_blockchain_interface_instance, get_p2sh_vbyte, set_config) +from .configure import ( + load_program_config, get_p2pk_vbyte, jm_single, get_network, + validate_address, get_irc_mchannels, get_blockchain_interface_instance, + get_p2sh_vbyte, set_config) from .blockchaininterface import (BlockchainInterface, sync_wallet, RegtestBitcoinCoreInterface, BitcoinCoreInterface) from .electruminterface import ElectrumInterface @@ -42,8 +43,10 @@ from .commitment_utils import get_utxo_info, validate_utxo_data, quit from .taker_utils import (tumbler_taker_finished_update, restart_waiter, restart_wait, get_tumble_log, direct_send, tumbler_filter_orders_callback) -from .wallet_utils import (wallet_tool_main, wallet_generate_recover_bip39, - wallet_display) +from .wallet_utils import ( + wallet_tool_main, wallet_generate_recover_bip39, open_wallet, + open_test_wallet_maybe, create_wallet, get_wallet_cls, get_wallet_path, + wallet_display) from .maker import Maker from .yieldgenerator import YieldGenerator, YieldGeneratorBasic, ygmain # Set default logging handler to avoid "No handler found" warnings. diff --git a/jmclient/jmclient/wallet_utils.py b/jmclient/jmclient/wallet_utils.py index c9b1ab6..9afe41e 100644 --- a/jmclient/jmclient/wallet_utils.py +++ b/jmclient/jmclient/wallet_utils.py @@ -870,7 +870,8 @@ def open_test_wallet_maybe(path, seed, max_mixdepth, **kwargs): return open_wallet(path, **kwargs) -def open_wallet(path, ask_for_password=True, read_only=False, **kwargs): +def open_wallet(path, ask_for_password=True, password=None, read_only=False, + **kwargs): """ Open the wallet file at path and return the corresponding wallet object. @@ -878,6 +879,7 @@ def open_wallet(path, ask_for_password=True, read_only=False, **kwargs): path: str, full path to wallet file ask_for_password: bool, if False password is assumed unset and user will not be asked to type it + password: password for storage, ignored if ask_for_password is True read_only: bool, if True, open wallet in read-only mode kwargs: additional options to pass to wallet's init method @@ -898,7 +900,7 @@ def open_wallet(path, ask_for_password=True, read_only=False, **kwargs): raise e break else: - storage = Storage(path, read_only=read_only) + storage = Storage(path, password, read_only=read_only) wallet_cls = get_wallet_cls(storage) wallet = wallet_cls(storage, **kwargs) diff --git a/jmclient/jmclient/yieldgenerator.py b/jmclient/jmclient/yieldgenerator.py index 564933d..4fc89ad 100644 --- a/jmclient/jmclient/yieldgenerator.py +++ b/jmclient/jmclient/yieldgenerator.py @@ -8,7 +8,7 @@ import abc from twisted.python.log import startLogging from optparse import OptionParser from jmclient import (Maker, jm_single, get_network, load_program_config, get_log, - get_wallet_cls, sync_wallet, JMClientProtocolFactory, + sync_wallet, JMClientProtocolFactory, start_reactor, calc_cj_fee, WalletError) from .wallet_utils import open_test_wallet_maybe, get_wallet_path diff --git a/scripts/add-utxo.py b/scripts/add-utxo.py index 86c8367..1c9d6a7 100644 --- a/scripts/add-utxo.py +++ b/scripts/add-utxo.py @@ -6,7 +6,6 @@ users to retry transactions more often without getting banned by the anti-snooping feature employed by makers. """ -import binascii import sys import os import json @@ -14,12 +13,12 @@ from pprint import pformat from optparse import OptionParser import jmclient.btc as btc -from jmbase import get_password -from jmclient import (load_program_config, jm_single, get_p2pk_vbyte, get_wallet_cls, - WalletError, sync_wallet, add_external_commitments, - generate_podle, update_commitments, PoDLE, - set_commitment_file, get_podle_commitments, - get_utxo_info, validate_utxo_data, quit) +from jmclient import ( + load_program_config, jm_single, get_p2pk_vbyte, open_wallet, WalletError, + sync_wallet, add_external_commitments, generate_podle, update_commitments, + PoDLE, set_commitment_file, get_podle_commitments, get_utxo_info, + validate_utxo_data, quit, get_wallet_path) + def add_ext_commitments(utxo_datas): """Persist the PoDLE commitments for this utxo @@ -174,20 +173,10 @@ def main(): #Three options (-w, -r, -R) for loading utxo and privkey pairs from a wallet, #csv file or json file. if options.loadwallet: - while True: - pwd = get_password("Enter wallet decryption passphrase: ") - try: - wallet = get_wallet_cls()(options.loadwallet, - pwd, - options.maxmixdepth, - options.gaplimit) - except WalletError: - print("Wrong password, try again.") - continue - except Exception as e: - print("Failed to load wallet, error message: " + repr(e)) - sys.exit(0) - break + # TODO: new wallet has no unspent attribute + raise NotImplementedError("This is not yet implemented.") + wallet_path = get_wallet_path(options.loadwallet, None) + wallet = open_wallet(wallet_path, gap_limit=options.gaplimit) sync_wallet(wallet, fast=options.fastsync) unsp = {} for u, av in wallet.unspent.iteritems(): diff --git a/scripts/jmtainter.py b/scripts/jmtainter.py index 6849fec..a168a32 100644 --- a/scripts/jmtainter.py +++ b/scripts/jmtainter.py @@ -9,17 +9,14 @@ can spread to other outputs included. This is a tool for Joinmarket wallets specifically. """ import binascii -import os -import sys -import random from optparse import OptionParser from pprint import pformat import jmbitcoin as btc -from jmclient import (load_program_config, validate_address, jm_single, - WalletError, sync_wallet, RegtestBitcoinCoreInterface, - estimate_tx_fee, SegwitWallet, get_p2pk_vbyte, - get_p2sh_vbyte, get_wallet_cls) -from jmbase.support import get_password +from jmclient import ( + load_program_config, validate_address, jm_single, WalletError, sync_wallet, + RegtestBitcoinCoreInterface, estimate_tx_fee, get_p2pk_vbyte, + get_p2sh_vbyte, open_test_wallet_maybe, get_wallet_path) + def get_parser(): parser = OptionParser( @@ -79,20 +76,10 @@ def is_utxo(utxo): return True def cli_get_wallet(wallet_name, sync=True): - if not os.path.exists(os.path.join('wallets', wallet_name)): - wallet = get_wallet_cls()(wallet_name, None, max_mix_depth=options.amtmixdepths) - else: - while True: - try: - pwd = get_password("Enter wallet decryption passphrase: ") - wallet = get_wallet_cls()(wallet_name, pwd, max_mix_depth=options.amtmixdepths) - except WalletError: - print("Wrong password, try again.") - continue - except Exception as e: - print("Failed to load wallet, error message: " + repr(e)) - sys.exit(0) - break + wallet_path = get_wallet_path(wallet_name, None) + wallet = open_test_wallet_maybe( + wallet_path, wallet_name, options.amtmixdepths, gap_limit=options.gaplimit) + if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == "electrum-server": jm_single().bc_interface.synctype = "with-script" diff --git a/scripts/joinmarket-qt.py b/scripts/joinmarket-qt.py index 3af31dc..277beb7 100644 --- a/scripts/joinmarket-qt.py +++ b/scripts/joinmarket-qt.py @@ -52,17 +52,16 @@ JM_CORE_VERSION = '0.3.5' #Version of this Qt script specifically JM_GUI_VERSION = '7' -from jmclient import (load_program_config, get_network, SegwitWallet, - get_p2sh_vbyte, get_p2pk_vbyte, jm_single, validate_address, - get_log, weighted_order_choose, Taker, - JMClientProtocolFactory, WalletError, - start_reactor, get_schedule, get_tumble_schedule, - schedule_to_text, create_wallet_file, - get_blockchain_interface_instance, sync_wallet, direct_send, - RegtestBitcoinCoreInterface, tweak_tumble_schedule, - human_readable_schedule_entry, tumbler_taker_finished_update, - get_tumble_log, restart_wait, tumbler_filter_orders_callback, - wallet_generate_recover_bip39, wallet_display) +from jmclient import ( + load_program_config, get_network, open_wallet, get_wallet_path, + get_p2sh_vbyte, get_p2pk_vbyte, jm_single, validate_address, get_log, + weighted_order_choose, Taker, JMClientProtocolFactory, WalletError, + start_reactor, get_schedule, get_tumble_schedule, schedule_to_text, + get_blockchain_interface_instance, sync_wallet, + direct_send, RegtestBitcoinCoreInterface, tweak_tumble_schedule, + human_readable_schedule_entry, tumbler_taker_finished_update, + get_tumble_log, restart_wait, tumbler_filter_orders_callback, + wallet_generate_recover_bip39, wallet_display) from qtsupport import (ScheduleWizard, TumbleRestartWizard, warnings, config_tips, config_types, TaskThread, QtHandler, XStream, Buttons, @@ -1373,16 +1372,14 @@ class JMMainWindow(QMainWindow): def loadWalletFromBlockchain(self, firstarg=None, pwd=None, restart_cb=None): if (firstarg and pwd) or (firstarg and get_network() == 'testnet'): + wallet_path = get_wallet_path(str(firstarg), None) try: - self.wallet = SegwitWallet( - str(firstarg), - pwd, - max_mix_depth=jm_single().config.getint( - "GUI", "max_mix_depth"), - gaplimit=jm_single().config.getint("GUI", "gaplimit")) - except WalletError: + self.wallet = open_wallet( + wallet_path, ask_for_password=False, password=pwd, + gap_limit=jm_single().config.getint("GUI", "gaplimit")) + except Exception as e: JMQtMessageBox(self, - "Wrong password", + str(e), mbtype='warn', title="Error") return False diff --git a/scripts/tumbler.py b/scripts/tumbler.py index e3664d7..e843c6a 100644 --- a/scripts/tumbler.py +++ b/scripts/tumbler.py @@ -1,26 +1,19 @@ #! /usr/bin/env python from __future__ import absolute_import, print_function -import random import sys -import threading -from optparse import OptionParser from twisted.internet import reactor -import time import os import pprint -import copy -import logging from twisted.python.log import startLogging -from jmclient import (Taker, load_program_config, get_schedule, - weighted_order_choose, JMClientProtocolFactory, - start_reactor, validate_address, jm_single, WalletError, - get_wallet_cls, sync_wallet, get_tumble_schedule, - RegtestBitcoinCoreInterface, estimate_tx_fee, - tweak_tumble_schedule, human_readable_schedule_entry, - schedule_to_text, restart_waiter, get_tumble_log, - tumbler_taker_finished_update, tumbler_filter_orders_callback) - +from jmclient import ( + Taker, load_program_config, get_schedule, weighted_order_choose, + JMClientProtocolFactory, start_reactor, validate_address, jm_single, + get_wallet_path, open_test_wallet_maybe, WalletError, sync_wallet, + get_tumble_schedule, RegtestBitcoinCoreInterface, estimate_tx_fee, + tweak_tumble_schedule, human_readable_schedule_entry, schedule_to_text, + restart_waiter, get_tumble_log, tumbler_taker_finished_update, + tumbler_filter_orders_callback) from jmbase.support import get_log, debug_dump_object, get_password from cli_options import get_tumbler_parser log = get_log() @@ -39,20 +32,9 @@ def main(): #Load the wallet wallet_name = args[0] max_mix_depth = options['mixdepthsrc'] + options['mixdepthcount'] - if not os.path.exists(os.path.join('wallets', wallet_name)): - wallet = get_wallet_cls()(wallet_name, None, max_mix_depth) - else: - while True: - try: - pwd = get_password("Enter wallet decryption passphrase: ") - wallet = get_wallet_cls()(wallet_name, pwd, max_mix_depth) - except WalletError: - print("Wrong password, try again.") - continue - except Exception as e: - print("Failed to load wallet, error message: " + repr(e)) - sys.exit(0) - break + wallet_path = get_wallet_path(wallet_name, None) + wallet = open_test_wallet_maybe(wallet_path, wallet_name, max_mix_depth) + if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == "electrum-server": jm_single().bc_interface.synctype = "with-script" From 89b5cd4280020281387dcf7445a33e7b8e5b3c6c Mon Sep 17 00:00:00 2001 From: undeath Date: Mon, 25 Dec 2017 13:51:21 +0100 Subject: [PATCH 07/23] add new wallet classes to existing tests --- jmclient/jmclient/__init__.py | 2 +- jmclient/jmclient/taker.py | 14 +- jmclient/jmclient/wallet_utils.py | 23 +- jmclient/test/commontest.py | 94 +++--- jmclient/test/taker_test_data.py | 6 +- jmclient/test/test_podle.py | 5 - jmclient/test/test_taker.py | 61 ++-- jmclient/test/test_tx_creation.py | 18 +- jmclient/test/test_wallets.py | 522 +----------------------------- test/common.py | 36 ++- test/test_segwit.py | 203 ++++++------ 11 files changed, 222 insertions(+), 762 deletions(-) diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index 8174bd9..1dd44b1 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -46,7 +46,7 @@ from .taker_utils import (tumbler_taker_finished_update, restart_waiter, from .wallet_utils import ( wallet_tool_main, wallet_generate_recover_bip39, open_wallet, open_test_wallet_maybe, create_wallet, get_wallet_cls, get_wallet_path, - wallet_display) + wallet_display, SewgitTestWallet) from .maker import Maker from .yieldgenerator import YieldGenerator, YieldGeneratorBasic, ygmain # Set default logging handler to avoid "No handler found" warnings. diff --git a/jmclient/jmclient/taker.py b/jmclient/jmclient/taker.py index 0214a40..43e5295 100644 --- a/jmclient/jmclient/taker.py +++ b/jmclient/jmclient/taker.py @@ -4,9 +4,7 @@ from __future__ import print_function import base64 import pprint import random -import sys -import time -import copy +from itertools import chain import btc from jmclient.configure import jm_single, get_p2pk_vbyte, get_p2sh_vbyte @@ -99,6 +97,7 @@ class Taker(object): self.waiting_for_conf = False self.txid = None self.schedule_index = -1 + self.utxos = {} self.tdestaddrs = [] if not tdestaddrs else tdestaddrs #allow custom wallet-based clients to use their own signing code; #currently only setting "wallet" is allowed, calls wallet.sign_tx(tx) @@ -665,9 +664,12 @@ class Taker(object): #in the transaction, about to be consumed, rather than use #random utxos that will persist after. At this step we also #allow use of external utxos in the json file. - if self.wallet.unspent: + if any(self.wallet.get_utxos_by_mixdepth_().values()): + utxos = {} + for mdutxo in self.wallet.get_utxos_by_mixdepth().values(): + utxos.update(mdutxo) priv_utxo_pairs, to, ts = priv_utxo_pairs_from_utxos( - self.wallet.unspent, age, amt) + utxos, age, amt) #Pre-filter the set of external commitments that work for this #transaction according to its size and age. dummy, extdict = get_podle_commitments() @@ -688,7 +690,7 @@ class Taker(object): "Commitment sourced OK") else: errmsgheader, errmsg = generate_podle_error_string(priv_utxo_pairs, - to, ts, self.wallet.unspent, self.cjamount, + to, ts, self.wallet.get_utxos_by_mixdepth(), self.cjamount, jm_single().config.get("POLICY", "taker_utxo_age"), jm_single().config.get("POLICY", "taker_utxo_amtpercent")) diff --git a/jmclient/jmclient/wallet_utils.py b/jmclient/jmclient/wallet_utils.py index 9afe41e..fbef685 100644 --- a/jmclient/jmclient/wallet_utils.py +++ b/jmclient/jmclient/wallet_utils.py @@ -1,14 +1,12 @@ from __future__ import print_function import json import os -import pprint import sys import sqlite3 import binascii from datetime import datetime from mnemonic import Mnemonic from optparse import OptionParser -import getpass from jmclient import (get_network, WALLET_IMPLEMENTATIONS, Storage, podle, encryptData, get_p2sh_vbyte, get_p2pk_vbyte, jm_single, mn_decode, mn_encode, BitcoinCoreInterface, JsonRpcError, sync_wallet, WalletError, @@ -16,6 +14,11 @@ from jmclient import (get_network, WALLET_IMPLEMENTATIONS, Storage, podle, from jmbase.support import get_password import jmclient.btc as btc + +class SewgitTestWallet(ImportWalletMixin, BIP49Wallet): + TYPE = 'p2sh-p2wpkh' + + def get_wallettool_parser(): description = ( 'Use this script to monitor and manage your Joinmarket wallet.\n' @@ -836,7 +839,8 @@ def create_wallet(path, password, max_mixdepth, **kwargs): **kwargs) -def open_test_wallet_maybe(path, seed, max_mixdepth, **kwargs): +def open_test_wallet_maybe(path, seed, max_mixdepth, + test_wallet_cls=SewgitTestWallet, **kwargs): """ Create a volatile test wallet if path is a hex-encoded string of length 64, otherwise run open_wallet(). @@ -850,22 +854,19 @@ def open_test_wallet_maybe(path, seed, max_mixdepth, **kwargs): returns: wallet object """ - class SewgitTestWallet(ImportWalletMixin, BIP49Wallet): - TYPE = 'p2sh-p2wpkh' - - if len(seed) == SewgitTestWallet.ENTROPY_BYTES * 2: + if len(seed) == test_wallet_cls.ENTROPY_BYTES * 2: try: seed = binascii.unhexlify(seed) except binascii.Error: pass else: storage = VolatileStorage() - SewgitTestWallet.initialize( + test_wallet_cls.initialize( storage, get_network(), max_mixdepth=max_mixdepth, entropy=seed) assert 'ask_for_password' not in kwargs assert 'read_only' not in kwargs - return SewgitTestWallet(storage, **kwargs) + return test_wallet_cls(storage, **kwargs) return open_wallet(path, **kwargs) @@ -920,8 +921,8 @@ def get_wallet_cls_from_storage(storage): def wallet_sanity_check(wallet): if wallet.network != get_network(): - raise Exception("Wallet network mismatch: we are on {} but wallet is " - "on {}".format(get_network(), wallet.network)) + raise Exception("Wallet network mismatch: we are on '{}' but wallet " + "is on '{}'.".format(get_network(), wallet.network)) def get_wallet_path(file_name, wallet_dir): diff --git a/jmclient/test/commontest.py b/jmclient/test/commontest.py index 8cfba04..9ce12c2 100644 --- a/jmclient/test/commontest.py +++ b/jmclient/test/commontest.py @@ -1,19 +1,15 @@ #! /usr/bin/env python -from __future__ import absolute_import +from __future__ import absolute_import, print_function '''Some helper functions for testing''' -import sys import os -import time import binascii -import pexpect import random -import subprocess -import platform from decimal import Decimal -from jmclient import (jm_single, Wallet, get_log, estimate_tx_fee, - BlockchainInterface, get_p2sh_vbyte) +from jmclient import ( + jm_single, open_test_wallet_maybe, get_log, estimate_tx_fee, + BlockchainInterface, get_p2sh_vbyte, BIP32Wallet, SegwitLegacyWallet) from jmbase.support import chunks import jmbitcoin as btc @@ -35,6 +31,8 @@ class DummyBlockchainInterface(BlockchainInterface): pass def sync_unspent(self, wallet): pass + def import_addresses(self, addr_list, wallet_name): + pass def outputs_watcher(self, wallet_name, notifyaddr, tx_output_set, uf, cf, tf): pass @@ -106,29 +104,21 @@ class DummyBlockchainInterface(BlockchainInterface): def estimate_fee_per_kb(self, N): return 30000 -class TestWallet(Wallet): - """Implementation of wallet - that allows passing in a password - for removal of command line interrupt. - """ - def __init__(self, - seedarg, - max_mix_depth=2, - gaplimit=6, - extend_mixdepth=False, - storepassword=False, - pwd=None): - self.given_pwd = pwd - super(TestWallet, self).__init__(seedarg, - max_mix_depth, - gaplimit, - extend_mixdepth, - storepassword) - - def read_wallet_file_data(self, filename): - return super(TestWallet, self).read_wallet_file_data( - filename, self.given_pwd) +def create_wallet_for_sync(wallet_structure, a, **kwargs): + #We need a distinct seed for each run so as not to step over each other; + #make it through a deterministic hash + seedh = btc.sha256("".join([str(x) for x in a]))[:32] + return make_wallets( + 1, [wallet_structure], fixed_seeds=[seedh], **kwargs)[0]['wallet'] + + +def binarize_tx(tx): + for o in tx['outs']: + o['script'] = binascii.unhexlify(o['script']) + for i in tx['ins']: + i['outpoint']['hash'] = binascii.unhexlify(i['outpoint']['hash']) + def make_sign_and_push(ins_full, wallet, @@ -150,17 +140,17 @@ def make_sign_and_push(ins_full, 'address': output_addr}, {'value': total - amount - fee_est, 'address': change_addr}] - tx = btc.mktx(ins, outs) - de_tx = btc.deserialize(tx) + de_tx = btc.deserialize(btc.mktx(ins, outs)) + scripts = {} for index, ins in enumerate(de_tx['ins']): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - addr = ins_full[utxo]['address'] - priv = wallet.get_key_from_addr(addr) - if index % 2: - priv = binascii.unhexlify(priv) - tx = btc.sign(tx, index, priv, hashcode=hashcode) + script = wallet.addr_to_script(ins_full[utxo]['address']) + scripts[index] = (script, ins_full[utxo]['value']) + binarize_tx(de_tx) + de_tx = wallet.sign_tx(de_tx, scripts, hashcode=hashcode) #pushtx returns False on any error - print btc.deserialize(tx) + print(de_tx) + tx = binascii.hexlify(btc.serialize(de_tx)) push_succeed = jm_single().bc_interface.pushtx(tx) if push_succeed: return btc.txhash(tx) @@ -173,8 +163,9 @@ def make_wallets(n, sdev_amt=0, start_index=0, fixed_seeds=None, - test_wallet=False, - passwords=None): + wallet_cls=SegwitLegacyWallet, + mixdepths=5, + populate_internal=False): '''n: number of wallets to be created wallet_structure: array of n arrays , each subarray specifying the number of addresses to be populated with coins @@ -182,34 +173,33 @@ def make_wallets(n, mean_amt: the number of coins (in btc units) in each address as above sdev_amt: if randomness in amouts is desired, specify here. Returns: a dict of dicts of form {0:{'seed':seed,'wallet':Wallet object},1:..,} - Default Wallet constructor is joinmarket.Wallet, else use TestWallet, - which takes a password parameter as in the list passwords. ''' + # FIXME: this is basically the same code as test/common.py + assert mixdepths > 0 if len(wallet_structures) != n: raise Exception("Number of wallets doesn't match wallet structures") if not fixed_seeds: - seeds = chunks(binascii.hexlify(os.urandom(15 * n)), 15 * 2) + seeds = chunks(binascii.hexlify(os.urandom(BIP32Wallet.ENTROPY_BYTES * n)), + BIP32Wallet.ENTROPY_BYTES * 2) else: seeds = fixed_seeds wallets = {} for i in range(n): - if test_wallet: - w = Wallet(seeds[i], passwords[i], max_mix_depth=5) - else: - w = Wallet(seeds[i], None, max_mix_depth=5) + assert len(seeds[i]) == BIP32Wallet.ENTROPY_BYTES * 2 + + w = open_test_wallet_maybe(seeds[i], seeds[i], mixdepths - 1, + test_wallet_cls=wallet_cls) + wallets[i + start_index] = {'seed': seeds[i], 'wallet': w} - for j in range(5): + for j in range(mixdepths): for k in range(wallet_structures[i][j]): deviation = sdev_amt * random.random() amt = mean_amt - sdev_amt / 2.0 + deviation if amt < 0: amt = 0.001 amt = float(Decimal(amt).quantize(Decimal(10)**-8)) jm_single().bc_interface.grab_coins( - wallets[i + start_index]['wallet'].get_external_addr(j), - amt) - #reset the index so the coins can be seen if running in same script - wallets[i + start_index]['wallet'].index[j][0] -= wallet_structures[i][j] + w.get_new_addr(j, populate_internal), amt) return wallets diff --git a/jmclient/test/taker_test_data.py b/jmclient/test/taker_test_data.py index 278d10d..0ee3c09 100644 --- a/jmclient/test/taker_test_data.py +++ b/jmclient/test/taker_test_data.py @@ -45,11 +45,11 @@ t_chosen_orders = {u'J659UPUSLLjHJpaB': {u'cjfee': u'0.0002', """ t_utxos_by_mixdepth = {0: {u'534b635ed8891f16c4ec5b8236ae86164783903e8e8bb47fa9ef2ca31f3c2d7a:0': {'address': u'mrcNu71ztWjAQA6ww9kHiW3zBWSQidHXTQ', 'value': 200000000}}, - 1: {u'0780d6e5e381bff01a3519997bb4fcba002493103a198fde334fd264f9835d75:1': {'address': u'mvtY8DVgn3TtvjHbVsauYoSQjAhNqVyqmM', + 1: {u'0780d6e5e381bff01a3519997bb4fcba002493103a198fde334fd264f9835d75:1': {'address': u'n31WD8pkfAjg2APV78GnbDTdZb1QonBi5D', 'value': 200000000}, - u'7e574db96a4d43a99786b3ea653cda9e4388f377848f489332577e018380cff1:0': {'address': u'n3nELhmU2D7ebGYzJnGFWgVDK3cYErmTcQ', + u'7e574db96a4d43a99786b3ea653cda9e4388f377848f489332577e018380cff1:0': {'address': u'mmVEKH61BZbLbnVEmk9VmojreB4G4PmBPd', 'value': 200000000}, - u'dd9711a2ef340750db21efb761f5f7d665d94b312332dc354e252c77e9c48349:0': {'address': u'mxeLuX8PP7qLkcM8uarHmdZyvP1b5e1Ynf', + u'dd9711a2ef340750db21efb761f5f7d665d94b312332dc354e252c77e9c48349:0': {'address': u'msxyyydNXTiBmt3SushXbH5Qh2ukBAThk3', 'value': 200000000}}, 2: {}, 3: {}, diff --git a/jmclient/test/test_podle.py b/jmclient/test/test_podle.py index f69bc5a..7cf7a2c 100644 --- a/jmclient/test/test_podle.py +++ b/jmclient/test/test_podle.py @@ -7,11 +7,6 @@ import binascii import json import pytest import copy -import subprocess -import signal -from commontest import make_wallets -import time -from pprint import pformat from jmclient import (load_program_config, get_log, jm_single, generate_podle, generate_podle_error_string, set_commitment_file, get_commitment_file, PoDLE, get_podle_commitments, diff --git a/jmclient/test/test_taker.py b/jmclient/test/test_taker.py index b91bca7..4d9310a 100644 --- a/jmclient/test/test_taker.py +++ b/jmclient/test/test_taker.py @@ -9,26 +9,53 @@ import shutil import pytest import json from base64 import b64encode -from jmclient import (load_program_config, jm_single, set_commitment_file, - get_commitment_file, AbstractWallet, Taker, SegwitWallet, - get_p2sh_vbyte, get_p2pk_vbyte) +from jmclient import ( + load_program_config, jm_single, set_commitment_file, get_commitment_file, + LegacyWallet, Taker, VolatileStorage, get_p2sh_vbyte, get_network) from taker_test_data import (t_utxos_by_mixdepth, t_selected_utxos, t_orderbook, t_maker_response, t_chosen_orders, t_dummy_ext) -class DummyWallet(AbstractWallet): +class DummyWallet(LegacyWallet): def __init__(self): - super(DummyWallet, self).__init__() - self.max_mix_depth = 5 + storage = VolatileStorage() + super(DummyWallet, self).initialize(storage, get_network(), + max_mixdepth=5) + super(DummyWallet, self).__init__(storage) + self._add_utxos() self.inject_addr_get_failure = False + def _add_utxos(self): + for md, utxo in t_utxos_by_mixdepth.items(): + for i, (txid, data) in enumerate(utxo.items()): + txid, index = txid.split(':') + self._utxos.add_utxo(binascii.unhexlify(txid), int(index), + (b'dummy', md, i), data['value'], md) + def get_utxos_by_mixdepth(self, verbose=True): return t_utxos_by_mixdepth + def get_utxos_by_mixdepth_(self, verbose=True): + utxos = self.get_utxos_by_mixdepth(verbose) + + utxos_conv = {} + for md, utxo_data in utxos.items(): + md_utxo = utxos_conv.setdefault(md, {}) + for i, (utxo_hex, data) in enumerate(utxo_data.items()): + utxo, index = utxo_hex.split(':') + data_conv = { + 'script': self._ENGINE.address_to_script(data['address']), + 'path': (b'dummy', md, i), + 'value': data['value'] + } + md_utxo[(binascii.unhexlify(utxo), int(index))] = data_conv + + return utxos_conv + def select_utxos(self, mixdepth, amount): if amount > self.get_balance_by_mixdepth()[mixdepth]: raise Exception("Not enough funds") - return self.get_utxos_by_mixdepth()[mixdepth] + return t_utxos_by_mixdepth[mixdepth] def get_internal_addr(self, mixing_depth): if self.inject_addr_get_failure: @@ -135,22 +162,8 @@ def test_make_commitment(createcmtdata, failquery, external): mixdepth = 0 amount = 110000000 taker = get_taker([(mixdepth, amount, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw")]) - taker.wallet.unspent = {'f34b635ed8891f16c4ec5b8236ae86164783903e8e8bb47fa9ef2ca31f3c2d7a:0': - {'address': u'n31WD8pkfAjg2APV78GnbDTdZb1QonBi5D', - 'value': 10000000}, - 'f780d6e5e381bff01a3519997bb4fcba002493103a198fde334fd264f9835d75:1': - {'address': u'mmVEKH61BZbLbnVEmk9VmojreB4G4PmBPd', - 'value': 20000000}, - 'fe574db96a4d43a99786b3ea653cda9e4388f377848f489332577e018380cff1:0': - {'address': u'msxyyydNXTiBmt3SushXbH5Qh2ukBAThk3', - 'value': 500000000}, - 'fd9711a2ef340750db21efb761f5f7d665d94b312332dc354e252c77e9c48349:0': - {'address': u'musGZczug3BAbqobmYherywCwL9REgNaNm', - 'value': 500000000}} taker.cjamount = amount - taker.input_utxos = {'f34b635ed8891f16c4ec5b8236ae86164783903e8e8bb47fa9ef2ca31f3c2d7a:0': - {'address': u'n31WD8pkfAjg2APV78GnbDTdZb1QonBi5D', - 'value': 10000000}} + taker.input_utxos = t_utxos_by_mixdepth[0] if failquery: jm_single().bc_interface.setQUSFail(True) taker.make_commitment() @@ -461,7 +474,3 @@ def createcmtdata(request): load_program_config() jm_single().bc_interface = DummyBlockchainInterface() jm_single().config.set("BLOCKCHAIN", "network", "testnet") - - - - \ No newline at end of file diff --git a/jmclient/test/test_tx_creation.py b/jmclient/test/test_tx_creation.py index 79cc17c..07975a7 100644 --- a/jmclient/test/test_tx_creation.py +++ b/jmclient/test/test_tx_creation.py @@ -3,18 +3,15 @@ from __future__ import absolute_import '''Test of unusual transaction types creation and push to network to check validity.''' -import sys -import os import time import binascii -import random from commontest import make_wallets, make_sign_and_push import jmbitcoin as bitcoin import pytest -from jmclient import (load_program_config, jm_single, sync_wallet, - get_p2pk_vbyte, get_log, Wallet, select_gradual, - select, select_greedy, select_greediest, estimate_tx_fee) +from jmclient import ( + load_program_config, jm_single, sync_wallet, get_p2pk_vbyte, get_log, + select_gradual, select, select_greedy, select_greediest, estimate_tx_fee) log = get_log() #just a random selection of pubkeys for receiving multisigs; @@ -177,15 +174,6 @@ def test_create_sighash_txs(setup_tx_creation): txid = make_sign_and_push(ins_full, wallet, amount, hashcode=sighash) assert txid - #Create an invalid sighash single (too many inputs) - extra = wallet.select_utxos(4, 100000000) #just a few more inputs - ins_full.update(extra) - with pytest.raises(Exception) as e_info: - txid = make_sign_and_push(ins_full, - wallet, - amount, - hashcode=bitcoin.SIGHASH_SINGLE) - #trigger insufficient funds with pytest.raises(Exception) as e_info: fake_utxos = wallet.select_utxos(4, 1000000000) diff --git a/jmclient/test/test_wallets.py b/jmclient/test/test_wallets.py index d16147e..c9bce63 100644 --- a/jmclient/test/test_wallets.py +++ b/jmclient/test/test_wallets.py @@ -6,27 +6,15 @@ import sys import os import time import binascii -import random -import subprocess -import datetime -import unittest from mnemonic import Mnemonic -from ConfigParser import SafeConfigParser, NoSectionError -from decimal import Decimal -from commontest import (interact, make_wallets, - make_sign_and_push) +from commontest import create_wallet_for_sync, make_sign_and_push import json -import jmbitcoin as bitcoin import pytest -from jmclient import (load_program_config, jm_single, sync_wallet, - AbstractWallet, get_p2pk_vbyte, get_log, Wallet, select, - select_gradual, select_greedy, select_greediest, - estimate_tx_fee, encryptData, get_network, WalletError, - BitcoinCoreInterface, SegwitWallet, - wallet_generate_recover_bip39, decryptData, encryptData) -from jmbase.support import chunks -from taker_test_data import t_obtained_tx, t_raw_signed_tx +from jmclient import ( + load_program_config, jm_single, sync_wallet, get_log, + estimate_tx_fee, BitcoinCoreInterface) +from taker_test_data import t_raw_signed_tx testdir = os.path.dirname(os.path.realpath(__file__)) log = get_log() @@ -51,8 +39,7 @@ def do_tx(wallet, amount): def test_query_utxo_set(setup_wallets): load_program_config() jm_single().bc_interface.tick_forward_chain_interval = 1 - wallet = create_wallet_for_sync("wallet4utxo.json", "4utxo", - [2, 3, 0, 0, 0], + wallet = create_wallet_for_sync([2, 3, 0, 0, 0], ["wallet4utxo.json", "4utxo", [2, 3]]) sync_wallet(wallet, fast=True) txid = do_tx(wallet, 90000000) @@ -72,199 +59,9 @@ def test_query_utxo_set(setup_wallets): assert res3 == [None] -def create_wallet_for_sync(wallet_file, password, wallet_structure, a): - #Prepare a testnet wallet file for this wallet - password_key = bitcoin.bin_dbl_sha256(password) - #We need a distinct seed for each run so as not to step over each other; - #make it through a deterministic hash - seedh = bitcoin.sha256("".join([str(x) for x in a]))[:32] - encrypted_seed = encryptData(password_key, seedh.decode('hex')) - timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") - walletfilejson = {'creator': 'joinmarket project', - 'creation_time': timestamp, - 'encrypted_seed': encrypted_seed.encode('hex'), - 'network': get_network()} - walletfile = json.dumps(walletfilejson) - if not os.path.exists('wallets'): - os.makedirs('wallets') - with open(os.path.join('wallets', wallet_file), "wb") as f: - f.write(walletfile) - #The call to Wallet() in make_wallets should now find the file - #and read from it: - return make_wallets(1, - [wallet_structure], - fixed_seeds=[wallet_file], - test_wallet=True, - passwords=[password])[0]['wallet'] - - -@pytest.mark.parametrize( - "num_txs, fake_count, wallet_structure, amount, wallet_file, password", - [ - (3, 13, [11, 3, 4, 5, 6], 150000000, 'test_import_wallet.json', - 'import-pwd'), - #Uncomment all these for thorough tests. Passing currently. - #Lots of used addresses - #(7, 1, [51, 3, 4, 5, 6], 150000000, 'test_import_wallet.json', - # 'import-pwd'), - #(3, 1, [3, 1, 4, 5, 6], 50000000, 'test_import_wallet.json', - # 'import-pwd'), - #No spams/fakes - #(2, 0, [5, 20, 1, 1, 1], 50000000, 'test_import_wallet.json', - # 'import-pwd'), - #Lots of transactions and fakes - #(25, 30, [30, 20, 1, 1, 1], 50000000, 'test_import_wallet.json', - # 'import-pwd'), - ]) -def test_wallet_sync_with_fast(setup_wallets, num_txs, fake_count, - wallet_structure, amount, wallet_file, password): - jm_single().bc_interface.tick_forward_chain_interval = 1 - wallet = create_wallet_for_sync(wallet_file, password, wallet_structure, - [num_txs, fake_count, wallet_structure, - amount, wallet_file, password]) - sync_count = 0 - jm_single().bc_interface.wallet_synced = False - while not jm_single().bc_interface.wallet_synced: - sync_wallet(wallet) - sync_count += 1 - #avoid infinite loop - assert sync_count < 10 - log.debug("Tried " + str(sync_count) + " times") - - assert jm_single().bc_interface.wallet_synced - assert not jm_single().bc_interface.fast_sync_called - #do some transactions with the wallet, then close, then resync - for i in range(num_txs): - do_tx(wallet, amount) - log.debug("After doing a tx, index is now: " + str(wallet.index)) - #simulate a spammer requesting a bunch of transactions. This - #mimics what happens in CoinJoinOrder.__init__() - for j in range(fake_count): - #Note that as in a real script run, - #the initial call to sync_wallet will - #have set wallet_synced to True, so these will - #trigger actual imports. - cj_addr = wallet.get_internal_addr(0) - change_addr = wallet.get_internal_addr(0) - wallet.update_cache_index() - log.debug("After doing a spam, index is now: " + str(wallet.index)) - - assert wallet.index[0][1] == num_txs + fake_count * 2 * num_txs - - #Attempt re-sync, simulating a script restart. - - jm_single().bc_interface.wallet_synced = False - sync_count = 0 - #Probably should be fixed in main code: - #wallet.index_cache is only assigned in Wallet.__init__(), - #meaning a second sync in the same script, after some transactions, - #will not know about the latest index_cache value (see is_index_ahead_of_cache), - #whereas a real re-sync will involve reading the cache from disk. - #Hence, simulation of the fact that the cache index will - #be read from the file on restart: - wallet.index_cache = wallet.index - - while not jm_single().bc_interface.wallet_synced: - #Wallet.__init__() resets index to zero. - wallet.index = [] - for i in range(5): - wallet.index.append([0, 0]) - #Wallet.__init__() also updates the cache index - #from file, but we can reuse from the above pre-loop setting, - #since nothing else in sync will overwrite the cache. - - #for regtest add_watchonly_addresses does not exit(), so can - #just repeat as many times as possible. This might - #be usable for non-test code (i.e. no need to restart the - #script over and over again)? - sync_count += 1 - log.debug("TRYING SYNC NUMBER: " + str(sync_count)) - sync_wallet(wallet, fast=True) - assert jm_single().bc_interface.fast_sync_called - #avoid infinite loop on failure. - assert sync_count < 10 - #Wallet should recognize index_cache on fast sync, so should not need to - #run sync process more than once. - assert sync_count == 1 - #validate the wallet index values after sync - for i, ws in enumerate(wallet_structure): - assert wallet.index[i][0] == ws #spends into external only - #Same number as above; note it includes the spammer's extras. - assert wallet.index[0][1] == num_txs + fake_count * 2 * num_txs - assert wallet.index[1][1] == num_txs #one change per transaction - for i in range(2, 5): - assert wallet.index[i][1] == 0 #unused - - #Now try to do more transactions as sanity check. - do_tx(wallet, 50000000) - - -@pytest.mark.parametrize( - "wallet_structure, wallet_file, password, ic", - [ - #As usual, more test cases are preferable but time - #of build test is too long, so only one activated. - #([11,3,4,5,6], 'test_import_wallet.json', 'import-pwd', - # [(12,3),(100,99),(7, 40), (200, 201), (10,0)] - # ), - ([1, 3, 0, 2, 9], 'test_import_wallet.json', 'import-pwd', - [(1, 7), (100, 99), (0, 0), (200, 201), (21, 41)]), - ]) -def test_wallet_sync_from_scratch(setup_wallets, wallet_structure, wallet_file, - password, ic): - """Simulate a scenario in which we use a new bitcoind, thusly: - generate a new wallet and simply pretend that it has an existing - index_cache. This will force import of all addresses up to - the index_cache values. - """ - wallet = create_wallet_for_sync(wallet_file, password, wallet_structure, - [wallet_structure, wallet_file, password, - ic]) - sync_count = 0 - jm_single().bc_interface.wallet_synced = False - wallet.index_cache = ic - while not jm_single().bc_interface.wallet_synced: - wallet.index = [] - for i in range(5): - wallet.index.append([0, 0]) - #will call with fast=False but index_cache exists; should use slow-sync - sync_wallet(wallet) - sync_count += 1 - #avoid infinite loop - assert sync_count < 10 - log.debug("Tried " + str(sync_count) + " times") - #after #586 we expect to ALWAYS succeed within 2 rounds - assert sync_count <= 2 - #for each external branch, the new index may be higher than - #the original index_cache if there was a higher used address - expected_wallet_index = [] - for i, val in enumerate(wallet_structure): - if val > wallet.index_cache[i][0]: - expected_wallet_index.append([val, wallet.index_cache[i][1]]) - else: - expected_wallet_index.append([wallet.index_cache[i][0], - wallet.index_cache[i][1]]) - assert wallet.index == expected_wallet_index - log.debug("This is wallet unspent: ") - log.debug(json.dumps(wallet.unspent, indent=4)) - - """Purely blockchaininterface related error condition tests""" -def test_index_ahead_cache(setup_wallets): - """Artificial test; look into finding a sync mode that triggers this - """ - - class NonWallet(object): - pass - - wallet = NonWallet() - wallet.index_cache = [[0, 0], [0, 2]] - from jmclient.blockchaininterface import is_index_ahead_of_cache - assert is_index_ahead_of_cache(wallet, 3, 1) - - def test_wrong_network_bci(setup_wallets): rpc = jm_single().bc_interface.jsonRpc with pytest.raises(Exception) as e_info: @@ -293,31 +90,6 @@ def test_absurd_fee(setup_wallets): load_program_config() -def test_abstract_wallet(setup_wallets): - - class DoNothingWallet(AbstractWallet): - pass - - for algo in ["default", "gradual", "greedy", "greediest", "none"]: - jm_single().config.set("POLICY", "merge_algorithm", algo) - if algo == "none": - with pytest.raises(Exception) as e_info: - dnw = DoNothingWallet() - #also test if the config is blank - jm_single().config = SafeConfigParser() - dnw = DoNothingWallet() - assert dnw.utxo_selector == select - else: - dnw = DoNothingWallet() - assert not dnw.get_key_from_addr("a") - assert not dnw.get_utxos_by_mixdepth() - assert not dnw.get_external_addr(1) - assert not dnw.get_internal_addr(0) - dnw.update_cache_index() - dnw.remove_old_utxos("a") - dnw.add_new_utxos("b", "c") - load_program_config() - def check_bip39_case(vectors, language="english"): mnemo = Mnemonic(language) for v in vectors: @@ -346,288 +118,6 @@ def test_bip39_vectors(setup_wallets): vectors = filter(lambda x: len(x[1].split())==12, vectors) check_bip39_case(vectors) -@pytest.mark.parametrize( - "pwd, me, valid", [ - ("asingleword", "1234aaaaaaaaaaaaaaaaa", True), - ("a whole set of words", "a whole set of words", True), - ("wordwithtrailingspaces ", "A few words with trailing ", True), - ("monkey", "verylongpasswordindeedxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", True), - ("blablah", "invalidcontainsnonascii\xee", False) - ]) -def test_create_bip39_with_me(setup_wallets, pwd, me, valid): - def dummyDisplayWords(a, b): - pass - def getMnemonic(): - return ("legal winner thank year wave sausage worth useful legal winner thank yellow", - me) - def getPassword(): - return pwd - def getWalletFileName(): - return "bip39-test-wallet-name-from-callback.json" - def promptMnemonicExtension(): - return me - if os.path.exists(os.path.join("wallets", getWalletFileName())): - os.remove(os.path.join("wallets", getWalletFileName())) - success = wallet_generate_recover_bip39("generate", - "wallets", - "wallet.json", - callbacks=(dummyDisplayWords, - getMnemonic, - getPassword, - getWalletFileName, - promptMnemonicExtension)) - if not valid: - #wgrb39 returns false for failed wallet creation case - assert not success - return - assert success - #open the wallet file, and decrypt the encrypted mnemonic extension and check - #it's the one we intended. - with open(os.path.join("wallets", getWalletFileName()), 'r') as f: - walletdata = json.load(f) - password_key = bitcoin.bin_dbl_sha256(getPassword()) - cleartext = decryptData(password_key, - walletdata['encrypted_mnemonic_extension'].decode('hex')) - assert len(cleartext) >= 79 - #throws if not len == 3 - padding, me2, checksum = cleartext.split('\xff') - strippedme = me.strip() - assert strippedme == me2 - assert checksum == bitcoin.dbl_sha256(strippedme)[:8] - #also test recovery from this combination of mnemonic + extension - if os.path.exists(os.path.join("wallets", getWalletFileName())): - os.remove(os.path.join("wallets", getWalletFileName())) - success = wallet_generate_recover_bip39("recover", "wallets", "wallet.json", - callbacks=(dummyDisplayWords, - getMnemonic, - getPassword, - getWalletFileName, - None)) - assert success - with open(os.path.join("wallets", getWalletFileName()), 'r') as f: - walletdata = json.load(f) - password_key = bitcoin.bin_dbl_sha256(getPassword()) - cleartext = decryptData(password_key, - walletdata['encrypted_entropy'].decode('hex')).encode('hex') - assert cleartext == "7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f" - -def create_default_testnet_wallet(): - walletdir = "wallets" - testwalletname = "testwallet.json" - pathtowallet = os.path.join(walletdir, testwalletname) - if os.path.exists(pathtowallet): - os.remove(pathtowallet) - seed = "deadbeef" - return (walletdir, pathtowallet, testwalletname, - SegwitWallet(seed, - None, - 5, - 6, - extend_mixdepth=False, - storepassword=False)) - -@pytest.mark.parametrize( - "includecache, wrongnet, storepwd, extendmd, pwdnumtries", [ - (False, False, False, False, 100), - (True, False, False, True, 1), - (False, True, False, False, 1), - (False, False, True, False, 1) - ]) -def test_wallet_create(setup_wallets, includecache, wrongnet, storepwd, - extendmd, pwdnumtries): - walletdir, pathtowallet, testwalletname, wallet = create_default_testnet_wallet( - ) - assert wallet.get_key( - 4, 1, - 17) == "96095d7542e4e832c476b9df7e49ca9e5be61ad3bb8c8a3bdd8e141e2f4caf9101" - assert wallet.get_addr(2, 0, 5) == "2NBUxbEQrGPKrYCV6d4o7Y4AtJ34Uy6gZZg" - jm_single().bc_interface.wallet_synced = True - assert wallet.get_new_addr(1, 0) == "2Mz817RE6zqywgkG2h9cATUoiXwnFSxufk2" - assert wallet.get_external_addr(3) == "2N3gn65WXEzbLnjk5FLDZPc1pL6ebvZAmoA" - addr3internal = wallet.get_internal_addr(3) - assert addr3internal == "2N5NMTYogAyrGhDtWBnVQUp1kgwwFzcf7UM" - assert wallet.get_key_from_addr( - addr3internal) == "089a7173314d29f99e02a37e36da517ce41537a317c83284db1f33dda0af0cc201" - dummyaddr = "mvw1NazKDRbeNufFANqpYNAANafsMC2zVU" - assert not wallet.get_key_from_addr(dummyaddr) - #Make a new Wallet(), and prepare a testnet wallet file for this wallet - - password = "dummypassword" - password_key = bitcoin.bin_dbl_sha256(password) - seed = bitcoin.sha256("\xaa" * 64)[:32] - encrypted_seed = encryptData(password_key, seed.decode('hex')) - timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") - net = get_network() if not wrongnet else 'mainnnet' - walletfilejson = {'creator': 'joinmarket project', - 'creation_time': timestamp, - 'encrypted_seed': encrypted_seed.encode('hex'), - 'network': net} - if includecache: - mmd = wallet.max_mix_depth if not extendmd else wallet.max_mix_depth + 5 - print("using mmd: " + str(mmd)) - walletfilejson.update({'index_cache': [[0, 0]] * mmd}) - walletfile = json.dumps(walletfilejson) - if not os.path.exists(walletdir): - os.makedirs(walletdir) - with open(pathtowallet, "wb") as f: - f.write(walletfile) - if wrongnet: - with pytest.raises(ValueError) as e_info: - SegwitWallet(testwalletname, - password, - 5, - 6, - extend_mixdepth=extendmd, - storepassword=storepwd) - return - from string import ascii_letters - for i in range( - pwdnumtries): #multiple tries to ensure pkcs7 error is triggered - with pytest.raises(WalletError) as e_info: - wrongpwd = "".join([random.choice(ascii_letters) for _ in range(20) - ]) - SegwitWallet(testwalletname, - wrongpwd, - 5, - 6, - extend_mixdepth=extendmd, - storepassword=storepwd) - - with pytest.raises(WalletError) as e_info: - SegwitWallet(testwalletname, - None, - 5, - 6, - extend_mixdepth=extendmd, - storepassword=storepwd) - newwallet = SegwitWallet(testwalletname, - password, - 5, - 6, - extend_mixdepth=extendmd, - storepassword=storepwd) - assert newwallet.seed == wallet.wallet_data_to_seed(seed) - #now we have a functional wallet + file, update the cache; first try - #with failed paths - oldpath = newwallet.path - newwallet.path = None - newwallet.update_cache_index() - newwallet.path = "fake-path-definitely-doesnt-exist" - newwallet.update_cache_index() - #with real path - newwallet.path = oldpath - newwallet.index = [[1, 1]] * 5 - newwallet.update_cache_index() - - #ensure we cannot find a mainnet wallet from seed - seed = "goodbye" - jm_single().config.set("BLOCKCHAIN", "network", "mainnet") - with pytest.raises(IOError) as e_info: - Wallet(seed, 5, 6, False, False) - load_program_config() - - -def test_imported_privkey(setup_wallets): - for n in ["mainnet", "testnet"]: - privkey = "7d998b45c219a1e38e99e7cbd312ef67f77a455a9b50c730c27f02c6f730dfb401" - jm_single().config.set("BLOCKCHAIN", "network", n) - password = "dummypassword" - password_key = bitcoin.bin_dbl_sha256(password) - wifprivkey = bitcoin.wif_compressed_privkey(privkey, get_p2pk_vbyte()) - #mainnet is "L1RrrnXkcKut5DEMwtDthjwRcTTwED36thyL1DebVrKuwvohjMNi" - #to verify use from_wif_privkey and privkey_to_address - if n == "mainnet": - iaddr = "1LDsjB43N2NAQ1Vbc2xyHca4iBBciN8iwC" - else: - iaddr = "mzjq2E92B3oRB7yDKbwM7XnPaAnKfRERw2" - privkey_bin = bitcoin.from_wif_privkey( - wifprivkey, - vbyte=get_p2pk_vbyte()).decode('hex')[:-1] - encrypted_privkey = encryptData(password_key, privkey_bin) - encrypted_privkey_bad = encryptData(password_key, privkey_bin[:6]) - walletdir = "wallets" - testwalletname = "test" + n - pathtowallet = os.path.join(walletdir, testwalletname) - seed = bitcoin.sha256("\xaa" * 64)[:32] - encrypted_seed = encryptData(password_key, seed.decode('hex')) - timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") - for ep in [encrypted_privkey, encrypted_privkey_bad]: - walletfilejson = {'creator': 'joinmarket project', - 'creation_time': timestamp, - 'encrypted_seed': encrypted_seed.encode('hex'), - 'network': n, - 'index_cache': [[0, 0]] * 5, - 'imported_keys': [ - {'encrypted_privkey': ep.encode('hex'), - 'mixdepth': 0} - ]} - walletfile = json.dumps(walletfilejson) - if not os.path.exists(walletdir): - os.makedirs(walletdir) - with open(pathtowallet, "wb") as f: - f.write(walletfile) - if ep == encrypted_privkey_bad: - with pytest.raises(Exception) as e_info: - Wallet(testwalletname, password, 5, 6, False, False) - continue - newwallet = Wallet(testwalletname, password, 5, 6, False, False) - assert newwallet.seed == seed - #test accessing the key from the addr - assert newwallet.get_key_from_addr( - iaddr) == bitcoin.from_wif_privkey(wifprivkey, - vbyte=get_p2pk_vbyte()) - if n == "testnet": - jm_single().bc_interface.sync_wallet(newwallet) - load_program_config() - -def test_add_remove_utxos(setup_wallets): - #Make a fake wallet and inject and then remove fake utxos - walletdir, pathtowallet, testwalletname, wallet = create_default_testnet_wallet() - assert wallet.get_addr(2, 0, 5) == "2NBUxbEQrGPKrYCV6d4o7Y4AtJ34Uy6gZZg" - wallet.addr_cache["2NBUxbEQrGPKrYCV6d4o7Y4AtJ34Uy6gZZg"] = (2, 0, 5) - #'a914c80b3c03b96c0da5ef983942d9e541cb788aed8787' - #these calls automatically update the addr_cache: - assert wallet.get_new_addr(1, 0) == "2Mz817RE6zqywgkG2h9cATUoiXwnFSxufk2" - #a9144b6b3836a1708fd38d4728e41b86e69d5bb15d5187 - assert wallet.get_external_addr(3) == "2N3gn65WXEzbLnjk5FLDZPc1pL6ebvZAmoA" - #a914728673d95ceafa892ed82f9cc23c8bf1700b6c6187 - #using the above pubkey scripts: - faketxforwallet = {'outs': [ - {'script': 'a914c80b3c03b96c0da5ef983942d9e541cb788aed8787', - 'value': 110000000}, - {'script': 'a9144b6b3836a1708fd38d4728e41b86e69d5bb15d5187', - 'value': 89910900}, - {'script': 'a914728673d95ceafa892ed82f9cc23c8bf1700b6c6187', - 'value': 90021000}, - {'script': - '76a9145ece2dac945c8ff5b2b6635360ca0478ade305d488ac', #not ours - 'value': 110000000} - ], - 'version': 1} - wallet.add_new_utxos(faketxforwallet, "aa" * 32) - faketxforspending = {'ins': [ - {'outpoint': {'hash': 'aa' * 32, - 'index': 0}}, {'outpoint': {'hash': 'aa' * 32, - 'index': 1}}, {'outpoint': - {'hash': - 'aa' * 32, - 'index': 2}}, - {'outpoint': - {'hash': - '3f3ea820d706e08ad8dc1d2c392c98facb1b067ae4c671043ae9461057bd2a3c', - 'index': 1}, - 'script': '', - 'sequence': 4294967295} - ]} - wallet.select_utxos(1, 100000) - with pytest.raises(Exception) as e_info: - wallet.select_utxos(0, 100000) - #ensure get_utxos_by_mixdepth can handle utxos outside of maxmixdepth - wallet.max_mix_depth = 2 - mul = wallet.get_utxos_by_mixdepth() - assert mul[3] != {} - wallet.remove_old_utxos(faketxforspending) - @pytest.fixture(scope="module") def setup_wallets(): diff --git a/test/common.py b/test/common.py index cda088a..d503023 100644 --- a/test/common.py +++ b/test/common.py @@ -4,7 +4,6 @@ from __future__ import absolute_import import sys import os -import time import binascii import random from decimal import Decimal @@ -12,7 +11,8 @@ from decimal import Decimal data_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.insert(0, os.path.join(data_dir)) -from jmclient import get_wallet_cls, get_log, estimate_tx_fee, jm_single +from jmclient import open_test_wallet_maybe, BIP32Wallet, SegwitLegacyWallet, \ + get_log, estimate_tx_fee, jm_single import jmbitcoin as btc from jmbase import chunks @@ -63,7 +63,8 @@ def make_wallets(n, fixed_seeds=None, test_wallet=False, passwords=None, - walletclass=None): + walletclass=SegwitLegacyWallet, + mixdepths=5): '''n: number of wallets to be created wallet_structure: array of n arrays , each subarray specifying the number of addresses to be populated with coins @@ -74,35 +75,36 @@ def make_wallets(n, Default Wallet constructor is joinmarket.Wallet, else use TestWallet, which takes a password parameter as in the list passwords. ''' + # FIXME: this is basically the same code as jmclient/test/commontest.py if len(wallet_structures) != n: raise Exception("Number of wallets doesn't match wallet structures") if not fixed_seeds: - seeds = chunks(binascii.hexlify(os.urandom(15 * n)), 15 * 2) + seeds = chunks(binascii.hexlify(os.urandom(BIP32Wallet.ENTROPY_BYTES * n)), + BIP32Wallet.ENTROPY_BYTES * 2) else: seeds = fixed_seeds wallets = {} for i in range(n): - if test_wallet: - w = TestWallet(seeds[i], max_mix_depth=5, pwd=passwords[i]) + assert len(seeds[i]) == BIP32Wallet.ENTROPY_BYTES * 2 + + # FIXME: pwd is ignored (but do we really need this anyway?) + if test_wallet and passwords and i < len(passwords): + pwd = passwords[i] else: - if walletclass: - wc = walletclass - else: - wc = get_wallet_cls() - w = wc(seeds[i], pwd=None, max_mix_depth=5) + pwd = None + + w = open_test_wallet_maybe(seeds[i], seeds[i], mixdepths, + test_wallet_cls=walletclass) + wallets[i + start_index] = {'seed': seeds[i], 'wallet': w} - for j in range(5): + for j in range(mixdepths): for k in range(wallet_structures[i][j]): deviation = sdev_amt * random.random() amt = mean_amt - sdev_amt / 2.0 + deviation if amt < 0: amt = 0.001 amt = float(Decimal(amt).quantize(Decimal(10)**-8)) - jm_single().bc_interface.grab_coins( - wallets[i + start_index]['wallet'].get_external_addr(j), - amt) - #reset the index so the coins can be seen if running in same script - wallets[i + start_index]['wallet'].index[j][0] -= wallet_structures[i][j] + jm_single().bc_interface.grab_coins(w.get_external_addr(j), amt) return wallets diff --git a/test/test_segwit.py b/test/test_segwit.py index 8a9d0ca..cfe8b1f 100644 --- a/test/test_segwit.py +++ b/test/test_segwit.py @@ -2,17 +2,13 @@ from __future__ import absolute_import '''Test creation of segwit transactions.''' -import sys -import os -import time import binascii import json from common import make_wallets from pprint import pformat import jmbitcoin as btc import pytest -from jmclient import (load_program_config, jm_single, get_p2pk_vbyte, - get_log, get_p2sh_vbyte, Wallet) +from jmclient import load_program_config, jm_single, get_log, LegacyWallet log = get_log() @@ -30,80 +26,11 @@ def test_segwit_valid_txs(setup_segwit): #and compare the json values -def get_utxo_from_txid(txid, addr): - """Given a txid and an address for one of the outputs, - return "txid:n" where n is the index of the output - """ - rawtx = jm_single().bc_interface.rpc("getrawtransaction", [txid, 1]) - ins = [] - for u in rawtx["vout"]: - if u["scriptPubKey"]["addresses"][0] == addr: - ins.append(txid + ":" + str(u["n"])) - assert len(ins) == 1 - return ins[0] - - -def make_sign_and_push(ins_sw, - wallet, - amount, - other_ins=None, - output_addr=None, - change_addr=None, - hashcode=btc.SIGHASH_ALL): - """A more complicated version of the function in test_tx_creation; - will merge to this one once finished. - ins_sw have this structure: - {"txid:n":(amount, priv, index), "txid2:n2":(amount2, priv2, index2), ..} - if other_ins is not None, it has the same format, - these inputs are assumed to be plain p2pkh. - All of these inputs in these two sets will be consumed. - They are ordered according to the "index" fields (to allow testing - of specific ordering) - It's assumed that they contain sufficient coins to satisy the - required output specified in "amount", plus some extra for fees and a - change output. - The output_addr and change_addr, if None, are taken from the wallet - and are ordinary p2pkh outputs. - All amounts are in satoshis and only converted to btc for grab_coins - """ - #total value of all inputs - print ins_sw - print other_ins - total = sum([x[0] for x in ins_sw.values()]) - total += sum([x[0] for x in other_ins.values()]) - #construct the other inputs - ins1 = other_ins - ins1.update(ins_sw) - ins1 = sorted(ins1.keys(), key=lambda k: ins1[k][2]) - #random output address and change addr - output_addr = wallet.get_new_addr(1, 1) if not output_addr else output_addr - change_addr = wallet.get_new_addr(1, 0) if not change_addr else change_addr - outs = [{'value': amount, - 'address': output_addr}, {'value': total - amount - 10000, - 'address': change_addr}] - tx = btc.mktx(ins1, outs) - de_tx = btc.deserialize(tx) - for index, ins in enumerate(de_tx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - temp_ins = ins_sw if utxo in ins_sw.keys() else other_ins - amt, priv, n = temp_ins[utxo] - temp_amt = amt if utxo in ins_sw.keys() else None - #for better test code coverage - print "signing tx index: " + str(index) + ", priv: " + priv - if index % 2: - priv = binascii.unhexlify(priv) - ms = "other" if not temp_amt else "amount: " + str(temp_amt) - print ms - tx = btc.sign(tx, index, priv, hashcode=hashcode, amount=temp_amt) - print pformat(btc.deserialize(tx)) - txid = jm_single().bc_interface.pushtx(tx) - time.sleep(3) - received = jm_single().bc_interface.get_received_by_addr( - [output_addr], None)['data'][0]['balance'] - #check coins were transferred as expected - assert received == amount - #pushtx returns False on any error - return txid +def binarize_tx(tx): + for o in tx['outs']: + o['script'] = binascii.unhexlify(o['script']) + for i in tx['ins']: + i['outpoint']['hash'] = binascii.unhexlify(i['outpoint']['hash']) @pytest.mark.parametrize( @@ -111,7 +38,7 @@ def make_sign_and_push(ins_sw, ([[1, 0, 0, 0, 0]], 1, 1000000, 1, [0, 1, 2], []), ([[4, 0, 0, 0, 1]], 3, 100000000, 1, [0, 2], [1, 3]), ([[4, 0, 0, 0, 1]], 3, 100000000, 1, [0, 5], [1, 2, 3, 4]), - ([[2, 0, 0, 0, 2]], 2, 200000007, 0.3, [0, 1, 4, 5], [2, 3, 6]), + ([[4, 0, 0, 0, 0]], 2, 200000007, 0.3, [0, 1, 4, 5], [2, 3, 6]), ]) def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, segwit_amt, segwit_ins, o_ins): @@ -127,38 +54,94 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, segwit_ins is a list of input indices (where to place the funding segwit utxos) other_ins is a list of input indices (where to place the funding non-sw utxos) """ - wallet = make_wallets(1, wallet_structure, in_amt, walletclass=Wallet)[0]['wallet'] - jm_single().bc_interface.sync_wallet(wallet, fast=True) - other_ins = {} - ctr = 0 - for k, v in wallet.unspent.iteritems(): - #only extract as many non-segwit utxos as we need; - #doesn't matter which they are - if ctr == len(o_ins): - break - other_ins[k] = (v["value"], wallet.get_key_from_addr(v["address"]), - o_ins[ctr]) - ctr += 1 - ins_sw = {} - for i in range(len(segwit_ins)): - #build segwit ins from "deterministic-random" keys; - #intended to be the same for each run with the same parameters - seed = json.dumps([i, wallet_structure, in_amt, amount, segwit_ins, - other_ins]) - priv = btc.sha256(seed) + "01" - pub = btc.privtopub(priv) - #magicbyte is testnet p2sh - addr1 = btc.pubkey_to_p2sh_p2wpkh_address(pub, magicbyte=196) - print "got address for p2shp2wpkh: " + addr1 - txid = jm_single().bc_interface.grab_coins(addr1, segwit_amt) - #TODO - int cast, fix? - ins_sw[get_utxo_from_txid(txid, addr1)] = (int(segwit_amt * 100000000), - priv, segwit_ins[i]) - #make_sign_and_push will sanity check the received amount is correct - txid = make_sign_and_push(ins_sw, wallet, amount, other_ins) - #will always be False if it didn't push. + MIXDEPTH = 0 + + # set up wallets and inputs + nsw_wallet = make_wallets(1, wallet_structure, in_amt, + walletclass=LegacyWallet)[0]['wallet'] + jm_single().bc_interface.sync_wallet(nsw_wallet, fast=True) + sw_wallet = make_wallets(1, [[len(segwit_ins), 0, 0, 0, 0]], segwit_amt)[0]['wallet'] + jm_single().bc_interface.sync_wallet(sw_wallet, fast=True) + + nsw_utxos = nsw_wallet.get_utxos_by_mixdepth_()[MIXDEPTH] + sw_utxos = sw_wallet.get_utxos_by_mixdepth_()[MIXDEPTH] + assert len(o_ins) <= len(nsw_utxos), "sync failed" + assert len(segwit_ins) <= len(sw_utxos), "sync failed" + + total_amt_in_sat = 0 + + nsw_ins = {} + for nsw_in_index in o_ins: + total_amt_in_sat += in_amt * 10**8 + nsw_ins[nsw_in_index] = nsw_utxos.popitem() + + sw_ins = {} + for sw_in_index in segwit_ins: + total_amt_in_sat += int(segwit_amt * 10**8) + sw_ins[sw_in_index] = sw_utxos.popitem() + + all_ins = {} + all_ins.update(nsw_ins) + all_ins.update(sw_ins) + + # sanity checks + assert len(all_ins) == len(nsw_ins) + len(sw_ins), \ + "test broken, duplicate index" + for k in all_ins: + assert 0 <= k < len(all_ins), "test broken, missing input index" + + # FIXME: encoding mess, mktx should accept binary input formats + tx_ins = [] + for i, (txid, data) in sorted(all_ins.items(), key=lambda x: x[0]): + tx_ins.append('{}:{}'.format(binascii.hexlify(txid[0]), txid[1])) + + # create outputs + FEE = 50000 + assert FEE < total_amt_in_sat - amount, "test broken, not enough funds" + + cj_script = nsw_wallet.get_new_script(MIXDEPTH + 1, True) + change_script = nsw_wallet.get_new_script(MIXDEPTH, True) + change_amt = total_amt_in_sat - amount - FEE + + tx_outs = [ + {'script': binascii.hexlify(cj_script), + 'value': amount}, + {'script': binascii.hexlify(change_script), + 'value': change_amt}] + tx = btc.deserialize(btc.mktx(tx_ins, tx_outs)) + binarize_tx(tx) + + # import new addresses to bitcoind + jm_single().bc_interface.import_addresses( + [nsw_wallet.script_to_address(x) + for x in [cj_script, change_script]], + jm_single().bc_interface.get_wallet_name(nsw_wallet)) + + # sign tx + scripts = {} + for nsw_in_index in o_ins: + inp = nsw_ins[nsw_in_index][1] + scripts[nsw_in_index] = (inp['script'], inp['value']) + nsw_wallet.sign_tx(tx, scripts) + + scripts = {} + for sw_in_index in segwit_ins: + inp = sw_ins[sw_in_index][1] + scripts[sw_in_index] = (inp['script'], inp['value']) + sw_wallet.sign_tx(tx, scripts) + + print(tx) + + # push and verify + txid = jm_single().bc_interface.pushtx(binascii.hexlify(btc.serialize(tx))) assert txid + balances = jm_single().bc_interface.get_received_by_addr( + [nsw_wallet.script_to_address(cj_script), + nsw_wallet.script_to_address(change_script)], None)['data'] + assert balances[0]['balance'] == amount + assert balances[1]['balance'] == change_amt + @pytest.fixture(scope="module") def setup_segwit(): From 705d41d299518ce65b5b839e1d1e9021f32733e9 Mon Sep 17 00:00:00 2001 From: undeath Date: Fri, 29 Dec 2017 13:24:44 +0100 Subject: [PATCH 08/23] remove usages of wallet.unspent --- jmclient/jmclient/blockchaininterface.py | 28 +++++++++---- jmclient/jmclient/electruminterface.py | 41 ++++++++++-------- scripts/add-utxo.py | 19 ++++----- scripts/jmtainter.py | 53 ++++++++++++++---------- test/ygrunner.py | 2 +- 5 files changed, 82 insertions(+), 61 deletions(-) diff --git a/jmclient/jmclient/blockchaininterface.py b/jmclient/jmclient/blockchaininterface.py index 67b8648..447255c 100644 --- a/jmclient/jmclient/blockchaininterface.py +++ b/jmclient/jmclient/blockchaininterface.py @@ -51,7 +51,7 @@ class BlockchainInterface(object): @staticmethod def get_wallet_name(wallet): - return 'joinmarket-wallet-' + btc.dbl_sha256(wallet.keys[0][0])[:6] + return 'joinmarket-wallet-' + wallet.get_wallet_id() @abc.abstractmethod def sync_addresses(self, wallet): @@ -60,8 +60,7 @@ class BlockchainInterface(object): @abc.abstractmethod def sync_unspent(self, wallet): - """Finds the unspent transaction outputs belonging to this wallet, - sets wallet.unspent """ + """Finds the unspent transaction outputs belonging to this wallet""" def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr, wallet_name=None, timeoutfun=None, spentfun=None, txid_flag=True, @@ -642,7 +641,7 @@ class BitcoinCoreInterface(BlockchainInterface): def sync_unspent(self, wallet): st = time.time() wallet_name = self.get_wallet_name(wallet) - wallet.unspent = {} + wallet.reset_utxos() listunspent_args = [] if 'listunspent_args' in jm_single().config.options('POLICY'): @@ -655,16 +654,29 @@ class BitcoinCoreInterface(BlockchainInterface): continue if u['account'] != wallet_name: continue + # TODO if u['address'] not in wallet.addr_cache: continue - wallet.unspent[u['txid'] + ':' + str(u['vout'])] = { - 'address': u['address'], - 'value': int(Decimal(str(u['amount'])) * Decimal('1e8')) - } + self._add_unspent_utxo(wallet, u) et = time.time() log.debug('bitcoind sync_unspent took ' + str((et - st)) + 'sec') self.wallet_synced = True + @staticmethod + def _add_unspent_utxo(wallet, utxo): + """ + Add a UTXO as returned by rpc's listunspent call to the wallet. + + params: + wallet: wallet + utxo: single utxo dict as returned by listunspent + """ + txid = binascii.unhexlify(utxo['txid']) + script = binascii.unhexlify(utxo['scriptPubKey']) + value = int(Decimal(str(utxo['amount'])) * Decimal('1e8')) + + wallet.add_utxo(txid, int(utxo['vout']), script, value) + def get_deser_from_gettransaction(self, rpcretval): """Get full transaction deserialization from a call to `gettransaction` diff --git a/jmclient/jmclient/electruminterface.py b/jmclient/jmclient/electruminterface.py index d06c017..cf3cd5b 100644 --- a/jmclient/jmclient/electruminterface.py +++ b/jmclient/jmclient/electruminterface.py @@ -6,11 +6,9 @@ import pprint import random import socket import threading -import time -import sys import ssl -from twisted.python.log import startLogging -from twisted.internet.protocol import ClientFactory, Protocol +import binascii +from twisted.internet.protocol import ClientFactory from twisted.internet.ssl import ClientContextFactory from twisted.protocols.basic import LineReceiver from twisted.internet import reactor, task, defer @@ -328,10 +326,10 @@ class ElectrumInterface(BlockchainInterface): def sync_unspent(self, wallet): # finds utxos in the wallet - wallet.unspent = {} + wallet.reset_utxos() #Prepare list of all used addresses - addrs = [] - for m in range(wallet.max_mix_depth): + addrs = set() + for m in range(wallet.max_mixdepth): for fc in [0, 1]: branch_list = [] for k, v in self.temp_addr_history[m][fc].iteritems(): @@ -339,7 +337,7 @@ class ElectrumInterface(BlockchainInterface): continue if v["used"]: branch_list.append(v["addr"]) - addrs.extend(branch_list) + addrs.update(branch_list) if len(addrs) == 0: log.debug('no tx used') self.wallet_synced = True @@ -348,21 +346,28 @@ class ElectrumInterface(BlockchainInterface): return #make sure to add any addresses during the run (a subset of those #added to the address cache) - addrs = list(set(self.wallet.addr_cache.keys()).union(set(addrs))) - self.listunspent_calls = 0 + for md in range(wallet.max_mixdepth): + for internal in (True, False): + for index in range(wallet.get_next_unused_index(md, internal)): + addrs.add(wallet.get_addr(md, internal, index)) + for path in wallet.get_imported_paths(md): + addrs.add(wallet.get_addr_path(path)) + + self.listunspent_calls = len(addrs) for a in addrs: + # FIXME: update to protocol version 1.1 and use scripthash instead + script = wallet.address_to_script(a) d = self.get_from_electrum('blockchain.address.listunspent', a) - d.addCallback(self.process_listunspent_data, wallet, a, len(addrs)) + d.addCallback(self.process_listunspent_data, wallet, script) - def process_listunspent_data(self, unspent_info, wallet, address, n): - self.listunspent_calls += 1 + def process_listunspent_data(self, unspent_info, wallet, script): res = unspent_info['result'] for u in res: - wallet.unspent[str(u['tx_hash']) + ':' + str( - u['tx_pos'])] = {'address': address, 'value': int(u['value'])} - if self.listunspent_calls == n: - for u in wallet.spent_utxos: - wallet.unspent.pop(u, None) + txid = binascii.unhexlify(u['tx_hash']) + wallet.add_utxo(txid, int(u['tx_pos']), script, int(u['value'])) + + self.listunspent_calls -= 1 + if self.listunspent_calls == 0: self.wallet_synced = True if self.synctype == "sync-only": reactor.stop() diff --git a/scripts/add-utxo.py b/scripts/add-utxo.py index 1c9d6a7..1a1c29a 100644 --- a/scripts/add-utxo.py +++ b/scripts/add-utxo.py @@ -9,6 +9,7 @@ the anti-snooping feature employed by makers. import sys import os import json +import binascii from pprint import pformat from optparse import OptionParser @@ -173,20 +174,16 @@ def main(): #Three options (-w, -r, -R) for loading utxo and privkey pairs from a wallet, #csv file or json file. if options.loadwallet: - # TODO: new wallet has no unspent attribute - raise NotImplementedError("This is not yet implemented.") wallet_path = get_wallet_path(options.loadwallet, None) wallet = open_wallet(wallet_path, gap_limit=options.gaplimit) sync_wallet(wallet, fast=options.fastsync) - unsp = {} - for u, av in wallet.unspent.iteritems(): - addr = av['address'] - key = wallet.get_key_from_addr(addr) - wifkey = btc.wif_compressed_privkey(key, vbyte=get_p2pk_vbyte()) - unsp[u] = {'address': av['address'], - 'value': av['value'], 'privkey': wifkey} - for u, pva in unsp.iteritems(): - utxo_data.append((u, pva['privkey'])) + + for md, utxos in wallet.get_utxos_by_mixdepth_().items(): + for (txid, index), utxo in utxos.items(): + txhex = binascii.hexlify(txid) + ':' + str(index) + wif = wallet.get_wif_path(utxo['path']) + utxo_data.append((txhex, wif)) + elif options.in_file: with open(options.in_file, "rb") as f: utxo_info = f.readlines() diff --git a/scripts/jmtainter.py b/scripts/jmtainter.py index a168a32..24f216c 100644 --- a/scripts/jmtainter.py +++ b/scripts/jmtainter.py @@ -103,18 +103,24 @@ def serialize_derivation(roc, i): return x #======================================================= -def get_privkey_amount_from_utxo(wallet, utxo): + +def get_script_amount_from_utxo(wallet, utxo): """Given a JM wallet and a utxo string, find the corresponding private key and amount controlled in satoshis. """ - for k, v in wallet.unspent.iteritems(): - if k == utxo: - print("Found utxo, its value is: ", v['value']) - return wallet.get_key_from_addr(v['address']), v['value'] - return (None, None) + for md, utxos in wallet.get_utxos_by_mixdepth_().items(): + for (txid, index), utxo in utxos.items(): + txhex = binascii.hexlify(txid) + ':' + str(index) + if txhex != utxo: + continue + script = wallet.get_script_path(utxo['path']) + print("Found utxo, its value is: {}".format(utxo['value'])) + return script, utxo['value'] + return None, None + -def create_single_acp_pair(utxo_in, priv, addr_out, amount, bump, segwit=False): +def create_single_acp_pair(wallet, utxo_in, script, addr_out, amount, bump, segwit=False): """Given a utxo and a signing key for it, and its amout in satoshis, sign a "transaction" consisting of only 1 input and one output, signed with single|acp sighash flags so it can be grafted into a bigger @@ -129,10 +135,9 @@ def create_single_acp_pair(utxo_in, priv, addr_out, amount, bump, segwit=False): assert bump >= 0, "Output of single|acp pair must be bigger than input for safety." out = {"address": addr_out, "value": amount + bump} tx = btc.mktx([utxo_in], [out]) - amt = amount if segwit else None - return btc.sign(tx, 0, priv, - hashcode=btc.SIGHASH_SINGLE|btc.SIGHASH_ANYONECANPAY, - amount=amt) + return wallet.sign_tx(tx, {0: (script, amount)}, + hashcode=btc.SIGHASH_SINGLE|btc.SIGHASH_ANYONECANPAY) + def graft_onto_single_acp(wallet, txhex, amount, destaddr): """Given a serialized txhex which is checked to be of @@ -187,13 +192,15 @@ def graft_onto_single_acp(wallet, txhex, amount, destaddr): df['ins'][0]['script'] = d['ins'][0]['script'] if 'txinwitness' in d['ins'][0]: df['ins'][0]['txinwitness'] = d['ins'][0]['txinwitness'] - fulltx = btc.serialize(df) + for i, iu in enumerate(input_utxos): - priv, inamt = get_privkey_amount_from_utxo(wallet, iu) - print("Signing index: ", i+1, " with privkey: ", priv, " and amount: ", inamt, " for utxo: ", iu) - fulltx = btc.sign(fulltx, i+1, priv, amount=inamt) - return (True, fulltx) - + script, inamt = get_script_amount_from_utxo(wallet, iu) + print("Signing index: ", i+1, " with script: ", script, " and amount: ", inamt, " for utxo: ", iu) + fulltx = wallet.sign_tx(df, {i: (script, inamt)}) + + return True, btc.serialize(fulltx) + + if __name__ == "__main__": parser = get_parser() (options, args) = parser.parse_args() @@ -214,22 +221,22 @@ if __name__ == "__main__": "Use wallet-tool.py method 'showutxos' to select one") exit(0) utxo_in = args[2] - priv, amount = get_privkey_amount_from_utxo(wallet, utxo_in) - if not priv: + script, amount = get_script_amount_from_utxo(wallet, utxo_in) + if not script: print("Failed to find the utxo's private key from the wallet; check " "if this utxo is actually contained in the wallet using " "wallet-tool.py showutxos") exit(0) #destination sourced from wallet addr_out = wallet.get_new_addr((options.mixdepth+1)%options.amtmixdepths, 1) - serialized_single_acp = create_single_acp_pair(utxo_in, priv, addr_out, amount, - options.bump, segwit=True) + single_acp = create_single_acp_pair(wallet, utxo_in, script, addr_out, amount, + options.bump, segwit=True) print("Created the following one-in, one-out transaction, which will not " "be valid to broadcast itself (negative fee). Pass it to your " "counterparty:") - print(pformat(btc.deserialize(serialized_single_acp))) + print(pformat(single_acp)) print("Pass the following raw hex to your counterparty:") - print(serialized_single_acp) + print(btc.serialize(single_acp)) exit(0) elif args[1] == "take": try: diff --git a/test/ygrunner.py b/test/ygrunner.py index d3ed0d6..80957d2 100644 --- a/test/ygrunner.py +++ b/test/ygrunner.py @@ -110,7 +110,7 @@ def test_start_ygs(setup_ygrunner, num_ygs, wallet_structures, mean_amt, print("Seed : " + wallets[num_ygs]['seed']) #useful to see the utxos on screen sometimes sync_wallet(wallet, fast=True) - print(wallet.unspent) + print(wallet.get_utxos_by_mixdepth()) txfee = 1000 cjfee_a = 4200 cjfee_r = '0.001' From 1f309678ca82aeb2bbc8a24a356a3574f4d077f3 Mon Sep 17 00:00:00 2001 From: undeath Date: Fri, 6 Jul 2018 14:57:39 +0200 Subject: [PATCH 09/23] adopt blockchaininterface for new wallet --- jmclient/jmclient/blockchaininterface.py | 328 ++++++++++------------- jmclient/jmclient/electruminterface.py | 2 +- test/test_segwit.py | 6 +- 3 files changed, 149 insertions(+), 187 deletions(-) diff --git a/jmclient/jmclient/blockchaininterface.py b/jmclient/jmclient/blockchaininterface.py index 447255c..d5d7f8e 100644 --- a/jmclient/jmclient/blockchaininterface.py +++ b/jmclient/jmclient/blockchaininterface.py @@ -17,11 +17,6 @@ from jmbase.support import get_log log = get_log() -def is_index_ahead_of_cache(wallet, mix_depth, forchange): - if mix_depth >= len(wallet.index_cache): - return True - return wallet.index[mix_depth][forchange] >= wallet.index_cache[mix_depth][ - forchange] def sync_wallet(wallet, fast=False): """Wrapper function to choose fast syncing where it's @@ -55,8 +50,7 @@ class BlockchainInterface(object): @abc.abstractmethod def sync_addresses(self, wallet): - """Finds which addresses have been used and sets - wallet.index appropriately""" + """Finds which addresses have been used""" @abc.abstractmethod def sync_unspent(self, wallet): @@ -421,41 +415,37 @@ class BitcoinCoreInterface(BlockchainInterface): wallet_name = self.get_wallet_name(wallet) agd = self.rpc('listaddressgroupings', []) #flatten all groups into a single list; then, remove duplicates - fagd = [tuple(item) for sublist in agd for item in sublist] + fagd = (tuple(item) for sublist in agd for item in sublist) #"deduplicated flattened address grouping data" = dfagd - dfagd = list(set(fagd)) - #for lookup, want dict of form {"address": amount} - used_address_dict = {} + dfagd = set(fagd) + used_addresses = set() for addr_info in dfagd: if len(addr_info) < 3 or addr_info[2] != wallet_name: continue - used_address_dict[addr_info[0]] = (addr_info[1], addr_info[2]) + used_addresses.add(addr_info[0]) + #for a first run, import first chunk - if len(used_address_dict.keys()) == 0: + if not used_addresses: log.info("Detected new wallet, performing initial import") - for i in range(wallet.max_mix_depth): - for j in [0, 1]: - addrs_to_import = [] - for k in range(wallet.gaplimit + 10): # a few more for safety - addrs_to_import.append(wallet.get_addr(i, j, k)) - self.import_addresses(addrs_to_import, wallet_name) - wallet.index[i][j] = 0 + # delegate inital address import to sync_addresses + # this should be fast because "getaddressesbyaccount" should return + # an empty list in this case + self.sync_addresses(wallet) self.wallet_synced = True return + #Wallet has been used; scan forwards. log.debug("Fast sync in progress. Got this many used addresses: " + str( - len(used_address_dict))) + len(used_addresses))) #Need to have wallet.index point to the last used address - #and fill addr_cache. #Algo: - # 1. Scan batch 1 of each branch, accumulate wallet addresses into dict. - # 2. Find matches between that dict and used addresses, add those to - # used_indices dict and add to address cache. - # 3. Check if all addresses in 'used addresses' have been matched, if + # 1. Scan batch 1 of each branch, record matched wallet addresses. + # 2. Check if all addresses in 'used addresses' have been matched, if # so, break. - # 4. Repeat the above for batch 2, 3.. up to max 20 batches. - # 5. If after all 20 batches not all used addresses were matched, + # 3. Repeat the above for batch 2, 3.. up to max 20 batches. + # 4. If after all 20 batches not all used addresses were matched, # quit with error. + # 5. Calculate used indices. # 6. If all used addresses were matched, set wallet index to highest # found index in each branch and mark wallet sync complete. #Rationale for this algo: @@ -466,170 +456,143 @@ class BitcoinCoreInterface(BlockchainInterface): # not be exposed to the user; it is not the same as gap limit, in fact, # the concept of gap limit does not apply to this kind of sync, which # *assumes* that the most recent usage of addresses is indeed recorded. - used_indices = {} - local_addr_cache = {} - found_addresses = [] + remaining_used_addresses = used_addresses.copy() + addresses, saved_indices = self._collect_addresses_init(wallet) + for addr in addresses: + remaining_used_addresses.discard(addr) + BATCH_SIZE = 100 - for j in range(20): - for md in range(wallet.max_mix_depth): - if md not in used_indices: - used_indices[md] = {} - for fc in [0, 1]: - if fc not in used_indices[md]: - used_indices[md][fc] = [] - for i in range(j*BATCH_SIZE, (j+1)*BATCH_SIZE): - local_addr_cache[(md, fc, i)] = wallet.get_addr(md, fc, i) - batch_found_addresses = [x for x in local_addr_cache.iteritems( - ) if x[1] in used_address_dict.keys()] - for x in batch_found_addresses: - md, fc, i = x[0] - addr = x[1] - used_indices[md][fc].append(i) - wallet.addr_cache[addr] = (md, fc, i) - found_addresses.extend(batch_found_addresses) - if len(found_addresses) == len(used_address_dict.keys()): + MAX_ITERATIONS = 20 + for j in range(MAX_ITERATIONS): + if not remaining_used_addresses: break - if j == 19: + for addr in \ + self._collect_addresses_gap(wallet, gap_limit=BATCH_SIZE): + remaining_used_addresses.discard(addr) + else: raise Exception("Failed to sync in fast mode after 20 batches; " "please re-try wallet sync without --fast flag.") - #Find the highest index in each branch and set the wallet index - for md in range(wallet.max_mix_depth): - for fc in [0, 1]: - if len(used_indices[md][fc]): - used_indices[md][fc].sort() - wallet.index[md][fc] = used_indices[md][fc][-1] + 1 - else: - wallet.index[md][fc] = 0 - if not is_index_ahead_of_cache(wallet, md, fc): - wallet.index[md][fc] = wallet.index_cache[md][fc] - self.wallet_synced = True + # creating used_indices on-the-fly would be more efficient, but the + # overall performance gain is probably negligible + used_indices = self._get_used_indices(wallet, used_addresses) + self._rewind_wallet_indices(wallet, used_indices, saved_indices) + self.wallet_synced = True def sync_addresses(self, wallet, restart_cb=None): - log.debug('requesting detailed wallet history') + log.debug("requesting detailed wallet history") wallet_name = self.get_wallet_name(wallet) - #TODO It is worth considering making this user configurable: - addr_req_count = 20 - wallet_addr_list = [] - for mix_depth in range(wallet.max_mix_depth): - for forchange in [0, 1]: - #If we have an index-cache available, we can use it - #to decide how much to import (note that this list - #*always* starts from index 0 on each branch). - #In cases where the Bitcoin Core instance is fresh, - #this will allow the entire import+rescan to occur - #in 2 steps only. - if wallet.index_cache != [[0, 0]] * wallet.max_mix_depth: - #Need to request N*addr_req_count where N is least s.t. - #N*addr_req_count > index_cache val. This is so that the batching - #process in the main loop *always* has already imported enough - #addresses to complete. - req_count = int(wallet.index_cache[mix_depth][forchange] / - addr_req_count) + 1 - req_count *= addr_req_count - else: - #If we have *nothing* - no index_cache, and no info - #in Core wallet (imports), we revert to a batching mode - #with a default size. - #In this scenario it could require several restarts *and* - #rescans; perhaps user should set addr_req_count high - #(see above TODO) - req_count = addr_req_count - wallet_addr_list += [wallet.get_new_addr(mix_depth, forchange) - for _ in range(req_count)] - #Indices are reset here so that the next algorithm step starts - #from the beginning of each branch - wallet.index[mix_depth][forchange] = 0 - # makes more sense to add these in an account called "joinmarket-imported" but its much - # simpler to add to the same account here - for privkey_list in wallet.imported_privkeys.values(): - for privkey in privkey_list: - imported_addr = btc.privtoaddr(privkey, - magicbyte=get_p2pk_vbyte()) - wallet_addr_list.append(imported_addr) - imported_addr_list = self.rpc('getaddressesbyaccount', [wallet_name]) - if not set(wallet_addr_list).issubset(set(imported_addr_list)): - self.add_watchonly_addresses(wallet_addr_list, wallet_name, restart_cb) - return - buf = self.rpc('listtransactions', [wallet_name, 1000, 0, True]) - txs = buf - # If the buffer's full, check for more, until it ain't - while len(buf) == 1000: - buf = self.rpc('listtransactions', [wallet_name, 1000, len(txs), - True]) - txs += buf - # TODO check whether used_addr_list can be a set, may be faster (if - # its a hashset) and allows using issubset() here and setdiff() for - # finding which addresses need importing - - # TODO also check the fastest way to build up python lists, i suspect - # using += is slow - used_addr_list = [tx['address'] - for tx in txs if tx['category'] == 'receive'] - too_few_addr_mix_change = [] - for mix_depth in range(wallet.max_mix_depth): - for forchange in [0, 1]: - unused_addr_count = 0 - last_used_addr = '' - breakloop = False - while not breakloop: - if unused_addr_count >= wallet.gaplimit and \ - is_index_ahead_of_cache(wallet, mix_depth, - forchange): - break - mix_change_addrs = [ - wallet.get_new_addr(mix_depth, forchange) - for _ in range(addr_req_count) - ] - for mc_addr in mix_change_addrs: - if mc_addr not in imported_addr_list: - too_few_addr_mix_change.append((mix_depth, forchange - )) - breakloop = True - break - if mc_addr in used_addr_list: - last_used_addr = mc_addr - unused_addr_count = 0 - else: - unused_addr_count += 1 -#index setting here depends on whether we broke out of the loop -#early; if we did, it means we need to prepare the index -#at the level of the last used address or zero so as to not -#miss any imports in add_watchonly_addresses. -#If we didn't, we need to respect the index_cache to avoid -#potential address reuse. - if breakloop: - if last_used_addr == '': - wallet.index[mix_depth][forchange] = 0 - else: - wallet.index[mix_depth][forchange] = \ - wallet.addr_cache[last_used_addr][2] + 1 - else: - if last_used_addr == '': - next_avail_idx = max([wallet.index_cache[mix_depth][ - forchange], 0]) - else: - next_avail_idx = max([wallet.addr_cache[last_used_addr][ - 2] + 1, wallet.index_cache[mix_depth][forchange]]) - wallet.index[mix_depth][forchange] = next_avail_idx - - wallet_addr_list = [] - if len(too_few_addr_mix_change) > 0: - indices = [wallet.index[mc[0]][mc[1]] - for mc in too_few_addr_mix_change] - log.debug('too few addresses in ' + str(too_few_addr_mix_change) + - ' at ' + str(indices)) - for mix_depth, forchange in too_few_addr_mix_change: - wallet_addr_list += [ - wallet.get_new_addr(mix_depth, forchange) - for _ in range(addr_req_count * 3) - ] - - self.add_watchonly_addresses(wallet_addr_list, wallet_name, restart_cb) + addresses, saved_indices = self._collect_addresses_init(wallet) + imported_addresses = set(self.rpc('getaddressesbyaccount', [wallet_name])) + + if not addresses.issubset(imported_addresses): + self.add_watchonly_addresses(addresses - imported_addresses, + wallet_name, restart_cb) return - self.wallet_synced = True + used_addresses_gen = (tx['address'] + for tx in self._yield_transactions(wallet_name) + if tx['category'] == 'receive') + + used_indices = self._get_used_indices(wallet, used_addresses_gen) + log.debug("got used indices: {}".format(used_indices)) + gap_limit_used = not self._check_gap_indices(wallet, used_indices) + self._rewind_wallet_indices(wallet, used_indices, saved_indices) + + new_addresses = self._collect_addresses_gap(wallet) + if not new_addresses.issubset(imported_addresses): + log.debug("Syncing iteration finished, additional step required") + self.add_watchonly_addresses(new_addresses - imported_addresses, + wallet_name, restart_cb) + self.wallet_synced = False + elif gap_limit_used: + log.debug("Syncing iteration finished, additional step required") + self.wallet_synced = False + else: + log.debug("Wallet successfully synced") + self._rewind_wallet_indices(wallet, used_indices, saved_indices) + self.wallet_synced = True + + @staticmethod + def _rewind_wallet_indices(wallet, used_indices, saved_indices): + for md in used_indices: + for int_type in (0, 1): + index = max(used_indices[md][int_type], + saved_indices[md][int_type]) + wallet.set_next_index(md, int_type, index, force=True) + + @staticmethod + def _get_used_indices(wallet, addr_gen): + indices = {x: [0, 0] for x in range(wallet.max_mixdepth + 1)} + + for addr in addr_gen: + if not wallet.is_known_addr(addr): + continue + md, internal, index = wallet.get_details( + wallet.addr_to_path(addr)) + if internal not in (0, 1): + assert internal == 'imported' + continue + indices[md][internal] = max(indices[md][internal], index + 1) + + return indices + + @staticmethod + def _check_gap_indices(wallet, used_indices): + for md in used_indices: + for internal in (0, 1): + if used_indices[md][internal] >\ + max(wallet.get_next_unused_index(md, internal), 0): + return False + return True + + @staticmethod + def _collect_addresses_init(wallet): + addresses = set() + saved_indices = dict() + + for md in range(wallet.max_mixdepth + 1): + saved_indices[md] = [0, 0] + for internal in (0, 1): + next_unused = wallet.get_next_unused_index(md, internal) + for index in range(next_unused): + addresses.add(wallet.get_addr(md, internal, index)) + for index in range(wallet.gap_limit): + addresses.add(wallet.get_new_addr(md, internal)) + wallet.set_next_index(md, internal, next_unused) + saved_indices[md][internal] = next_unused + for path in wallet.yield_imported_paths(md): + addresses.add(wallet.get_addr_path(path)) + + return addresses, saved_indices + + @staticmethod + def _collect_addresses_gap(wallet, gap_limit=None): + gap_limit = gap_limit or wallet.gap_limit + addresses = set() + + for md in range(wallet.max_mixdepth + 1): + for internal in (True, False): + old_next = wallet.get_next_unused_index(md, internal) + for index in range(gap_limit): + addresses.add(wallet.get_new_addr(md, internal)) + wallet.set_next_index(md, internal, old_next) + + return addresses + + def _yield_transactions(self, wallet_name): + batch_size = 1000 + iteration = 0 + while True: + new = self.rpc( + 'listtransactions', + [wallet_name, batch_size, iteration * batch_size, True]) + for tx in new: + yield tx + if len(new) < batch_size: + return + iteration += 1 def start_unspent_monitoring(self, wallet): self.unspent_monitoring_loop = task.LoopingCall(self.sync_unspent, wallet) @@ -654,8 +617,7 @@ class BitcoinCoreInterface(BlockchainInterface): continue if u['account'] != wallet_name: continue - # TODO - if u['address'] not in wallet.addr_cache: + if not wallet.is_known_addr(u['address']): continue self._add_unspent_utxo(wallet, u) et = time.time() diff --git a/jmclient/jmclient/electruminterface.py b/jmclient/jmclient/electruminterface.py index cf3cd5b..b7edd86 100644 --- a/jmclient/jmclient/electruminterface.py +++ b/jmclient/jmclient/electruminterface.py @@ -356,7 +356,7 @@ class ElectrumInterface(BlockchainInterface): self.listunspent_calls = len(addrs) for a in addrs: # FIXME: update to protocol version 1.1 and use scripthash instead - script = wallet.address_to_script(a) + script = wallet.addr_to_script(a) d = self.get_from_electrum('blockchain.address.listunspent', a) d.addCallback(self.process_listunspent_data, wallet, script) diff --git a/test/test_segwit.py b/test/test_segwit.py index cfe8b1f..b590500 100644 --- a/test/test_segwit.py +++ b/test/test_segwit.py @@ -113,7 +113,7 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, # import new addresses to bitcoind jm_single().bc_interface.import_addresses( - [nsw_wallet.script_to_address(x) + [nsw_wallet.script_to_addr(x) for x in [cj_script, change_script]], jm_single().bc_interface.get_wallet_name(nsw_wallet)) @@ -137,8 +137,8 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, assert txid balances = jm_single().bc_interface.get_received_by_addr( - [nsw_wallet.script_to_address(cj_script), - nsw_wallet.script_to_address(change_script)], None)['data'] + [nsw_wallet.script_to_addr(cj_script), + nsw_wallet.script_to_addr(change_script)], None)['data'] assert balances[0]['balance'] == amount assert balances[1]['balance'] == change_amt From cdbb345bf9516814c58344c34b72ffaa8ab40853 Mon Sep 17 00:00:00 2001 From: undeath Date: Tue, 3 Jul 2018 14:59:17 +0200 Subject: [PATCH 10/23] remove uses of internal wallet data from electruminterface. NOTE: changes untested, probably breaks electruminterface somehow --- jmclient/jmclient/electruminterface.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/jmclient/jmclient/electruminterface.py b/jmclient/jmclient/electruminterface.py index b7edd86..00517d2 100644 --- a/jmclient/jmclient/electruminterface.py +++ b/jmclient/jmclient/electruminterface.py @@ -12,7 +12,7 @@ from twisted.internet.protocol import ClientFactory from twisted.internet.ssl import ClientContextFactory from twisted.protocols.basic import LineReceiver from twisted.internet import reactor, task, defer -from .blockchaininterface import BlockchainInterface, is_index_ahead_of_cache +from .blockchaininterface import BlockchainInterface from .configure import get_p2sh_vbyte from .support import get_log from .electrum_data import (get_default_ports, get_default_servers, @@ -248,10 +248,10 @@ class ElectrumInterface(BlockchainInterface): reactor.callLater(0.2, self.sync_addresses, wallet, restart_cb) return log.debug("downloading wallet history from Electrum server ...") - for mixdepth in range(wallet.max_mix_depth): + for mixdepth in range(wallet.max_mixdepth + 1): for forchange in [0, 1]: #start from a clean index - wallet.index[mixdepth][forchange] = 0 + wallet.set_next_index(mixdepth, forchange, 0) self.synchronize_batch(wallet, mixdepth, forchange, 0) def synchronize_batch(self, wallet, mixdepth, forchange, start_index): @@ -295,9 +295,8 @@ class ElectrumInterface(BlockchainInterface): #existing index_cache from the wallet file; if both true, end, else, continue #to next batch if all([tah[i]['used'] is False for i in range( - start_index+self.BATCH_SIZE-wallet.gaplimit, - start_index+self.BATCH_SIZE)]) and is_index_ahead_of_cache( - wallet, mixdepth, forchange): + start_index + self.BATCH_SIZE - wallet.gap_limit, + start_index + self.BATCH_SIZE)]): last_used_addr = None #to find last used, note that it may be in the *previous* batch; #may as well just search from the start, since it takes no time. @@ -305,12 +304,11 @@ class ElectrumInterface(BlockchainInterface): if tah[i]['used']: last_used_addr = tah[i]['addr'] if last_used_addr: - wallet.index[mixdepth][forchange] = wallet.addr_cache[last_used_addr][2] + 1 + wallet.set_next_index( + mixdepth, forchange, + wallet.get_next_unused_index(mixdepth, forchange)) else: - wallet.index[mixdepth][forchange] = 0 - #account for index_cache - if not is_index_ahead_of_cache(wallet, mixdepth, forchange): - wallet.index[mixdepth][forchange] = wallet.index_cache[mixdepth][forchange] + wallet.set_next_index(mixdepth, forchange, 0) tah["finished"] = True #check if all branches are finished to trigger next stage of sync. addr_sync_complete = True @@ -350,7 +348,7 @@ class ElectrumInterface(BlockchainInterface): for internal in (True, False): for index in range(wallet.get_next_unused_index(md, internal)): addrs.add(wallet.get_addr(md, internal, index)) - for path in wallet.get_imported_paths(md): + for path in wallet.yield_imported_paths(md): addrs.add(wallet.get_addr_path(path)) self.listunspent_calls = len(addrs) From 914a40e3a1ab56257ef7d025293e1f21005a88bb Mon Sep 17 00:00:00 2001 From: undeath Date: Fri, 29 Jun 2018 14:13:01 +0200 Subject: [PATCH 11/23] adopt wallet_utils for new wallet --- jmclient/jmclient/__init__.py | 2 +- jmclient/jmclient/wallet_utils.py | 457 ++++++++++++++++-------------- 2 files changed, 241 insertions(+), 218 deletions(-) diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index 1dd44b1..8174bd9 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -46,7 +46,7 @@ from .taker_utils import (tumbler_taker_finished_update, restart_waiter, from .wallet_utils import ( wallet_tool_main, wallet_generate_recover_bip39, open_wallet, open_test_wallet_maybe, create_wallet, get_wallet_cls, get_wallet_path, - wallet_display, SewgitTestWallet) + wallet_display) from .maker import Maker from .yieldgenerator import YieldGenerator, YieldGeneratorBasic, ygmain # Set default logging handler to avoid "No handler found" warnings. diff --git a/jmclient/jmclient/wallet_utils.py b/jmclient/jmclient/wallet_utils.py index fbef685..e7bc12a 100644 --- a/jmclient/jmclient/wallet_utils.py +++ b/jmclient/jmclient/wallet_utils.py @@ -5,20 +5,16 @@ import sys import sqlite3 import binascii from datetime import datetime -from mnemonic import Mnemonic from optparse import OptionParser from jmclient import (get_network, WALLET_IMPLEMENTATIONS, Storage, podle, - encryptData, get_p2sh_vbyte, get_p2pk_vbyte, jm_single, mn_decode, - mn_encode, BitcoinCoreInterface, JsonRpcError, sync_wallet, WalletError, - BIP49Wallet, ImportWalletMixin, VolatileStorage, StoragePasswordError) + jm_single, BitcoinCoreInterface, JsonRpcError, sync_wallet, WalletError, + VolatileStorage, StoragePasswordError, + is_segwit_mode, SegwitLegacyWallet, LegacyWallet) from jmbase.support import get_password +from cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH import jmclient.btc as btc -class SewgitTestWallet(ImportWalletMixin, BIP49Wallet): - TYPE = 'p2sh-p2wpkh' - - def get_wallettool_parser(): description = ( 'Use this script to monitor and manage your Joinmarket wallet.\n' @@ -46,8 +42,9 @@ def get_wallettool_parser(): '--maxmixdepth', action='store', type='int', - dest='maxmixdepth', - help='how many mixing depths to display, default=5') + dest='mixdepths', + help='how many mixing depths to initialize in the wallet', + default=5) parser.add_option('-g', '--gap-limit', type="int", @@ -86,9 +83,30 @@ def get_wallettool_parser(): type='str', dest='hd_path', help='hd wallet path (e.g. m/0/0/0/000)') + parser.add_option('--key-type', # note: keep in sync with map_key_type + type='choice', + choices=('standard', 'segwit-p2sh'), + action='store', + dest='key_type', + default=None, + help=("Key type when importing private keys.\n" + "If your address starts with '1' use 'standard', " + "if your address starts with '3' use 'segwit-p2sh.\n" + "Native segwit addresses (starting with 'bc') are" + "not yet supported.")) return parser +def map_key_type(parser_key_choice): + if not parser_key_choice: + return parser_key_choice + if parser_key_choice == 'standard': + return TYPE_P2PKH + if parser_key_choice == 'segwit-p2sh': + return TYPE_P2SH_P2WPKH + raise Exception("Unknown key type choice '{}'.".format(parser_key_choice)) + + """The classes in this module manage representations of wallet states; but they know nothing about Bitcoin, so do not attempt to validate addresses, keys, BIP32 or relationships. @@ -126,10 +144,9 @@ WalletView* classes manage wallet representations. """ class WalletViewBase(object): - def __init__(self, bip32path, children=None, serclass=str, + def __init__(self, wallet_path_repr, children=None, serclass=str, custom_separator=None): - assert bip32pathparse(bip32path) - self.bip32path = bip32path + self.wallet_path_repr = wallet_path_repr self.children = children self.serclass = serclass self.separator = custom_separator if custom_separator else "\t" @@ -143,11 +160,10 @@ class WalletViewBase(object): return "{0:.08f}".format(self.get_balance(include_unconf)) class WalletViewEntry(WalletViewBase): - def __init__(self, bip32path, account, forchange, aindex, addr, amounts, + def __init__(self, wallet_path_repr, account, forchange, aindex, addr, amounts, used = 'new', serclass=str, priv=None, custom_separator=None): - self.bip32path = bip32path - super(WalletViewEntry, self).__init__(bip32path, serclass=serclass, - custom_separator=custom_separator) + super(WalletViewEntry, self).__init__(wallet_path_repr, serclass=serclass, + custom_separator=custom_separator) self.account = account assert forchange in [0, 1, -1] self.forchange =forchange @@ -175,10 +191,7 @@ class WalletViewEntry(WalletViewBase): return self.serclass(self.separator.join([left, addr, amounts, extradata])) def serialize_wallet_position(self): - bippath = self.bip32path + bip32sep + str(self.account) + "'" + \ - bip32sep + str(self.forchange) + bip32sep + "{0:03d}".format(self.aindex) - assert bip32pathparse(bippath) - return self.serclass(bippath) + return self.wallet_path_repr.ljust(20) def serialize_address(self): return self.serclass(self.address) @@ -198,11 +211,11 @@ class WalletViewEntry(WalletViewBase): return self.serclass(ed) class WalletViewBranch(WalletViewBase): - def __init__(self, bip32path, account, forchange, branchentries=None, + def __init__(self, wallet_path_repr, account, forchange, branchentries=None, xpub=None, serclass=str, custom_separator=None): - super(WalletViewBranch, self).__init__(bip32path, children=branchentries, - serclass=serclass, - custom_separator=custom_separator) + super(WalletViewBranch, self).__init__(wallet_path_repr, children=branchentries, + serclass=serclass, + custom_separator=custom_separator) self.account = account assert forchange in [0, 1, -1] self.forchange = forchange @@ -223,20 +236,18 @@ class WalletViewBranch(WalletViewBase): return self.serclass(entryseparator.join(lines)) def serialize_branch_header(self): - bippath = self.bip32path + bip32sep + str(self.account) + "'" + \ - bip32sep + str(self.forchange) - assert bip32pathparse(bippath) start = "external addresses" if self.forchange == 0 else "internal addresses" if self.forchange == -1: start = "Imported keys" - return self.serclass(self.separator.join([start, bippath, self.xpub])) + return self.serclass(self.separator.join([start, self.wallet_path_repr, + self.xpub])) class WalletViewAccount(WalletViewBase): - def __init__(self, bip32path, account, branches=None, account_name="mixdepth", + def __init__(self, wallet_path_repr, account, branches=None, account_name="mixdepth", serclass=str, custom_separator=None, xpub=None): - super(WalletViewAccount, self).__init__(bip32path, children=branches, - serclass=serclass, - custom_separator=custom_separator) + super(WalletViewAccount, self).__init__(wallet_path_repr, children=branches, + serclass=serclass, + custom_separator=custom_separator) self.account = account self.account_name = account_name self.xpub = xpub @@ -259,12 +270,11 @@ class WalletViewAccount(WalletViewBase): x.serialize(entryseparator) for x in self.branches] + [footer])) class WalletView(WalletViewBase): - def __init__(self, bip32path, accounts, wallet_name="JM wallet", + def __init__(self, wallet_path_repr, accounts, wallet_name="JM wallet", serclass=str, custom_separator=None): - super(WalletView, self).__init__(bip32path, children=accounts, - serclass=serclass, - custom_separator=custom_separator) - self.bip32path = bip32path + super(WalletView, self).__init__(wallet_path_repr, children=accounts, + serclass=serclass, + custom_separator=custom_separator) self.wallet_name = wallet_name assert all([isinstance(x, WalletViewAccount) for x in accounts]) self.accounts = accounts @@ -280,40 +290,41 @@ class WalletView(WalletViewBase): x.serialize(entryseparator, summarize=False) for x in self.accounts] + [footer])) def get_imported_privkey_branch(wallet, m, showprivkey): - if m in wallet.imported_privkeys: - entries = [] - for i, privkey in enumerate(wallet.imported_privkeys[m]): - pub = btc.privkey_to_pubkey(privkey) - addr = btc.pubkey_to_p2sh_p2wpkh_address(pub, magicbyte=get_p2sh_vbyte()) - balance = 0.0 - for addrvalue in wallet.unspent.values(): - if addr == addrvalue['address']: - balance += addrvalue['value'] - used = ('used' if balance > 0.0 else 'empty') - if showprivkey: - wip_privkey = btc.wif_compressed_privkey( - privkey, get_p2pk_vbyte()) - else: - wip_privkey = '' - entries.append(WalletViewEntry("m/0", m, -1, - i, addr, [balance, balance], - used=used,priv=wip_privkey)) + entries = [] + for path in wallet.yield_imported_paths(m): + addr = wallet.get_addr_path(path) + script = wallet.get_script_path(path) + balance = 0.0 + for data in wallet.get_utxos_by_mixdepth_()[m].values(): + if script == data['script']: + balance += data['value'] + used = ('used' if balance > 0.0 else 'empty') + if showprivkey: + wip_privkey = wallet.get_wif_path(path) + else: + wip_privkey = '' + entries.append(WalletViewEntry(wallet.get_path_repr(path), m, -1, + 0, addr, [balance, balance], + used=used, priv=wip_privkey)) + + if entries: return WalletViewBranch("m/0", m, -1, branchentries=entries) return None def wallet_showutxos(wallet, showprivkey): unsp = {} max_tries = jm_single().config.getint("POLICY", "taker_utxo_retries") - for u, av in wallet.unspent.iteritems(): - key = wallet.get_key_from_addr(av['address']) - tries = podle.get_podle_tries(u, key, max_tries) - tries_remaining = max(0, max_tries - tries) - unsp[u] = {'address': av['address'], 'value': av['value'], - 'tries': tries, 'tries_remaining': tries_remaining, - 'external': False} - if showprivkey: - wifkey = btc.wif_compressed_privkey(key, vbyte=get_p2pk_vbyte()) - unsp[u]['privkey'] = wifkey + utxos = wallet.get_utxos_by_mixdepth() + for md in utxos: + for u, av in utxos[md].items(): + key = wallet.get_key_from_addr(av['address']) + tries = podle.get_podle_tries(u, key, max_tries) + tries_remaining = max(0, max_tries - tries) + unsp[u] = {'address': av['address'], 'value': av['value'], + 'tries': tries, 'tries_remaining': tries_remaining, + 'external': False} + if showprivkey: + unsp[u]['privkey'] = wallet.get_wif_path(av['path']) used_commitments, external_commitments = podle.get_podle_commitments() for u, ec in external_commitments.iteritems(): @@ -331,9 +342,9 @@ def wallet_display(wallet, gaplimit, showprivkey, displayall=False, then return its serialization directly if serialized, else return the WalletView object. """ + wallet.close() acctlist = [] - rootpath = wallet.get_root_path() - for m in xrange(wallet.max_mixdepth): + for m in xrange(wallet.max_mixdepth + 1): branchlist = [] for forchange in [0, 1]: entrylist = [] @@ -343,33 +354,37 @@ def wallet_display(wallet, gaplimit, showprivkey, displayall=False, else: xpub_key = "" - for k in xrange(wallet.get_next_unused_index(m, forchange) + gaplimit): + unused_index = wallet.get_next_unused_index(m, forchange) + for k in xrange(unused_index + gaplimit): path = wallet.get_path(m, forchange, k) addr = wallet.get_addr_path(path) balance = 0 for utxodata in wallet.get_utxos_by_mixdepth_()[m].values(): if path == utxodata['path']: balance += utxodata['value'] - used = 'used' if k < wallet.get_next_unused_index(m, forchange) else 'new' + used = 'used' if k < unused_index else 'new' if showprivkey: privkey = wallet.get_wif_path(path) else: privkey = '' if (displayall or balance > 0 or - (used == 'new' and forchange == 0)): - entrylist.append(WalletViewEntry(rootpath, m, forchange, k, - addr, [balance, balance], - priv=privkey, used=used)) - branchlist.append(WalletViewBranch(rootpath, m, forchange, - entrylist, xpub=xpub_key)) + (used == 'new' and forchange == 0)): + entrylist.append(WalletViewEntry( + wallet.get_path_repr(path), m, forchange, k, addr, + [balance, balance], priv=privkey, used=used)) + path = wallet.get_path_repr(wallet.get_path(m, forchange)) + branchlist.append(WalletViewBranch(path, m, forchange, entrylist, + xpub=xpub_key)) ipb = get_imported_privkey_branch(wallet, m, showprivkey) if ipb: branchlist.append(ipb) #get the xpub key of the whole account xpub_account = wallet.get_bip32_pub_export(mixdepth=m) - acctlist.append(WalletViewAccount(rootpath, m, branchlist, + path = wallet.get_path_repr(wallet.get_path(m)) + acctlist.append(WalletViewAccount(path, m, branchlist, xpub=xpub_account)) - walletview = WalletView(rootpath, acctlist) + path = wallet.get_path_repr(wallet.get_path()) + walletview = WalletView(path, acctlist) if serialized: return walletview.serialize(summarize=summarized) else: @@ -384,7 +399,7 @@ def cli_get_wallet_passphrase_check(): return password def cli_get_wallet_file_name(): - return raw_input('Input wallet file name (default: wallet.json): ') + return raw_input('Input wallet file name (default: wallet.jmdat): ') def cli_display_user_words(words, mnemonic_extension): text = 'Write down this wallet recovery mnemonic\n\n' + words +'\n' @@ -393,7 +408,7 @@ def cli_display_user_words(words, mnemonic_extension): print(text) def cli_user_mnemonic_entry(): - mnemonic_phrase = raw_input("Input 12 word mnemonic recovery phrase: ") + mnemonic_phrase = raw_input("Input mnemonic recovery phrase: ") mnemonic_extension = raw_input("Input mnemonic extension, leave blank if there isnt one: ") if len(mnemonic_extension.strip()) == 0: mnemonic_extension = None @@ -409,33 +424,8 @@ def cli_get_mnemonic_extension(): return raw_input("Enter mnemonic extension: ") -def persist_walletfile(walletspath, default_wallet_name, encrypted_entropy, - encrypted_mnemonic_extension=None, - callbacks=(cli_get_wallet_file_name,)): - timestamp = datetime.now().strftime("%Y/%m/%d %H:%M:%S") - walletjson = {'creator': 'joinmarket project', - 'creation_time': timestamp, - 'encrypted_entropy': encrypted_entropy.encode('hex'), - 'network': get_network()} - if encrypted_mnemonic_extension: - walletjson['encrypted_mnemonic_extension'] = encrypted_mnemonic_extension.encode('hex') - walletfile = json.dumps(walletjson) - walletname = callbacks[0]() - if len(walletname) == 0: - walletname = default_wallet_name - walletpath = os.path.join(walletspath, walletname) - # Does a wallet with the same name exist? - if os.path.isfile(walletpath): - print('ERROR: ' + walletpath + ' already exists. Aborting.') - return False - else: - fd = open(walletpath, 'w') - fd.write(walletfile) - fd.close() - print('saved to ' + walletname) - return True - def wallet_generate_recover_bip39(method, walletspath, default_wallet_name, + mixdepths=5, callbacks=(cli_display_user_words, cli_user_mnemonic_entry, cli_get_wallet_passphrase_check, @@ -449,68 +439,77 @@ def wallet_generate_recover_bip39(method, walletspath, default_wallet_name, 4 - enter mnemonic extension The defaults are for terminal entry. """ - #using 128 bit entropy, 12 words, mnemonic module - m = Mnemonic("english") + entropy = None + mnemonic_extension = None if method == "generate": mnemonic_extension = callbacks[4]() - words = m.generate() - callbacks[0](words, mnemonic_extension) elif method == 'recover': words, mnemonic_extension = callbacks[1]() + mnemonic_extension = mnemonic_extension and mnemonic_extension.strip() if not words: return False - entropy = str(m.to_entropy(words)) + try: + entropy = SegwitLegacyWallet.entropy_from_mnemonic(words) + except WalletError: + return False + else: + raise Exception("unknown method for wallet creation: '{}'" + .format(method)) + password = callbacks[2]() if not password: return False - password_key = btc.bin_dbl_sha256(password) - encrypted_entropy = encryptData(password_key, entropy) - encrypted_mnemonic_extension = None - if mnemonic_extension: - mnemonic_extension = mnemonic_extension.strip() - #check all ascii printable - if not all([a > '\x19' and a < '\x7f' for a in mnemonic_extension]): - return False - #padding to stop an adversary easily telling how long the mn extension is - #padding at the start because of how aes blocks are combined - #checksum in order to tell whether the decryption was successful - cleartext_length = 79 - padding_length = cleartext_length - 10 - len(mnemonic_extension) - if padding_length > 0: - padding = os.urandom(padding_length).replace('\xff', '\xfe') - else: - padding = '' - cleartext = (padding + '\xff' + mnemonic_extension + '\xff' - + btc.dbl_sha256(mnemonic_extension)[:8]) - encrypted_mnemonic_extension = encryptData(password_key, cleartext) - return persist_walletfile(walletspath, default_wallet_name, encrypted_entropy, - encrypted_mnemonic_extension, callbacks=(callbacks[3],)) + + wallet_name = callbacks[3]() + if not wallet_name: + wallet_name = default_wallet_name + wallet_path = os.path.join(walletspath, wallet_name) + + wallet = create_wallet(wallet_path, password, mixdepths - 1, + entropy=entropy, + entropy_extension=mnemonic_extension) + mnemonic, mnext = wallet.get_mnemonic_words() + callbacks[0] and callbacks[0](mnemonic, mnext or '') + wallet.close() + return True + def wallet_generate_recover(method, walletspath, - default_wallet_name='wallet.json'): - if jm_single().config.get("POLICY", "segwit") == "true": + default_wallet_name='wallet.jmdat', + mixdepths=5): + if is_segwit_mode(): #Here using default callbacks for scripts (not used in Qt) - return wallet_generate_recover_bip39(method, walletspath, - default_wallet_name) - if method == 'generate': - seed = btc.sha256(os.urandom(64))[:32] - words = mn_encode(seed) - print('Write down this wallet recovery seed\n\n' + ' '.join(words) + - '\n') - elif method == 'recover': - words = raw_input('Input 12 word recovery seed: ') - words = words.split() # default for split is 1 or more whitespace chars - if len(words) != 12: - print('ERROR: Recovery seed phrase must be exactly 12 words.') + return wallet_generate_recover_bip39( + method, walletspath, default_wallet_name, mixdepths=mixdepths) + + entropy = None + if method == 'recover': + seed = raw_input("Input 12 word recovery seed: ") + try: + entropy = LegacyWallet.entropy_from_mnemonic(seed) + except WalletError as e: + print("Unable to restore seed: {}".format(e.message)) return False - seed = mn_decode(words) - print(seed) + elif method != 'generate': + raise Exception("unknown method for wallet creation: '{}'" + .format(method)) + password = cli_get_wallet_passphrase_check() if not password: return False - password_key = btc.bin_dbl_sha256(password) - encrypted_seed = encryptData(password_key, seed.decode('hex')) - return persist_walletfile(walletspath, default_wallet_name, encrypted_seed) + + wallet_name = cli_get_wallet_file_name() + if not wallet_name: + wallet_name = default_wallet_name + wallet_path = os.path.join(walletspath, wallet_name) + + wallet = create_wallet(wallet_path, password, mixdepths - 1, + wallet_cls=LegacyWallet, entropy=entropy) + print("Write down and safely store this wallet recovery seed\n\n{}\n" + .format(wallet.get_mnemonic_words()[0])) + wallet.close() + return True + def wallet_fetch_history(wallet, options): # sort txes in a db because python can be really bad with large lists @@ -533,10 +532,12 @@ def wallet_fetch_history(wallet, options): in tx) tx_db.executemany('INSERT INTO transactions VALUES(?, ?, ?);', tx_data) - txes = tx_db.execute('SELECT DISTINCT txid, blockhash, blocktime ' - 'FROM transactions ORDER BY blocktime').fetchall() - wallet_addr_cache = wallet.addr_cache - wallet_addr_set = set(wallet_addr_cache.keys()) + + txes = tx_db.execute( + 'SELECT DISTINCT txid, blockhash, blocktime ' + 'FROM transactions ORDER BY blocktime').fetchall() + wallet_script_set = set(wallet.get_script_path(p) + for p in wallet.yield_known_paths()) def s(): return ',' if options.csv else ' ' @@ -575,13 +576,13 @@ def wallet_fetch_history(wallet, options): rpctx = jm_single().bc_interface.rpc('gettransaction', [tx['txid']]) txhex = str(rpctx['hex']) txd = btc.deserialize(txhex) - output_addr_values = dict(((btc.script_to_address(sv['script'], - get_p2sh_vbyte()), sv['value']) for sv in txd['outs'])) - our_output_addrs = wallet_addr_set.intersection( - output_addr_values.keys()) + output_script_values = {binascii.unhexlify(sv['script']): sv['value'] + for sv in txd['outs']} + our_output_scripts = wallet_script_set.intersection( + output_script_values.keys()) from collections import Counter - value_freq_list = sorted(Counter(output_addr_values.values()) + value_freq_list = sorted(Counter(output_script_values.values()) .most_common(), key=lambda x: -x[1]) non_cj_freq = 0 if len(value_freq_list)==1 else sum(zip( *value_freq_list[1:])[1]) @@ -601,12 +602,12 @@ def wallet_fetch_history(wallet, options): 'outpoint']['index']] rpc_inputs.append(input_dict) - rpc_input_addrs = set((btc.script_to_address(ind['script'], - get_p2sh_vbyte()) for ind in rpc_inputs)) - our_input_addrs = wallet_addr_set.intersection(rpc_input_addrs) - our_input_values = [ind['value'] for ind in rpc_inputs if btc. - script_to_address(ind['script'], get_p2sh_vbyte()) in - our_input_addrs] + rpc_input_scripts = set(binascii.unhexlify(ind['script']) + for ind in rpc_inputs) + our_input_scripts = wallet_script_set.intersection(rpc_input_scripts) + our_input_values = [ + ind['value'] for ind in rpc_inputs + if binascii.unhexlify(ind['script']) in our_input_scripts] our_input_value = sum(our_input_values) utxos_consumed = len(our_input_values) @@ -618,19 +619,19 @@ def wallet_fetch_history(wallet, options): mixdepth_dst = -1 #TODO this seems to assume all the input addresses are from the same # mixdepth, which might not be true - if len(our_input_addrs) == 0 and len(our_output_addrs) > 0: + if len(our_input_scripts) == 0 and len(our_output_scripts) > 0: #payment to us - amount = sum([output_addr_values[a] for a in our_output_addrs]) + amount = sum([output_script_values[a] for a in our_output_scripts]) tx_type = 'deposit ' cj_n = -1 delta_balance = amount - mixdepth_dst = tuple(wallet_addr_cache[a][0] for a in - our_output_addrs) + mixdepth_dst = tuple(wallet.get_script_mixdepth(a) + for a in our_output_scripts) if len(mixdepth_dst) == 1: mixdepth_dst = mixdepth_dst[0] - elif len(our_input_addrs) == 0 and len(our_output_addrs) == 0: + elif len(our_input_scripts) == 0 and len(our_output_scripts) == 0: continue # skip those that don't belong to our wallet - elif len(our_input_addrs) > 0 and len(our_output_addrs) == 0: + elif len(our_input_scripts) > 0 and len(our_output_scripts) == 0: # we swept coins elsewhere if is_coinjoin: tx_type = 'cj sweepout' @@ -638,13 +639,13 @@ def wallet_fetch_history(wallet, options): fees = our_input_value - cj_amount else: tx_type = 'sweep out ' - amount = sum([v for v in output_addr_values.values()]) + amount = sum([v for v in output_script_values.values()]) fees = our_input_value - amount delta_balance = -our_input_value - mixdepth_src = wallet_addr_cache[list(our_input_addrs)[0]][0] - elif len(our_input_addrs) > 0 and len(our_output_addrs) == 1: + mixdepth_src = wallet.get_script_mixdepth(list(our_input_scripts)[0]) + elif len(our_input_scripts) > 0 and len(our_output_scripts) == 1: # payment to somewhere with our change address getting the remaining - change_value = output_addr_values[list(our_output_addrs)[0]] + change_value = output_script_values[list(our_output_scripts)[0]] if is_coinjoin: tx_type = 'cj withdraw' amount = cj_amount @@ -655,25 +656,25 @@ def wallet_fetch_history(wallet, options): cj_n = -1 delta_balance = change_value - our_input_value fees = our_input_value - change_value - cj_amount - mixdepth_src = wallet_addr_cache[list(our_input_addrs)[0]][0] - elif len(our_input_addrs) > 0 and len(our_output_addrs) == 2: + mixdepth_src = wallet.get_script_mixdepth(list(our_input_scripts)[0]) + elif len(our_input_scripts) > 0 and len(our_output_scripts) == 2: #payment to self - out_value = sum([output_addr_values[a] for a in our_output_addrs]) + out_value = sum([output_script_values[a] for a in our_output_scripts]) if not is_coinjoin: print('this is wrong TODO handle non-coinjoin internal') tx_type = 'cj internal' amount = cj_amount delta_balance = out_value - our_input_value - mixdepth_src = wallet_addr_cache[list(our_input_addrs)[0]][0] - cj_addr = list(set([a for a,v in output_addr_values.iteritems() - if v == cj_amount]).intersection(our_output_addrs))[0] - mixdepth_dst = wallet_addr_cache[cj_addr][0] + mixdepth_src = wallet.get_script_mixdepth(list(our_input_scripts)[0]) + cj_script = list(set([a for a, v in output_script_values.iteritems() + if v == cj_amount]).intersection(our_output_scripts))[0] + mixdepth_dst = wallet.get_script_mixdepth(cj_script) else: tx_type = 'unknown type' - print('our utxos: ' + str(len(our_input_addrs)) \ - + ' in, ' + str(len(our_output_addrs)) + ' out') + print('our utxos: ' + str(len(our_input_scripts)) \ + + ' in, ' + str(len(our_output_scripts)) + ' out') balance += delta_balance - utxo_count += (len(our_output_addrs) - utxos_consumed) + utxo_count += (len(our_output_scripts) - utxos_consumed) index = '% 4d'%(i) timestamp = datetime.fromtimestamp(rpctx['blocktime'] ).strftime("%Y-%m-%d %H:%M") @@ -760,9 +761,10 @@ def wallet_fetch_history(wallet, options): print(('BUG ERROR: wallet balance (%s) does not match balance from ' + 'history (%s)') % (sat_to_str(total_wallet_balance), sat_to_str(balance))) - if utxo_count != len(wallet.unspent): + wallet_utxo_count = sum(map(len, wallet.get_utxos_by_mixdepth_().values())) + if utxo_count != wallet_utxo_count: print(('BUG ERROR: wallet utxo count (%d) does not match utxo count from ' + - 'history (%s)') % (len(wallet.unspent), utxo_count)) + 'history (%s)') % (wallet_utxo_count, utxo_count)) def wallet_showseed(wallet): @@ -773,7 +775,7 @@ def wallet_showseed(wallet): return text -def wallet_importprivkey(wallet, mixdepth): +def wallet_importprivkey(wallet, mixdepth, key_type): print("WARNING: This imported key will not be recoverable with your 12 " "word mnemonic phrase. Make sure you have backups.") print("WARNING: Handling of raw ECDSA bitcoin private keys can lead to " @@ -782,24 +784,35 @@ def wallet_importprivkey(wallet, mixdepth): privkeys = raw_input("Enter private key(s) to import: ") privkeys = privkeys.split(',') if ',' in privkeys else privkeys.split() imported_addr = [] + import_failed = 0 # TODO read also one key for each line for wif in privkeys: # TODO is there any point in only accepting wif format? check what # other wallets do - imported_addr.append(wallet.import_private_key(mixdepth, wif)) - wallet.save() + try: + path = wallet.import_private_key(mixdepth, wif, key_type=key_type) + except WalletError as e: + print("Failed to import key {}: {}".format(wif, e)) + import_failed += 1 + else: + imported_addr.append(wallet.get_addr_path(path)) if not imported_addr: print("Warning: No keys imported!") return + wallet.save() + # show addresses to user so they can verify everything went as expected - print("Imported keys for addresses:") - for addr in imported_addr: - print(addr) + print("Imported keys for addresses:\n{}".format('\n'.join(imported_addr))) + if import_failed: + print("Warning: failed to import {} keys".format(import_failed)) def wallet_dumpprivkey(wallet, hdpath): + if not hdpath: + print("Error: no hd wallet path supplied") + return False path = wallet.path_repr_to_path(hdpath) return wallet.get_wif_path(path) # will raise exception on invalid path @@ -807,17 +820,22 @@ def wallet_dumpprivkey(wallet, hdpath): def wallet_signmessage(wallet, hdpath, message): msg = message.encode('utf-8') + if not hdpath: + return "Error: no key path for signing specified" + if not message: + return "Error: no message specified" + path = wallet.path_repr_to_path(hdpath) sig = wallet.sign_message(msg, path) return ("Signature: {}\n" "To verify this in Bitcoin Core use the RPC command 'verifymessage'" - "".format(sig)) + .format(sig)) def get_wallet_type(): - if jm_single().config.get('POLICY', 'segwit') == 'true': - return 'p2sh-p2wpkh' - return 'p2pkh' + if is_segwit_mode(): + return TYPE_P2SH_P2WPKH + return TYPE_P2PKH def get_wallet_cls(wtype=None): @@ -832,15 +850,17 @@ def get_wallet_cls(wtype=None): return cls -def create_wallet(path, password, max_mixdepth, **kwargs): +def create_wallet(path, password, max_mixdepth, wallet_cls=None, **kwargs): storage = Storage(path, password, create=True) - wallet_cls = get_wallet_cls() + wallet_cls = wallet_cls or get_wallet_cls() wallet_cls.initialize(storage, get_network(), max_mixdepth=max_mixdepth, **kwargs) + storage.save() + return wallet_cls(storage) def open_test_wallet_maybe(path, seed, max_mixdepth, - test_wallet_cls=SewgitTestWallet, **kwargs): + test_wallet_cls=SegwitLegacyWallet, **kwargs): """ Create a volatile test wallet if path is a hex-encoded string of length 64, otherwise run open_wallet(). @@ -903,19 +923,18 @@ def open_wallet(path, ask_for_password=True, password=None, read_only=False, else: storage = Storage(path, password, read_only=read_only) - wallet_cls = get_wallet_cls(storage) + wallet_cls = get_wallet_cls_from_storage(storage) wallet = wallet_cls(storage, **kwargs) wallet_sanity_check(wallet) return wallet def get_wallet_cls_from_storage(storage): - wtype = storage.data.get([b'wallet_type']) + wtype = storage.data.get(b'wallet_type') - if not wtype: + if wtype is None: raise WalletError("File {} is not a valid wallet.".format(storage.path)) - wtype = wtype.decode('ascii') return get_wallet_cls(wtype) @@ -936,12 +955,6 @@ def wallet_tool_main(wallet_root_path): """ parser = get_wallettool_parser() (options, args) = parser.parse_args() - # if the index_cache stored in wallet.json is longer than the default - # then set maxmixdepth to the length of index_cache - maxmixdepth_configured = True - if not options.maxmixdepth: - maxmixdepth_configured = False - options.maxmixdepth = 5 noseed_methods = ['generate', 'recover'] methods = ['display', 'displayall', 'summary', 'showseed', 'importprivkey', @@ -953,6 +966,10 @@ def wallet_tool_main(wallet_root_path): parser.error('Needs a wallet file or method') sys.exit(0) + if options.mixdepths < 1: + parser.error("Must have at least one mixdepth.") + sys.exit(0) + if args[0] in noseed_methods: method = args[0] else: @@ -961,7 +978,7 @@ def wallet_tool_main(wallet_root_path): method = ('display' if len(args) == 1 else args[1].lower()) wallet = open_test_wallet_maybe( - wallet_path, seed, options.maxmixdepth, gap_limit=options.gaplimit) + wallet_path, seed, options.mixdepths - 1, gap_limit=options.gaplimit) if method not in noscan_methods: # if nothing was configured, we override bitcoind's options so that @@ -969,11 +986,13 @@ def wallet_tool_main(wallet_root_path): if 'listunspent_args' not in jm_single().config.options('POLICY'): jm_single().config.set('POLICY','listunspent_args', '[0]') sync_wallet(wallet, fast=options.fastsync) + wallet.save() #Now the wallet/data is prepared, execute the script according to the method if method == "display": return wallet_display(wallet, options.gaplimit, options.showprivkey) elif method == "displayall": - return wallet_display(wallet, options.gaplimit, options.showprivkey, displayall=True) + return wallet_display(wallet, options.gaplimit, options.showprivkey, + displayall=True) elif method == "summary": return wallet_display(wallet, options.gaplimit, options.showprivkey, summarized=True) elif method == "history": @@ -984,10 +1003,12 @@ def wallet_tool_main(wallet_root_path): else: return wallet_fetch_history(wallet, options) elif method == "generate": - retval = wallet_generate_recover("generate", wallet_root_path) + retval = wallet_generate_recover("generate", wallet_root_path, + mixdepths=options.mixdepths) return retval if retval else "Failed" elif method == "recover": - retval = wallet_generate_recover("recover", wallet_root_path) + retval = wallet_generate_recover("recover", wallet_root_path, + mixdepths=options.mixdepths) return retval if retval else "Failed" elif method == "showutxos": return wallet_showutxos(wallet, options.showprivkey) @@ -997,11 +1018,13 @@ def wallet_tool_main(wallet_root_path): return wallet_dumpprivkey(wallet, options.hd_path) elif method == "importprivkey": #note: must be interactive (security) - wallet_importprivkey(wallet, options.mixdepth) + wallet_importprivkey(wallet, options.mixdepth, + map_key_type(options.key_type)) return "Key import completed." elif method == "signmessage": return wallet_signmessage(wallet, options.hd_path, args[2]) + #Testing (can port to test modules, TODO) if __name__ == "__main__": if not test_bip32_pathparse(): From 8ca6cfc0575a306eb3f9d4eb1545e64271708171 Mon Sep 17 00:00:00 2001 From: undeath Date: Tue, 3 Jul 2018 15:00:34 +0200 Subject: [PATCH 12/23] make sure new addresses always get imported --- jmclient/jmclient/maker.py | 8 ++++++++ jmclient/jmclient/taker.py | 11 ++++++++++- jmclient/jmclient/taker_utils.py | 11 +++++++++++ jmclient/jmclient/yieldgenerator.py | 1 + 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/jmclient/jmclient/maker.py b/jmclient/jmclient/maker.py index 0ae3117..500a50b 100644 --- a/jmclient/jmclient/maker.py +++ b/jmclient/jmclient/maker.py @@ -216,3 +216,11 @@ class Maker(object): if len(oldorder_s) > 0: self.offerlist.remove(oldorder_s[0]) self.offerlist += to_announce + + def import_new_addresses(self, addr_list): + # FIXME: same code as in taker.py + bci = jm_single().bc_interface + if not hasattr(bci, 'import_addresses'): + return + assert hasattr(bci, 'get_wallet_name') + bci.import_addresses(addr_list, bci.get_wallet_name(self.wallet)) diff --git a/jmclient/jmclient/taker.py b/jmclient/jmclient/taker.py index 43e5295..a567645 100644 --- a/jmclient/jmclient/taker.py +++ b/jmclient/jmclient/taker.py @@ -4,7 +4,6 @@ from __future__ import print_function import base64 import pprint import random -from itertools import chain import btc from jmclient.configure import jm_single, get_p2pk_vbyte, get_p2sh_vbyte @@ -184,6 +183,7 @@ class Taker(object): jlog.info("Choosing a destination from mixdepth: " + str(next_mixdepth)) self.my_cj_addr = self.wallet.get_internal_addr(next_mixdepth) jlog.info("Chose destination address: " + self.my_cj_addr) + self.import_new_addresses([self.my_cj_addr]) self.outputs = [] self.cjfee_total = 0 self.maker_txfee_contributions = 0 @@ -268,6 +268,7 @@ class Taker(object): if self.cjamount != 0: try: self.my_change_addr = self.wallet.get_internal_addr(self.mixdepth) + self.import_new_addresses([self.my_change_addr]) except: self.taker_info_callback("ABORT", "Failed to get a change address") return False @@ -806,3 +807,11 @@ class Taker(object): waittime = self.schedule[self.schedule_index][4] self.on_finished_callback(True, fromtx=fromtx, waittime=waittime, txdetails=(txd, txid)) + + def import_new_addresses(self, addr_list): + # FIXME: same code as in maker.py + bci = jm_single().bc_interface + if not hasattr(bci, 'import_addresses'): + return + assert hasattr(bci, 'get_wallet_name') + bci.import_addresses(addr_list, bci.get_wallet_name(self.wallet)) diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index a91b108..32e8a8d 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -67,6 +67,7 @@ def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False, changeval = total_inputs_val - fee_est - amount outs = [{"value": amount, "address": destaddr}] change_addr = wallet.get_internal_addr(mixdepth) + import_new_addresses(wallet, [change_addr]) outs.append({"value": changeval, "address": change_addr}) #Now ready to construct transaction @@ -104,6 +105,16 @@ def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False, cb(successmsg) return txid + +def import_new_addresses(wallet, addr_list): + # FIXME: same code as in maker.py and taker.py + bci = jm_single().bc_interface + if not hasattr(bci, 'import_addresses'): + return + assert hasattr(bci, 'get_wallet_name') + bci.import_addresses(addr_list, bci.get_wallet_name(wallet)) + + def get_tumble_log(logsdir): tumble_log = logging.getLogger('tumbler') tumble_log.setLevel(logging.DEBUG) diff --git a/jmclient/jmclient/yieldgenerator.py b/jmclient/jmclient/yieldgenerator.py index 4fc89ad..e3152b6 100644 --- a/jmclient/jmclient/yieldgenerator.py +++ b/jmclient/jmclient/yieldgenerator.py @@ -132,6 +132,7 @@ class YieldGeneratorBasic(YieldGenerator): cj_addr = self.wallet.get_internal_addr((mixdepth + 1) % self.wallet.max_mix_depth) change_addr = self.wallet.get_internal_addr(mixdepth) + self.import_new_addresses([cj_addr, change_addr]) utxos = self.wallet.select_utxos(mixdepth, total_amount) my_total_in = sum([va['value'] for va in utxos.values()]) From 8b9abef4fb1d6bd42987fc4c1f939f93010e32fc Mon Sep 17 00:00:00 2001 From: undeath Date: Sun, 8 Jul 2018 13:17:14 +0200 Subject: [PATCH 13/23] add is_segwit_mode() utility function --- jmclient/jmclient/__init__.py | 2 +- jmclient/jmclient/configure.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index 8174bd9..d6dc3da 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -26,7 +26,7 @@ from .cryptoengine import BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, EngineError from .configure import ( load_program_config, get_p2pk_vbyte, jm_single, get_network, validate_address, get_irc_mchannels, get_blockchain_interface_instance, - get_p2sh_vbyte, set_config) + get_p2sh_vbyte, set_config, is_segwit_mode) from .blockchaininterface import (BlockchainInterface, sync_wallet, RegtestBitcoinCoreInterface, BitcoinCoreInterface) from .electruminterface import ElectrumInterface diff --git a/jmclient/jmclient/configure.py b/jmclient/jmclient/configure.py index fefb317..07ee465 100644 --- a/jmclient/jmclient/configure.py +++ b/jmclient/jmclient/configure.py @@ -414,3 +414,7 @@ def get_blockchain_interface_instance(_config): else: raise ValueError("Invalid blockchain source") return bc_interface + + +def is_segwit_mode(): + return jm_single().config.get('POLICY', 'segwit') != 'false' From 39e42762c754b5e0a46f17e342c21df5d2f371f6 Mon Sep 17 00:00:00 2001 From: undeath Date: Mon, 9 Jul 2018 17:41:46 +0200 Subject: [PATCH 14/23] change default wallet name --- scripts/README.md | 4 ++-- scripts/joinmarket-qt.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index f9977ab..01a6214 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -18,12 +18,12 @@ Brief explanation of the function of each of the scripts: Either use the same syntax as for normal Joinmarket: - `python sendpayment.py --fast -N 3 -m 1 -P wallet.json 50000000
` + `python sendpayment.py --fast -N 3 -m 1 -P wallet.jmdat 50000000
` or use the new schedule approach. For an example, see the [sample schedule file](https://github.com/AdamISZ/joinmarket-clientserver/blob/master/scripts/sample-schedule-for-testnet). Do: - `python sendpayment.py --fast -S sample-schedule-for-testnet wallet.json` + `python sendpayment.py --fast -S sample-schedule-for-testnet wallet.jmdat` Note that the magic string `INTERNAL` in the file creates a payment to a new address in the next mixdepth (wrapping around to zero if you reach the maximum mixdepth). diff --git a/scripts/joinmarket-qt.py b/scripts/joinmarket-qt.py index 277beb7..87d76d9 100644 --- a/scripts/joinmarket-qt.py +++ b/scripts/joinmarket-qt.py @@ -1320,7 +1320,7 @@ class JMMainWindow(QMainWindow): def recoverWallet(self): success = wallet_generate_recover_bip39("recover", "wallets", - "wallet.json", + "wallet.jmdat", callbacks=(None, self.seedEntry, self.getPassword, self.getWalletFileName)) @@ -1470,7 +1470,7 @@ class JMMainWindow(QMainWindow): def getWalletFileName(self): walletname, ok = QInputDialog.getText(self, 'Choose wallet name', 'Enter wallet file name:', - QLineEdit.Normal, "wallet.json") + QLineEdit.Normal, "wallet.jmdat") if not ok: JMQtMessageBox(self, "Create wallet aborted", mbtype='warn') return None @@ -1513,7 +1513,7 @@ class JMMainWindow(QMainWindow): if not seed: success = wallet_generate_recover_bip39("generate", "wallets", - "wallet.json", + "wallet.jmdat", callbacks=(self.displayWords, None, self.getPassword, From 747c2277626a8e2e0932fb488b86795e33de7401 Mon Sep 17 00:00:00 2001 From: undeath Date: Wed, 11 Jul 2018 17:56:09 +0200 Subject: [PATCH 15/23] fix some max_mixdepth off-by-one errors --- jmclient/jmclient/taker.py | 2 +- jmclient/jmclient/yieldgenerator.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jmclient/jmclient/taker.py b/jmclient/jmclient/taker.py index a567645..bcc02a4 100644 --- a/jmclient/jmclient/taker.py +++ b/jmclient/jmclient/taker.py @@ -179,7 +179,7 @@ class Taker(object): #if destination is flagged "INTERNAL", choose a destination #from the next mixdepth modulo the maxmixdepth if self.my_cj_addr == "INTERNAL": - next_mixdepth = (self.mixdepth + 1) % self.wallet.max_mix_depth + next_mixdepth = (self.mixdepth + 1) % (self.wallet.max_mixdepth + 1) jlog.info("Choosing a destination from mixdepth: " + str(next_mixdepth)) self.my_cj_addr = self.wallet.get_internal_addr(next_mixdepth) jlog.info("Chose destination address: " + self.my_cj_addr) diff --git a/jmclient/jmclient/yieldgenerator.py b/jmclient/jmclient/yieldgenerator.py index e3152b6..1ec25df 100644 --- a/jmclient/jmclient/yieldgenerator.py +++ b/jmclient/jmclient/yieldgenerator.py @@ -129,8 +129,8 @@ class YieldGeneratorBasic(YieldGenerator): jlog.info('filling offer, mixdepth=' + str(mixdepth)) # mixdepth is the chosen depth we'll be spending from - cj_addr = self.wallet.get_internal_addr((mixdepth + 1) % - self.wallet.max_mix_depth) + cj_addr = self.wallet.get_internal_addr( + (mixdepth + 1) % (self.wallet.max_mixdepth + 1)) change_addr = self.wallet.get_internal_addr(mixdepth) self.import_new_addresses([cj_addr, change_addr]) From 34f8600996fad01fc9e6f8591259f0f17d4345fe Mon Sep 17 00:00:00 2001 From: undeath Date: Sat, 14 Jul 2018 19:44:23 +0200 Subject: [PATCH 16/23] fix wallet syncing --- jmclient/jmclient/wallet_utils.py | 4 ++-- jmclient/jmclient/yieldgenerator.py | 4 +++- scripts/add-utxo.py | 3 ++- scripts/jmtainter.py | 3 ++- scripts/sendpayment.py | 3 ++- scripts/tumbler.py | 3 ++- 6 files changed, 13 insertions(+), 7 deletions(-) diff --git a/jmclient/jmclient/wallet_utils.py b/jmclient/jmclient/wallet_utils.py index e7bc12a..7b4319e 100644 --- a/jmclient/jmclient/wallet_utils.py +++ b/jmclient/jmclient/wallet_utils.py @@ -985,8 +985,8 @@ def wallet_tool_main(wallet_root_path): # unconfirmed balance is included in the wallet display by default if 'listunspent_args' not in jm_single().config.options('POLICY'): jm_single().config.set('POLICY','listunspent_args', '[0]') - sync_wallet(wallet, fast=options.fastsync) - wallet.save() + while not jm_single().bc_interface.wallet_synced: + sync_wallet(wallet, fast=options.fastsync) #Now the wallet/data is prepared, execute the script according to the method if method == "display": return wallet_display(wallet, options.gaplimit, options.showprivkey) diff --git a/jmclient/jmclient/yieldgenerator.py b/jmclient/jmclient/yieldgenerator.py index 1ec25df..07641a4 100644 --- a/jmclient/jmclient/yieldgenerator.py +++ b/jmclient/jmclient/yieldgenerator.py @@ -240,7 +240,9 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffe if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == "electrum-server": jm_single().bc_interface.synctype = "with-script" - sync_wallet(wallet, fast=options.fastsync) + + while not jm_single().bc_interface.wallet_synced: + sync_wallet(wallet, fast=options.fastsync) maker = ygclass(wallet, [options.txfee, cjfee_a, cjfee_r, options.ordertype, options.minsize]) diff --git a/scripts/add-utxo.py b/scripts/add-utxo.py index 1a1c29a..e8254de 100644 --- a/scripts/add-utxo.py +++ b/scripts/add-utxo.py @@ -176,7 +176,8 @@ def main(): if options.loadwallet: wallet_path = get_wallet_path(options.loadwallet, None) wallet = open_wallet(wallet_path, gap_limit=options.gaplimit) - sync_wallet(wallet, fast=options.fastsync) + while not jm_single().bc_interface.wallet_synced: + sync_wallet(wallet, fast=options.fastsync) for md, utxos in wallet.get_utxos_by_mixdepth_().items(): for (txid, index), utxo in utxos.items(): diff --git a/scripts/jmtainter.py b/scripts/jmtainter.py index 24f216c..253121d 100644 --- a/scripts/jmtainter.py +++ b/scripts/jmtainter.py @@ -84,7 +84,8 @@ def cli_get_wallet(wallet_name, sync=True): "blockchain_source") == "electrum-server": jm_single().bc_interface.synctype = "with-script" if sync: - sync_wallet(wallet, fast=options.fastsync) + while not jm_single().bc_interface.wallet_synced: + sync_wallet(wallet, fast=options.fastsync) return wallet #======Electrum specific utils========================= diff --git a/scripts/sendpayment.py b/scripts/sendpayment.py index ff6598e..ed033fd 100644 --- a/scripts/sendpayment.py +++ b/scripts/sendpayment.py @@ -134,7 +134,8 @@ def main(): "blockchain_source") == "electrum-server" and options.makercount != 0: jm_single().bc_interface.synctype = "with-script" #wallet sync will now only occur on reactor start if we're joining. - sync_wallet(wallet, fast=options.fastsync) + while not jm_single().bc_interface.wallet_synced: + sync_wallet(wallet, fast=options.fastsync) if options.makercount == 0: direct_send(wallet, amount, mixdepth, destaddr, options.answeryes) return diff --git a/scripts/tumbler.py b/scripts/tumbler.py index e843c6a..5d05b3c 100644 --- a/scripts/tumbler.py +++ b/scripts/tumbler.py @@ -38,7 +38,8 @@ def main(): if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == "electrum-server": jm_single().bc_interface.synctype = "with-script" - sync_wallet(wallet, fast=options['fastsync']) + while not jm_single().bc_interface.wallet_synced: + sync_wallet(wallet, fast=options['fastsync']) #Parse options and generate schedule #Output information to log files From 703ae04dcc6fc66afc662a94aecadb1decc756e2 Mon Sep 17 00:00:00 2001 From: undeath Date: Sat, 14 Jul 2018 21:37:55 +0200 Subject: [PATCH 17/23] remove wallet.sign() --- jmclient/jmclient/maker.py | 36 ++++++++++++---------- jmclient/jmclient/taker.py | 53 ++++++++++++-------------------- jmclient/jmclient/taker_utils.py | 27 ++++++++++------ jmclient/test/test_maker.py | 11 +++---- jmclient/test/test_taker.py | 27 ++++++---------- 5 files changed, 70 insertions(+), 84 deletions(-) diff --git a/jmclient/jmclient/maker.py b/jmclient/jmclient/maker.py index 500a50b..9690d12 100644 --- a/jmclient/jmclient/maker.py +++ b/jmclient/jmclient/maker.py @@ -3,16 +3,13 @@ from __future__ import print_function import base64 import pprint -import random import sys -import time -import copy +from binascii import unhexlify import btc from jmclient.configure import jm_single, get_p2pk_vbyte, get_p2sh_vbyte from jmbase.support import get_log from jmclient.support import (calc_cj_fee) -from jmclient.wallet import estimate_tx_fee from jmclient.podle import (generate_podle, get_podle_commitments, verify_podle, PoDLE, PoDLEError, generate_podle_error_string) from twisted.internet import task @@ -75,7 +72,11 @@ class Maker(object): if res[0]['value'] < reqd_amt: reason = "commitment utxo too small: " + str(res[0]['value']) return reject(reason) - if res[0]['address'] != self.wallet.pubkey_to_address(cr_dict['P']): + + # FIXME: This only works if taker's commitment address is of same type + # as our wallet. + if res[0]['address'] != \ + self.wallet.pubkey_to_addr(unhexlify(cr_dict['P'])): reason = "Invalid podle pubkey: " + str(cr_dict['P']) return reject(reason) @@ -114,22 +115,25 @@ class Maker(object): jlog.info('goodtx') sigs = [] utxos = offerinfo["utxos"] + + our_inputs = {} for index, ins in enumerate(tx['ins']): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - if utxo not in utxos.keys(): + if utxo not in utxos: continue - addr = utxos[utxo]['address'] - amount = utxos[utxo]["value"] - txs = self.wallet.sign(txhex, index, - self.wallet.get_key_from_addr(addr), - amount=amount) - sigmsg = btc.deserialize(txs)["ins"][index]["script"].decode("hex") - if "txinwitness" in btc.deserialize(txs)["ins"][index].keys(): + script = self.wallet.addr_to_script(utxos[utxo]['address']) + amount = utxos[utxo]['value'] + our_inputs[index] = (script, amount) + + txs = self.wallet.sign_tx(btc.deserialize(unhexlify(txhex)), our_inputs) + + for index in our_inputs: + sigmsg = txs['ins'][index]['script'] + if 'txinwitness' in txs['ins'][index]: #We prepend the witness data since we want (sig, pub, scriptCode); #also, the items in witness are not serialize_script-ed. - sigmsg = "".join([btc.serialize_script_unit( - x.decode("hex")) for x in btc.deserialize( - txs)["ins"][index]["txinwitness"]]) + sigmsg + sigmsg = b''.join(btc.serialize_script_unit(x) + for x in txs['ins'][index]['txinwitness']) + sigmsg sigs.append(base64.b64encode(sigmsg)) return (True, sigs) diff --git a/jmclient/jmclient/taker.py b/jmclient/jmclient/taker.py index bcc02a4..dd75722 100644 --- a/jmclient/jmclient/taker.py +++ b/jmclient/jmclient/taker.py @@ -4,6 +4,7 @@ from __future__ import print_function import base64 import pprint import random +from binascii import hexlify, unhexlify import btc from jmclient.configure import jm_single, get_p2pk_vbyte, get_p2sh_vbyte @@ -25,7 +26,6 @@ class Taker(object): wallet, schedule, order_chooser=weighted_order_choose, - sign_method=None, callbacks=None, tdestaddrs=None, ignored_makers=None): @@ -98,9 +98,6 @@ class Taker(object): self.schedule_index = -1 self.utxos = {} self.tdestaddrs = [] if not tdestaddrs else tdestaddrs - #allow custom wallet-based clients to use their own signing code; - #currently only setting "wallet" is allowed, calls wallet.sign_tx(tx) - self.sign_method = sign_method self.filter_orders_callback = callbacks[0] self.taker_info_callback = callbacks[1] if not self.taker_info_callback: @@ -367,7 +364,9 @@ class Taker(object): #Construct the Bitcoin address for the auth_pub field #Ensure that at least one address from utxos corresponds. input_addresses = [d['address'] for d in utxo_data] - auth_address = self.wallet.pubkey_to_address(auth_pub) + # FIXME: This only works if taker's commitment address is of same type + # as our wallet. + auth_address = self.wallet.pubkey_to_addr(unhexlify(auth_pub)) if not auth_address in input_addresses: jlog.warn("ERROR maker's (" + nick + ")" " authorising pubkey is not included " @@ -710,37 +709,23 @@ class Taker(object): #Note: donation code removed (possibly temporarily) raise NotImplementedError - def sign_tx(self, tx, i, priv, amount): - if self.my_cj_addr: - return self.wallet.sign(tx, i, priv, amount) - else: - #Note: donation code removed (possibly temporarily) - raise NotImplementedError - def self_sign(self): # now sign it ourselves - tx = btc.serialize(self.latest_tx) - if self.sign_method == "wallet": - #Currently passes addresses of to-be-signed inputs - #to backend wallet; this is correct for Electrum, may need - #different info for other backends. - addrs = {} - for index, ins in enumerate(self.latest_tx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - if utxo not in self.input_utxos.keys(): - continue - addrs[index] = self.input_utxos[utxo]['address'] - tx = self.wallet.sign_tx(tx, addrs) - else: - for index, ins in enumerate(self.latest_tx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - if utxo not in self.input_utxos.keys(): - continue - addr = self.input_utxos[utxo]['address'] - amount = self.input_utxos[utxo]["value"] - tx = self.sign_tx(tx, index, self.wallet.get_key_from_addr(addr), - amount) - self.latest_tx = btc.deserialize(tx) + our_inputs = {} + for index, ins in enumerate(self.latest_tx['ins']): + utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + if utxo not in self.input_utxos.keys(): + continue + script = self.wallet.addr_to_script(self.input_utxos[utxo]['address']) + amount = self.input_utxos[utxo]['value'] + our_inputs[index] = (script, amount) + + # FIXME: ugly hack + tx_bin = btc.deserialize(unhexlify(btc.serialize(self.latest_tx))) + self.wallet.sign_tx(tx_bin, our_inputs) + + self.latest_tx = btc.deserialize(hexlify(btc.serialize(tx_bin))) + def push(self): tx = btc.serialize(self.latest_tx) diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index 32e8a8d..25eb295 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -5,10 +5,11 @@ import pprint import os import time import numbers +from binascii import hexlify, unhexlify from .configure import get_log, jm_single, validate_address from .schedule import human_readable_schedule_entry, tweak_tumble_schedule from .wallet import BaseWallet, estimate_tx_fee -from jmclient import mktx, deserialize, sign, txhash +from .btc import mktx, serialize, deserialize, sign, txhash log = get_log() """ @@ -74,14 +75,7 @@ def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False, log.info("Using a fee of : " + str(fee_est) + " satoshis.") if amount != 0: log.info("Using a change value of: " + str(changeval) + " satoshis.") - tx = mktx(utxos.keys(), outs) - stx = deserialize(tx) - for index, ins in enumerate(stx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str( - ins['outpoint']['index']) - addr = utxos[utxo]['address'] - amount = utxos[utxo]['value'] - tx = sign(tx, index, wallet.get_key_from_addr(addr), amount=amount) + tx = sign_tx(wallet, mktx(utxos.keys(), outs), utxos) txsigned = deserialize(tx) log.info("Got signed transaction:\n") log.info(tx + "\n") @@ -106,6 +100,21 @@ def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False, return txid +def sign_tx(wallet, tx, utxos): + stx = deserialize(tx) + our_inputs = {} + for index, ins in enumerate(stx['ins']): + utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + script = wallet.addr_to_script(utxos[utxo]['address']) + amount = utxos[utxo]['value'] + our_inputs[index] = (script, amount) + + # FIXME: ugly hack + tx_bin = deserialize(unhexlify(serialize(stx))) + wallet.sign_tx(tx_bin, our_inputs) + return hexlify(serialize(tx_bin)) + + def import_new_addresses(wallet, addr_list): # FIXME: same code as in maker.py and taker.py bci = jm_single().bc_interface diff --git a/jmclient/test/test_maker.py b/jmclient/test/test_maker.py index 6cdfbf7..4a1c823 100644 --- a/jmclient/test/test_maker.py +++ b/jmclient/test/test_maker.py @@ -2,10 +2,11 @@ from __future__ import print_function -from jmclient import AbstractWallet, Maker, btc, get_p2sh_vbyte, get_p2pk_vbyte, \ +from jmclient import Maker, btc, get_p2sh_vbyte, get_p2pk_vbyte, \ load_program_config, jm_single import jmclient from commontest import DummyBlockchainInterface +from test_taker import DummyWallet import struct import binascii @@ -13,10 +14,6 @@ from itertools import chain import pytest -class MockWallet(AbstractWallet): - pass - - class OfflineMaker(Maker): def try_to_create_my_orders(self): self.sync_wait_loop.stop() @@ -116,7 +113,7 @@ def test_verify_unsigned_tx_sw_valid(setup_env_nodeps): p2sh_gen = address_p2sh_generator() p2pkh_gen = address_p2pkh_generator() - wallet = MockWallet() + wallet = DummyWallet() maker = OfflineMaker(wallet) cj_addr, cj_script = next(p2sh_gen) @@ -149,7 +146,7 @@ def test_verify_unsigned_tx_nonsw_valid(setup_env_nodeps): p2sh_gen = address_p2sh_generator() p2pkh_gen = address_p2pkh_generator() - wallet = MockWallet() + wallet = DummyWallet() maker = OfflineMaker(wallet) cj_addr, cj_script = next(p2pkh_gen) diff --git a/jmclient/test/test_taker.py b/jmclient/test/test_taker.py index 4d9310a..585358b 100644 --- a/jmclient/test/test_taker.py +++ b/jmclient/test/test_taker.py @@ -11,12 +11,12 @@ import json from base64 import b64encode from jmclient import ( load_program_config, jm_single, set_commitment_file, get_commitment_file, - LegacyWallet, Taker, VolatileStorage, get_p2sh_vbyte, get_network) + SegwitLegacyWallet, Taker, VolatileStorage, get_p2sh_vbyte, get_network) from taker_test_data import (t_utxos_by_mixdepth, t_selected_utxos, t_orderbook, t_maker_response, t_chosen_orders, t_dummy_ext) -class DummyWallet(LegacyWallet): +class DummyWallet(SegwitLegacyWallet): def __init__(self): storage = VolatileStorage() super(DummyWallet, self).initialize(storage, get_network(), @@ -79,10 +79,6 @@ class DummyWallet(LegacyWallet): """ return 'p2sh-p2wpkh' - @classmethod - def pubkey_to_address(cls, pubkey): - return SegwitWallet.pubkey_to_address(pubkey) - def get_key_from_addr(self, addr): """usable addresses: privkey all 1s, 2s, 3s, ... :""" privs = [x*32 + "\x01" for x in [chr(y) for y in range(1,6)]] @@ -111,7 +107,7 @@ def dummy_filter_orderbook(orders_fees, cjamount): print("calling dummy filter orderbook") return True -def get_taker(schedule=None, schedule_len=0, sign_method=None, on_finished=None, +def get_taker(schedule=None, schedule_len=0, on_finished=None, filter_orders=None): if not schedule: #note, for taker.initalize() this will result in junk @@ -120,8 +116,7 @@ def get_taker(schedule=None, schedule_len=0, sign_method=None, on_finished=None, on_finished_callback = on_finished if on_finished else taker_finished filter_orders_callback = filter_orders if filter_orders else dummy_filter_orderbook return Taker(DummyWallet(), schedule, - callbacks=[filter_orders_callback, None, on_finished_callback], - sign_method=sign_method) + callbacks=[filter_orders_callback, None, on_finished_callback]) def test_filter_rejection(createcmtdata): def filter_orders_reject(orders_feesl, cjamount): @@ -340,8 +335,6 @@ def test_taker_init(createcmtdata, schedule, highfee, toomuchcoins, minmakers, taker.my_cj_addr = None with pytest.raises(NotImplementedError) as e_info: taker.prepare_my_bitcoin_data() - with pytest.raises(NotImplementedError) as e_info: - taker.sign_tx("a", "b", "c", "d") with pytest.raises(NotImplementedError) as e_info: a = taker.coinjoin_address() taker.wallet.inject_addr_get_failure = True @@ -377,14 +370,12 @@ def test_unconfirm_confirm(schedule_len): assert not test_unconfirm_confirm.txflag @pytest.mark.parametrize( - "dummyaddr, signmethod, schedule", + "dummyaddr, schedule", [ - ("mrcNu71ztWjAQA6ww9kHiW3zBWSQidHXTQ", None, - [(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)]), - ("mrcNu71ztWjAQA6ww9kHiW3zBWSQidHXTQ", "wallet", - [(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)]), + ("mrcNu71ztWjAQA6ww9kHiW3zBWSQidHXTQ", + [(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)]) ]) -def test_on_sig(createcmtdata, dummyaddr, signmethod, schedule): +def test_on_sig(createcmtdata, dummyaddr, schedule): #plan: create a new transaction with known inputs and dummy outputs; #then, create a signature with various inputs, pass in in b64 to on_sig. #in order for it to verify, the DummyBlockchainInterface will have to @@ -410,7 +401,7 @@ def test_on_sig(createcmtdata, dummyaddr, signmethod, schedule): de_tx = bitcoin.deserialize(tx) #prepare the Taker with the right intermediate data - taker = get_taker(schedule=schedule, sign_method=signmethod) + taker = get_taker(schedule=schedule) taker.nonrespondants=["cp1", "cp2", "cp3"] taker.latest_tx = de_tx #my inputs are the first 2 utxos From 98f41f7c052f9f6adf46b21dbeac1341bb6bbf8f Mon Sep 17 00:00:00 2001 From: undeath Date: Tue, 7 Aug 2018 15:37:51 +0200 Subject: [PATCH 18/23] make SimpleLruCache an actual LRU cache --- jmclient/jmclient/cryptoengine.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/jmclient/jmclient/cryptoengine.py b/jmclient/jmclient/cryptoengine.py index 88de28e..693dd5f 100644 --- a/jmclient/jmclient/cryptoengine.py +++ b/jmclient/jmclient/cryptoengine.py @@ -75,6 +75,12 @@ class SimpleLruCache(OrderedDict): OrderedDict.__setitem__(self, key, value) self._adjust_size() + def __getitem__(self, item): + e = OrderedDict.__getitem__(self, item) + del self[item] + OrderedDict.__setitem__(self, item, e) + return e + def _adjust_size(self): while len(self) > self.max_size: self.popitem(last=False) @@ -156,8 +162,7 @@ class BTCEngine(object): def _walk_bip32_path(cls, master_key, path): key = master_key for lvl in path[1:]: - assert lvl >= 0 - assert lvl < 2**32 + assert 0 <= lvl < 2**32 if (key, lvl) in cls.__LRU_KEY_CACHE: key = cls.__LRU_KEY_CACHE[(key, lvl)] else: From 9dd1dc7876e9e6087655ffd0eed18a9129a05ff2 Mon Sep 17 00:00:00 2001 From: undeath Date: Tue, 7 Aug 2018 16:11:09 +0200 Subject: [PATCH 19/23] fix wallet sync in fast mode --- jmclient/jmclient/blockchaininterface.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/jmclient/jmclient/blockchaininterface.py b/jmclient/jmclient/blockchaininterface.py index d5d7f8e..565eacc 100644 --- a/jmclient/jmclient/blockchaininterface.py +++ b/jmclient/jmclient/blockchaininterface.py @@ -6,6 +6,7 @@ import random import sys import time import binascii +from copy import deepcopy from decimal import Decimal from twisted.internet import reactor, task @@ -463,13 +464,22 @@ class BitcoinCoreInterface(BlockchainInterface): BATCH_SIZE = 100 MAX_ITERATIONS = 20 + current_indices = deepcopy(saved_indices) for j in range(MAX_ITERATIONS): if not remaining_used_addresses: break for addr in \ self._collect_addresses_gap(wallet, gap_limit=BATCH_SIZE): remaining_used_addresses.discard(addr) + + # increase wallet indices for next iteration + for md in current_indices: + current_indices[md][0] += BATCH_SIZE + current_indices[md][1] += BATCH_SIZE + self._rewind_wallet_indices(wallet, current_indices, + current_indices) else: + self._rewind_wallet_indices(wallet, saved_indices, saved_indices) raise Exception("Failed to sync in fast mode after 20 batches; " "please re-try wallet sync without --fast flag.") From aa2c1d961b8458c1c524c74be266fcf30dc6ae00 Mon Sep 17 00:00:00 2001 From: undeath Date: Wed, 8 Aug 2018 13:19:49 +0200 Subject: [PATCH 20/23] fix some bugs in wallet_utils --- jmclient/jmclient/wallet_utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/jmclient/jmclient/wallet_utils.py b/jmclient/jmclient/wallet_utils.py index 7b4319e..4d0c9ad 100644 --- a/jmclient/jmclient/wallet_utils.py +++ b/jmclient/jmclient/wallet_utils.py @@ -342,14 +342,13 @@ def wallet_display(wallet, gaplimit, showprivkey, displayall=False, then return its serialization directly if serialized, else return the WalletView object. """ - wallet.close() acctlist = [] for m in xrange(wallet.max_mixdepth + 1): branchlist = [] for forchange in [0, 1]: entrylist = [] - # FIXME: why does this if/else exist? if forchange == 0: + # users would only want to hand out the xpub for externals xpub_key = wallet.get_bip32_pub_export(m, forchange) else: xpub_key = "" @@ -372,6 +371,7 @@ def wallet_display(wallet, gaplimit, showprivkey, displayall=False, entrylist.append(WalletViewEntry( wallet.get_path_repr(path), m, forchange, k, addr, [balance, balance], priv=privkey, used=used)) + wallet.set_next_index(m, forchange, unused_index) path = wallet.get_path_repr(wallet.get_path(m, forchange)) branchlist.append(WalletViewBranch(path, m, forchange, entrylist, xpub=xpub_key)) @@ -884,8 +884,9 @@ def open_test_wallet_maybe(path, seed, max_mixdepth, test_wallet_cls.initialize( storage, get_network(), max_mixdepth=max_mixdepth, entropy=seed) - assert 'ask_for_password' not in kwargs - assert 'read_only' not in kwargs + assert 'ask_for_password' not in kwargs or\ + not kwargs['ask_for_password'] + assert 'read_only' not in kwargs or not kwargs['read_only'] return test_wallet_cls(storage, **kwargs) return open_wallet(path, **kwargs) From a929cf304db81eb7ebd9ef7a2e10dadeaa3287b8 Mon Sep 17 00:00:00 2001 From: undeath Date: Wed, 8 Aug 2018 21:44:52 +0200 Subject: [PATCH 21/23] make log output human-readable again --- jmclient/jmclient/__init__.py | 4 +- jmclient/jmclient/client_protocol.py | 7 +-- jmclient/jmclient/maker.py | 5 +- jmclient/jmclient/output.py | 69 ++++++++++++++++++++++++++++ jmclient/jmclient/podle.py | 43 ----------------- jmclient/jmclient/taker.py | 7 +-- jmclient/test/test_podle.py | 7 +-- jmclient/test/test_taker.py | 9 +++- 8 files changed, 94 insertions(+), 57 deletions(-) create mode 100644 jmclient/jmclient/output.py diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index d6dc3da..dd57eec 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -33,9 +33,11 @@ from .electruminterface import ElectrumInterface from .client_protocol import (JMTakerClientProtocol, JMClientProtocolFactory, start_reactor) from .podle import (set_commitment_file, get_commitment_file, - generate_podle_error_string, add_external_commitments, + add_external_commitments, PoDLE, generate_podle, get_podle_commitments, update_commitments) +from .output import generate_podle_error_string, fmt_utxos, fmt_utxo,\ + fmt_tx_data from .schedule import (get_schedule, get_tumble_schedule, schedule_to_text, tweak_tumble_schedule, human_readable_schedule_entry, schedule_to_text) diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index 32abe7c..06418af 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -15,9 +15,9 @@ import json import hashlib import os import sys -import pprint from jmclient import (jm_single, get_irc_mchannels, get_log, get_p2sh_vbyte, RegtestBitcoinCoreInterface) +from .output import fmt_tx_data from jmbase import _byteify import btc @@ -240,8 +240,9 @@ class JMMakerClientProtocol(JMClientProtocol): jlog.info("Failed to find notified unconfirmed transaction: " + txid) return removed_utxos = self.client.wallet.remove_old_utxos(txd) - jlog.info('saw tx on network, removed_utxos=\n{}'.format( - pprint.pformat(removed_utxos))) + jlog.info('saw tx on network, removed_utxos=\n{}'.format('\n'.join( + '{} - {}'.format(u, fmt_tx_data(tx_data, self.client.wallet)) + for u, tx_data in removed_utxos.items()))) to_cancel, to_announce = self.client.on_tx_unconfirmed(offerinfo, txid, removed_utxos) self.client.modify_orders(to_cancel, to_announce) diff --git a/jmclient/jmclient/maker.py b/jmclient/jmclient/maker.py index 9690d12..a697b3f 100644 --- a/jmclient/jmclient/maker.py +++ b/jmclient/jmclient/maker.py @@ -7,11 +7,10 @@ import sys from binascii import unhexlify import btc -from jmclient.configure import jm_single, get_p2pk_vbyte, get_p2sh_vbyte +from jmclient.configure import jm_single from jmbase.support import get_log from jmclient.support import (calc_cj_fee) -from jmclient.podle import (generate_podle, get_podle_commitments, verify_podle, - PoDLE, PoDLEError, generate_podle_error_string) +from jmclient.podle import verify_podle, PoDLE from twisted.internet import task jlog = get_log() diff --git a/jmclient/jmclient/output.py b/jmclient/jmclient/output.py new file mode 100644 index 0000000..7393064 --- /dev/null +++ b/jmclient/jmclient/output.py @@ -0,0 +1,69 @@ +from binascii import hexlify + + +def fmt_utxos(utxos, wallet, prefix=''): + output = [] + for u in utxos: + utxo_str = '{}{} - {}'.format( + prefix, fmt_utxo(u), fmt_tx_data(utxos[u], wallet)) + output.append(utxo_str) + return '\n'.join(output) + + +def fmt_utxo(utxo): + return '{}:{}'.format(hexlify(utxo[0]), utxo[1]) + + +def fmt_tx_data(tx_data, wallet): + return 'path: {}, address: {}, value: {}'.format( + wallet.get_path_repr(wallet.script_to_path(tx_data['script'])), + wallet.script_to_addr(tx_data['script']), tx_data['value']) + + +def generate_podle_error_string(priv_utxo_pairs, to, ts, wallet, cjamount, + taker_utxo_age, taker_utxo_amtpercent): + """Gives detailed error information on why commitment sourcing failed. + """ + errmsg = "" + errmsgheader = ("Failed to source a commitment; this debugging information" + " may help:\n\n") + errmsg += ("1: Utxos that passed age and size limits, but have " + "been used too many times (see taker_utxo_retries " + "in the config):\n") + if len(priv_utxo_pairs) == 0: + errmsg += ("None\n") + else: + for p, u in priv_utxo_pairs: + errmsg += (str(u) + "\n") + errmsg += "2: Utxos that have less than " + taker_utxo_age + " confirmations:\n" + if len(to) == 0: + errmsg += ("None\n") + else: + for t in to: + errmsg += (str(t) + "\n") + errmsg += ("3: Utxos that were not at least " + taker_utxo_amtpercent + \ + "% of the size of the coinjoin amount " + str(cjamount) + "\n") + if len(ts) == 0: + errmsg += ("None\n") + else: + for t in ts: + errmsg += (str(t) + "\n") + errmsg += ('***\n') + errmsg += ("Utxos that appeared in item 1 cannot be used again.\n") + errmsg += ("Utxos only in item 2 can be used by waiting for more " + "confirmations, (set by the value of taker_utxo_age).\n") + errmsg += ("Utxos only in item 3 are not big enough for this " + "coinjoin transaction, set by the value " + "of taker_utxo_amtpercent.\n") + errmsg += ("If you cannot source a utxo from your wallet according " + "to these rules, use the tool add-utxo.py to source a " + "utxo external to your joinmarket wallet. Read the help " + "with 'python add-utxo.py --help'\n\n") + errmsg += ("***\nFor reference, here are the utxos in your wallet:\n") + + for md, utxos in wallet.get_utxos_by_mixdepth_().items(): + if not utxos: + continue + errmsg += ("\nmixdepth {}:\n{}".format( + md, fmt_utxos(utxos, wallet, prefix=' '))) + return (errmsgheader, errmsg) diff --git a/jmclient/jmclient/podle.py b/jmclient/jmclient/podle.py index 4170bfd..0aa658e 100644 --- a/jmclient/jmclient/podle.py +++ b/jmclient/jmclient/podle.py @@ -21,49 +21,6 @@ def get_commitment_file(): return PODLE_COMMIT_FILE -def generate_podle_error_string(priv_utxo_pairs, to, ts, unspent, cjamount, - taker_utxo_age, taker_utxo_amtpercent): - """Gives detailed error information on why commitment sourcing failed. - """ - errmsg = "" - errmsgheader = ("Failed to source a commitment; this debugging information" - " may help:\n\n") - errmsg += ("1: Utxos that passed age and size limits, but have " - "been used too many times (see taker_utxo_retries " - "in the config):\n") - if len(priv_utxo_pairs) == 0: - errmsg += ("None\n") - else: - for p, u in priv_utxo_pairs: - errmsg += (str(u) + "\n") - errmsg += "2: Utxos that have less than " + taker_utxo_age + " confirmations:\n" - if len(to) == 0: - errmsg += ("None\n") - else: - for t in to: - errmsg += (str(t) + "\n") - errmsg += ("3: Utxos that were not at least " + taker_utxo_amtpercent + \ - "% of the size of the coinjoin amount " + str(cjamount) + "\n") - if len(ts) == 0: - errmsg += ("None\n") - else: - for t in ts: - errmsg += (str(t) + "\n") - errmsg += ('***\n') - errmsg += ("Utxos that appeared in item 1 cannot be used again.\n") - errmsg += ("Utxos only in item 2 can be used by waiting for more " - "confirmations, (set by the value of taker_utxo_age).\n") - errmsg += ("Utxos only in item 3 are not big enough for this " - "coinjoin transaction, set by the value " - "of taker_utxo_amtpercent.\n") - errmsg += ("If you cannot source a utxo from your wallet according " - "to these rules, use the tool add-utxo.py to source a " - "utxo external to your joinmarket wallet. Read the help " - "with 'python add-utxo.py --help'\n\n") - errmsg += ("***\nFor reference, here are the utxos in your wallet:\n") - errmsg += ("\n" + str(unspent)) - return (errmsgheader, errmsg) - class PoDLEError(Exception): pass diff --git a/jmclient/jmclient/taker.py b/jmclient/jmclient/taker.py index dd75722..bc06f54 100644 --- a/jmclient/jmclient/taker.py +++ b/jmclient/jmclient/taker.py @@ -12,8 +12,9 @@ from jmbase.support import get_log from jmclient.support import (calc_cj_fee, weighted_order_choose, choose_orders, choose_sweep_orders) from jmclient.wallet import estimate_tx_fee -from jmclient.podle import (generate_podle, get_podle_commitments, - PoDLE, PoDLEError, generate_podle_error_string) +from jmclient.podle import generate_podle, get_podle_commitments, PoDLE +from .output import generate_podle_error_string + jlog = get_log() @@ -690,7 +691,7 @@ class Taker(object): "Commitment sourced OK") else: errmsgheader, errmsg = generate_podle_error_string(priv_utxo_pairs, - to, ts, self.wallet.get_utxos_by_mixdepth(), self.cjamount, + to, ts, self.wallet, self.cjamount, jm_single().config.get("POLICY", "taker_utxo_age"), jm_single().config.get("POLICY", "taker_utxo_amtpercent")) diff --git a/jmclient/test/test_podle.py b/jmclient/test/test_podle.py index 7cf7a2c..6eaee75 100644 --- a/jmclient/test/test_podle.py +++ b/jmclient/test/test_podle.py @@ -12,6 +12,7 @@ from jmclient import (load_program_config, get_log, jm_single, generate_podle, get_commitment_file, PoDLE, get_podle_commitments, add_external_commitments, update_commitments) from jmclient.podle import verify_all_NUMS, verify_podle, PoDLEError +from commontest import make_wallets log = get_log() def test_commitments_empty(setup_podle): @@ -192,14 +193,14 @@ def test_podle_error_string(setup_podle): ('fakepriv2', 'fakeutxo2')] to = ['tooold1', 'tooold2'] ts = ['toosmall1', 'toosmall2'] - unspent = "dummyunspent" + wallet = make_wallets(1, [[1, 0, 0, 0, 0]])[0]['wallet'] cjamt = 100 tua = "3" tuamtper = "20" errmgsheader, errmsg = generate_podle_error_string(priv_utxo_pairs, to, ts, - unspent, + wallet, cjamt, tua, tuamtper) @@ -208,7 +209,7 @@ def test_podle_error_string(setup_podle): y = [x[1] for x in priv_utxo_pairs] assert all([errmsg.find(x) != -1 for x in to + ts + y]) #ensure OK with nothing - errmgsheader, errmsg = generate_podle_error_string([], [], [], unspent, + errmgsheader, errmsg = generate_podle_error_string([], [], [], wallet, cjamt, tua, tuamtper) @pytest.fixture(scope="module") diff --git a/jmclient/test/test_taker.py b/jmclient/test/test_taker.py index 585358b..e49e269 100644 --- a/jmclient/test/test_taker.py +++ b/jmclient/test/test_taker.py @@ -29,8 +29,11 @@ class DummyWallet(SegwitLegacyWallet): for md, utxo in t_utxos_by_mixdepth.items(): for i, (txid, data) in enumerate(utxo.items()): txid, index = txid.split(':') + path = (b'dummy', md, i) self._utxos.add_utxo(binascii.unhexlify(txid), int(index), - (b'dummy', md, i), data['value'], md) + path, data['value'], md) + script = self._ENGINE.address_to_script(data['address']) + self._script_map[script] = path def get_utxos_by_mixdepth(self, verbose=True): return t_utxos_by_mixdepth @@ -97,6 +100,10 @@ class DummyWallet(SegwitLegacyWallet): return binascii.hexlify(p) raise ValueError("No such keypair") + def _is_my_bip32_path(self, path): + return True + + def dummy_order_chooser(): return t_chosen_orders From 8885e613d71a51ebd36bd4a1302960106bfdba4f Mon Sep 17 00:00:00 2001 From: undeath Date: Thu, 9 Aug 2018 17:16:09 +0200 Subject: [PATCH 22/23] revert bad assert fix --- jmclient/jmclient/wallet_utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/jmclient/jmclient/wallet_utils.py b/jmclient/jmclient/wallet_utils.py index 4d0c9ad..053eba6 100644 --- a/jmclient/jmclient/wallet_utils.py +++ b/jmclient/jmclient/wallet_utils.py @@ -884,9 +884,8 @@ def open_test_wallet_maybe(path, seed, max_mixdepth, test_wallet_cls.initialize( storage, get_network(), max_mixdepth=max_mixdepth, entropy=seed) - assert 'ask_for_password' not in kwargs or\ - not kwargs['ask_for_password'] - assert 'read_only' not in kwargs or not kwargs['read_only'] + assert 'ask_for_password' not in kwargs + assert 'read_only' not in kwargs return test_wallet_cls(storage, **kwargs) return open_wallet(path, **kwargs) From a0c1d5afd1fa80cd01c2ba4c1b2b44dc6df52a08 Mon Sep 17 00:00:00 2001 From: undeath Date: Thu, 9 Aug 2018 17:19:43 +0200 Subject: [PATCH 23/23] add upgrade notes --- docs/release-notes/release-notes-future.md | 29 ++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 docs/release-notes/release-notes-future.md diff --git a/docs/release-notes/release-notes-future.md b/docs/release-notes/release-notes-future.md new file mode 100644 index 0000000..695c3b2 --- /dev/null +++ b/docs/release-notes/release-notes-future.md @@ -0,0 +1,29 @@ +Joinmarket-clientserver future: +=============================== + + +Upgrading +========= + +To upgrade: run the `install.sh` script as mentioned in the README. When prompted to overwrite the directory `jmvenv`, accept. + +A new wallet format has been introduced. Old wallets require conversion. In order to convert your existing wallet to the new format you can use the included conversion tool at `scripts/convert_old_wallet.py`. + +usage: + + python convert_old_wallet.py full/path/to/wallets/wallet.json + +This will place the newly converted `wallet.jmdat` file in the existing joinmarket `wallets/` directory. The wallet name will be adopted accordingly if it differs from `wallet`. + + +Notable changes +=============== + + +Credits +======= + +Thanks to everyone who directly contributed to this release - + + +And thanks also to those who submitted bug reports, tested and otherwise helped out.