diff --git a/jmbase/jmbase/__init__.py b/jmbase/jmbase/__init__.py index ad3931e..cdbbedb 100644 --- a/jmbase/jmbase/__init__.py +++ b/jmbase/jmbase/__init__.py @@ -2,7 +2,10 @@ from .support import (get_log, chunks, debug_silence, jmprint, joinmarket_alert, core_alert, get_password, set_logging_level, set_logging_color, - lookup_appdata_folder, - JM_WALLET_NAME_PREFIX, JM_APP_NAME) + lookup_appdata_folder, bintohex, bintolehex, + hextobin, lehextobin, utxostr_to_utxo, + utxo_to_utxostr, EXIT_ARGERROR, EXIT_FAILURE, + EXIT_SUCCESS, hexbin, dictchanger, listchanger, + cv, JM_WALLET_NAME_PREFIX, JM_APP_NAME) from .commands import * diff --git a/jmbase/jmbase/support.py b/jmbase/jmbase/support.py index 7175bb5..6afad0a 100644 --- a/jmbase/jmbase/support.py +++ b/jmbase/jmbase/support.py @@ -1,8 +1,9 @@ import logging, sys +import binascii from getpass import getpass from os import path, environ - +from functools import wraps # JoinMarket version JM_CORE_VERSION = '0.7.0dev' @@ -80,6 +81,71 @@ handler = JoinMarketStreamHandler() handler.setFormatter(logFormatter) log.addHandler(handler) +# hex/binary conversion routines used by dependent packages +def hextobin(h): + """Convert a hex string to bytes""" + return binascii.unhexlify(h.encode('utf8')) + + +def bintohex(b): + """Convert bytes to a hex string""" + return binascii.hexlify(b).decode('utf8') + + +def lehextobin(h): + """Convert a little-endian hex string to bytes + + Lets you write uint256's and uint160's the way the Satoshi codebase shows + them. + """ + return binascii.unhexlify(h.encode('utf8'))[::-1] + + +def bintolehex(b): + """Convert bytes to a little-endian hex string + + Lets you show uint256's and uint160's the way the Satoshi codebase shows + them. + """ + return binascii.hexlify(b[::-1]).decode('utf8') + +def utxostr_to_utxo(x): + if not isinstance(x, str): + return (False, "not a string") + y = x.split(":") + if len(y) != 2: + return (False, + "string is not two items separated by :") + try: + n = int(y[1]) + except: + return (False, "utxo index was not an integer.") + if n < 0: + return (False, "utxo index must not be negative.") + if len(y[0]) != 64: + return (False, "txid is not 64 hex characters.") + try: + txid = binascii.unhexlify(y[0]) + except: + return (False, "txid is not hex.") + return (True, (txid, n)) + +def utxo_to_utxostr(u): + if not isinstance(u, tuple): + return (False, "utxo is not a tuple.") + if not len(u) == 2: + return (False, "utxo should have two elements.") + if not isinstance(u[0], bytes): + return (False, "txid should be bytes.") + if not isinstance(u[1], int): + return (False, "index should be int.") + if u[1] < 0: + return (False, "index must be a positive integer.") + if not len(u[0]) == 32: + return (False, "txid must be 32 bytes.") + txid = binascii.hexlify(u[0]).decode("ascii") + return (True, txid + ":" + str(u[1])) + def jmprint(msg, level="info"): """ Provides the ability to print messages with consistent formatting, outside the logging system @@ -150,3 +216,67 @@ def lookup_appdata_folder(appname): def print_jm_version(option, opt_str, value, parser): print("JoinMarket " + JM_CORE_VERSION) sys.exit(EXIT_SUCCESS) + +# helper functions for conversions of format between over-the-wire JM +# and internal. See details in hexbin() docstring. + +def cv(x): + success, utxo = utxostr_to_utxo(x) + if success: + return utxo + else: + try: + b = hextobin(x) + return b + except: + return x + +def listchanger(l): + rlist = [] + for x in l: + if isinstance(x, list): + rlist.append(listchanger(x)) + elif isinstance(x, dict): + rlist.append(dictchanger(x)) + else: + rlist.append(cv(x)) + return rlist + +def dictchanger(d): + rdict = {} + for k, v in d.items(): + if isinstance(v, dict): + rdict[cv(k)] = dictchanger(v) + elif isinstance(v, list): + rdict[cv(k)] = listchanger(v) + else: + rdict[cv(k)] = cv(v) + return rdict + +def hexbin(func): + """ Decorator for functions of taker and maker receiving over + the wire AMP arguments that may be in hex or hextxid:n format + and converting all to binary. + Functions to which this decorator applies should have all arguments + be one of: + - hex string (keys), converted here to binary + - lists of keys or txid:n strings (converted here to binary, or + (txidbytes, n)) + - lists of lists or dicts, to which these rules apply recursively. + - any other string (unchanged) + - dicts with keys as per above; values are altered recursively according + to the rules above. + """ + @wraps(func) + def func_wrapper(inst, *args, **kwargs): + newargs = [] + for arg in args: + if isinstance(arg, (list, tuple)): + newargs.append(listchanger(arg)) + elif isinstance(arg, dict): + newargs.append(dictchanger(arg)) + else: + newargs.append(cv(arg)) + return func(inst, *newargs, **kwargs) + + return func_wrapper \ No newline at end of file diff --git a/jmbitcoin/jmbitcoin/__init__.py b/jmbitcoin/jmbitcoin/__init__.py index 784673a..00fc519 100644 --- a/jmbitcoin/jmbitcoin/__init__.py +++ b/jmbitcoin/jmbitcoin/__init__.py @@ -2,8 +2,14 @@ import coincurve as secp256k1 from jmbitcoin.secp256k1_main import * from jmbitcoin.secp256k1_transaction import * from jmbitcoin.secp256k1_deterministic import * -from jmbitcoin.btscript import * -from jmbitcoin.bech32 import * from jmbitcoin.amount import * from jmbitcoin.bip21 import * +from bitcointx import select_chain_params +from bitcointx.core import (x, b2x, b2lx, lx, COutPoint, CTxOut, CTxIn, + CTxInWitness, CTxWitness, CMutableTransaction, + Hash160, coins_to_satoshi, satoshi_to_coins) +from bitcointx.core.script import (CScript, OP_0, SignatureHash, SIGHASH_ALL, + SIGVERSION_WITNESS_V0, CScriptWitness) +from bitcointx.wallet import (CBitcoinSecret, P2WPKHBitcoinAddress, CCoinAddress, + P2SHCoinAddress) diff --git a/jmbitcoin/jmbitcoin/bech32.py b/jmbitcoin/jmbitcoin/bech32.py deleted file mode 100644 index 8d3f9bf..0000000 --- a/jmbitcoin/jmbitcoin/bech32.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright (c) 2017 Pieter Wuille -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -"""Reference implementation for Bech32 and segwit addresses.""" - -CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" - - -def bech32_polymod(values): - """Internal function that computes the Bech32 checksum.""" - generator = [0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3] - chk = 1 - for value in values: - top = chk >> 25 - chk = (chk & 0x1ffffff) << 5 ^ value - for i in range(5): - chk ^= generator[i] if ((top >> i) & 1) else 0 - return chk - - -def bech32_hrp_expand(hrp): - """Expand the HRP into values for checksum computation.""" - return [ord(x) >> 5 for x in hrp] + [0] + [ord(x) & 31 for x in hrp] - - -def bech32_verify_checksum(hrp, data): - """Verify a checksum given HRP and converted data characters.""" - return bech32_polymod(bech32_hrp_expand(hrp) + data) == 1 - - -def bech32_create_checksum(hrp, data): - """Compute the checksum values given HRP and data.""" - values = bech32_hrp_expand(hrp) + data - polymod = bech32_polymod(values + [0, 0, 0, 0, 0, 0]) ^ 1 - return [(polymod >> 5 * (5 - i)) & 31 for i in range(6)] - - -def bech32_encode(hrp, data): - """Compute a Bech32 string given HRP and data values.""" - combined = data + bech32_create_checksum(hrp, data) - return hrp + '1' + ''.join([CHARSET[d] for d in combined]) - - -def bech32_decode(bech): - """Validate a Bech32 string, and determine HRP and data.""" - if ((any(ord(x) < 33 or ord(x) > 126 for x in bech)) or - (bech.lower() != bech and bech.upper() != bech)): - return (None, None) - bech = bech.lower() - pos = bech.rfind('1') - if pos < 1 or pos + 7 > len(bech) or len(bech) > 90: - return (None, None) - if not all(x in CHARSET for x in bech[pos+1:]): - return (None, None) - hrp = bech[:pos] - data = [CHARSET.find(x) for x in bech[pos+1:]] - if not bech32_verify_checksum(hrp, data): - return (None, None) - return (hrp, data[:-6]) - - -def convertbits(data, frombits, tobits, pad=True): - """General power-of-2 base conversion.""" - acc = 0 - bits = 0 - ret = [] - maxv = (1 << tobits) - 1 - max_acc = (1 << (frombits + tobits - 1)) - 1 - for value in data: - if value < 0 or (value >> frombits): - return None - acc = ((acc << frombits) | value) & max_acc - bits += frombits - while bits >= tobits: - bits -= tobits - ret.append((acc >> bits) & maxv) - if pad: - if bits: - ret.append((acc << (tobits - bits)) & maxv) - elif bits >= frombits or ((acc << (tobits - bits)) & maxv): - return None - return ret - - -def bech32addr_decode(hrp, addr): - """Decode a segwit address.""" - hrpgot, data = bech32_decode(addr) - if hrpgot != hrp: - return (None, None) - decoded = convertbits(data[1:], 5, 8, False) - if decoded is None or len(decoded) < 2 or len(decoded) > 40: - return (None, None) - if data[0] > 16: - return (None, None) - if data[0] == 0 and len(decoded) != 20 and len(decoded) != 32: - return (None, None) - return (data[0], decoded) - - -def bech32addr_encode(hrp, witver, witprog): - """Encode a segwit address.""" - ret = bech32_encode(hrp, [witver] + convertbits(witprog, 8, 5)) - if bech32addr_decode(hrp, ret) == (None, None): - return None - return ret diff --git a/jmbitcoin/jmbitcoin/btscript.py b/jmbitcoin/jmbitcoin/btscript.py deleted file mode 100644 index c658c4e..0000000 --- a/jmbitcoin/jmbitcoin/btscript.py +++ /dev/null @@ -1,142 +0,0 @@ -#OP codes; disabled commented. - -# push value -OP_0 = 0x00 -OP_FALSE = OP_0 -OP_PUSHDATA1 = 0x4c -OP_PUSHDATA2 = 0x4d -OP_PUSHDATA4 = 0x4e -OP_1NEGATE = 0x4f -OP_RESERVED = 0x50 -OP_1 = 0x51 -OP_TRUE = OP_1 -OP_2 = 0x52 -OP_3 = 0x53 -OP_4 = 0x54 -OP_5 = 0x55 -OP_6 = 0x56 -OP_7 = 0x57 -OP_8 = 0x58 -OP_9 = 0x59 -OP_10 = 0x5a -OP_11 = 0x5b -OP_12 = 0x5c -OP_13 = 0x5d -OP_14 = 0x5e -OP_15 = 0x5f -OP_16 = 0x60 - -# control -OP_NOP = 0x61 -OP_VER = 0x62 -OP_IF = 0x63 -OP_NOTIF = 0x64 -#OP_VERIF = 0x65 -#OP_VERNOTIF = 0x66 -OP_ELSE = 0x67 -OP_ENDIF = 0x68 -OP_VERIFY = 0x69 -OP_RETURN = 0x6a - -# stack ops -OP_TOALTSTACK = 0x6b -OP_FROMALTSTACK = 0x6c -OP_2DROP = 0x6d -OP_2DUP = 0x6e -OP_3DUP = 0x6f -OP_2OVER = 0x70 -OP_2ROT = 0x71 -OP_2SWAP = 0x72 -OP_IFDUP = 0x73 -OP_DEPTH = 0x74 -OP_DROP = 0x75 -OP_DUP = 0x76 -OP_NIP = 0x77 -OP_OVER = 0x78 -OP_PICK = 0x79 -OP_ROLL = 0x7a -OP_ROT = 0x7b -OP_SWAP = 0x7c -OP_TUCK = 0x7d - -# splice ops -#OP_CAT = 0x7e -#OP_SUBSTR = 0x7f -#OP_LEFT = 0x80 -#OP_RIGHT = 0x81 -OP_SIZE = 0x82 - -# bit logic -#OP_INVERT = 0x83 -#OP_AND = 0x84 -#OP_OR = 0x85 -#OP_XOR = 0x86 -OP_EQUAL = 0x87 -OP_EQUALVERIFY = 0x88 -OP_RESERVED1 = 0x89 -OP_RESERVED2 = 0x8a - -# numeric -OP_1ADD = 0x8b -OP_1SUB = 0x8c -#OP_2MUL = 0x8d -#OP_2DIV = 0x8e -OP_NEGATE = 0x8f -OP_ABS = 0x90 -OP_NOT = 0x91 -OP_0NOTEQUAL = 0x92 - -OP_ADD = 0x93 -OP_SUB = 0x94 -#OP_MUL = 0x95 -#OP_DIV = 0x96 -#OP_MOD = 0x97 -#OP_LSHIFT = 0x98 -#OP_RSHIFT = 0x99 - -OP_BOOLAND = 0x9a -OP_BOOLOR = 0x9b -OP_NUMEQUAL = 0x9c -OP_NUMEQUALVERIFY = 0x9d -OP_NUMNOTEQUAL = 0x9e -OP_LESSTHAN = 0x9f -OP_GREATERTHAN = 0xa0 -OP_LESSTHANOREQUAL = 0xa1 -OP_GREATERTHANOREQUAL = 0xa2 -OP_MIN = 0xa3 -OP_MAX = 0xa4 - -OP_WITHIN = 0xa5 - -# crypto -OP_RIPEMD160 = 0xa6 -OP_SHA1 = 0xa7 -OP_SHA256 = 0xa8 -OP_HASH160 = 0xa9 -OP_HASH256 = 0xaa -OP_CODESEPARATOR = 0xab -OP_CHECKSIG = 0xac -OP_CHECKSIGVERIFY = 0xad -OP_CHECKMULTISIG = 0xae -OP_CHECKMULTISIGVERIFY = 0xaf - -# expansion -OP_NOP1 = 0xb0 -OP_NOP2 = 0xb1 -OP_CHECKLOCKTIMEVERIFY = OP_NOP2 -OP_NOP3 = 0xb2 -OP_NOP4 = 0xb3 -OP_NOP5 = 0xb4 -OP_NOP6 = 0xb5 -OP_NOP7 = 0xb6 -OP_NOP8 = 0xb7 -OP_NOP9 = 0xb8 -OP_NOP10 = 0xb9 - -# template matching params -OP_SMALLINTEGER = 0xfa -OP_PUBKEYS = 0xfb -OP_PUBKEYHASH = 0xfd -OP_PUBKEY = 0xfe - -OP_INVALIDOPCODE = 0xff diff --git a/jmbitcoin/jmbitcoin/secp256k1_deterministic.py b/jmbitcoin/jmbitcoin/secp256k1_deterministic.py index 1baad53..eb9023d 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_deterministic.py +++ b/jmbitcoin/jmbitcoin/secp256k1_deterministic.py @@ -2,6 +2,8 @@ from jmbitcoin.secp256k1_main import * import hmac import hashlib import struct +from bitcointx.core import Hash160, Hash +from bitcointx import base58 # Below code ASSUMES binary inputs and compressed pubkeys MAINNET_PRIVATE = b'\x04\x88\xAD\xE4' @@ -11,6 +13,8 @@ TESTNET_PUBLIC = b'\x04\x35\x87\xCF' PRIVATE = [MAINNET_PRIVATE, TESTNET_PRIVATE] PUBLIC = [MAINNET_PUBLIC, TESTNET_PUBLIC] +privtopub = privkey_to_pubkey + # BIP32 child key derivation def raw_bip32_ckd(rawtuple, i): @@ -19,25 +23,25 @@ def raw_bip32_ckd(rawtuple, i): if vbytes in PRIVATE: priv = key - pub = privtopub(key, False) + pub = privtopub(key) else: pub = key if i >= 2**31: if vbytes in PUBLIC: raise Exception("Can't do private derivation on public key!") - I = hmac.new(chaincode, b'\x00' + priv[:32] + encode(i, 256, 4), + I = hmac.new(chaincode, b'\x00' + priv[:32] + struct.pack(b'>L', i), hashlib.sha512).digest() else: - I = hmac.new(chaincode, pub + encode(i, 256, 4), + I = hmac.new(chaincode, pub + struct.pack(b'>L', i), hashlib.sha512).digest() if vbytes in PRIVATE: - newkey = add_privkeys(I[:32] + b'\x01', priv, False) - fingerprint = bin_hash160(privtopub(key, False))[:4] + newkey = add_privkeys(I[:32] + b'\x01', priv) + fingerprint = Hash160(privtopub(key))[:4] if vbytes in PUBLIC: - newkey = add_pubkeys([privtopub(I[:32] + b'\x01', False), key], False) - fingerprint = bin_hash160(key)[:4] + newkey = add_pubkeys([privtopub(I[:32] + b'\x01'), key]) + fingerprint = Hash160(key)[:4] return (vbytes, depth + 1, fingerprint, i, I[32:], newkey) @@ -47,18 +51,17 @@ def bip32_serialize(rawtuple): i = struct.pack(b'>L', i) chaincode = chaincode keydata = b'\x00' + key[:-1] if vbytes in PRIVATE else key - bindata = vbytes + from_int_to_byte( - depth % 256) + fingerprint + i + chaincode + keydata - return b58encode(bindata + bin_dbl_sha256(bindata)[:4]) + bindata = vbytes + struct.pack(b'B',depth % 256) + fingerprint + i + chaincode + keydata + return base58.encode(bindata + Hash(bindata)[:4]) def bip32_deserialize(data): - dbin = b58decode(data) - if bin_dbl_sha256(dbin[:-4])[:4] != dbin[-4:]: + dbin = base58.decode(data) + if Hash(dbin[:-4])[:4] != dbin[-4:]: raise Exception("Invalid checksum") vbytes = dbin[0:4] depth = dbin[4] fingerprint = dbin[5:9] - i = decode(dbin[9:13], 256) + i = struct.unpack(b'>L',dbin[9:13])[0] chaincode = dbin[13:45] key = dbin[46:78] + b'\x01' if vbytes in PRIVATE else dbin[45:78] return (vbytes, depth, fingerprint, i, chaincode, key) @@ -68,7 +71,7 @@ def raw_bip32_privtopub(rawtuple): if vbytes in PUBLIC: return rawtuple newvbytes = MAINNET_PUBLIC if vbytes == MAINNET_PRIVATE else TESTNET_PUBLIC - return (newvbytes, depth, fingerprint, i, chaincode, privtopub(key, False)) + return (newvbytes, depth, fingerprint, i, chaincode, privtopub(key)) def bip32_privtopub(data): return bip32_serialize(raw_bip32_privtopub(bip32_deserialize(data))) @@ -77,13 +80,12 @@ def bip32_ckd(data, i): return bip32_serialize(raw_bip32_ckd(bip32_deserialize(data), i)) def bip32_master_key(seed, vbytes=MAINNET_PRIVATE): - I = hmac.new( - from_string_to_bytes("Bitcoin seed"), seed, hashlib.sha512).digest() + I = hmac.new("Bitcoin seed".encode("utf-8"), seed, hashlib.sha512).digest() return bip32_serialize((vbytes, 0, b'\x00' * 4, 0, I[32:], I[:32] + b'\x01' )) def bip32_extract_key(data): - return safe_hexlify(bip32_deserialize(data)[-1]) + return bip32_deserialize(data)[-1] def bip32_descend(*args): if len(args) == 2: diff --git a/jmbitcoin/jmbitcoin/secp256k1_main.py b/jmbitcoin/jmbitcoin/secp256k1_main.py index e91fc59..db0e7cb 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_main.py +++ b/jmbitcoin/jmbitcoin/secp256k1_main.py @@ -7,213 +7,16 @@ import base64 import struct import coincurve as secp256k1 +from bitcointx import base58 +from bitcointx.core import Hash +from bitcointx.signmessage import BitcoinMessage + #Required only for PoDLE calculation: N = 115792089237316195423570985008687907852837564279074904382605163141518161494337 BTC_P2PK_VBYTE = {"mainnet": b'\x00', "testnet": b'\x6f', "regtest": 100} BTC_P2SH_VBYTE = {"mainnet": b'\x05', "testnet": b'\xc4'} -#Standard prefix for Bitcoin message signing. -BITCOIN_MESSAGE_MAGIC = b'\x18' + b'Bitcoin Signed Message:\n' - -string_types = (str) -string_or_bytes_types = (str, bytes) -int_types = (int, float) - -# Base switching -code_strings = { - 2: '01', - 10: '0123456789', - 16: '0123456789abcdef', - 32: 'abcdefghijklmnopqrstuvwxyz234567', - 58: '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz', - 256: ''.join([chr(x) for x in range(256)]) -} - -def lpad(msg, symbol, length): - if len(msg) >= length: - return msg - return symbol * (length - len(msg)) + msg - -def get_code_string(base): - if base in code_strings: - return code_strings[base] - else: - raise ValueError("Invalid base!") - -def bin_to_b58check(inp, magicbyte=b'\x00'): - if not isinstance(magicbyte, int): - magicbyte = struct.unpack(b'B', magicbyte)[0] - assert(0 <= magicbyte <= 0xff) - if magicbyte == 0: - inp_fmtd = struct.pack(b'B', magicbyte) + inp - while magicbyte > 0: - inp_fmtd = struct.pack(b'B', magicbyte % 256) + inp - magicbyte //= 256 - checksum = bin_dbl_sha256(inp_fmtd)[:4] - return b58encode(inp_fmtd + checksum) - -def safe_from_hex(s): - return binascii.unhexlify(s) - -def from_int_to_byte(a): - return struct.pack(b'B', a) - -def from_byte_to_int(a): - return struct.unpack(b'B', a)[0] - -def from_string_to_bytes(a): - return a if isinstance(a, bytes) else bytes(a, 'utf-8') - -def safe_hexlify(a): - return binascii.hexlify(a).decode('ascii') - -class SerializationError(Exception): - """Base class for serialization errors""" - - -class SerializationTruncationError(SerializationError): - """Serialized data was truncated - Thrown by deserialize() and stream_deserialize() - """ - -def ser_read(f, n): - """Read from a stream safely - Raises SerializationError and SerializationTruncationError appropriately. - Use this instead of f.read() in your classes stream_(de)serialization() - functions. - """ - MAX_SIZE = 0x02000000 - if n > MAX_SIZE: - raise SerializationError('Asked to read 0x%x bytes; MAX_SIZE exceeded' % n) - r = f.read(n) - if len(r) < n: - raise SerializationTruncationError('Asked to read %i bytes, but only got %i' % (n, len(r))) - return r - -B58_DIGITS = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' - -class Base58Error(Exception): - pass - -class InvalidBase58Error(Base58Error): - """Raised on generic invalid base58 data, such as bad characters. - Checksum failures raise Base58ChecksumError specifically. - """ - pass - -def b58encode(b): - """Encode bytes to a base58-encoded string""" - - # Convert big-endian bytes to integer - n = int('0x0' + binascii.hexlify(b).decode('ascii'), 16) - - # Divide that integer into bas58 - res = [] - while n > 0: - n, r = divmod(n, 58) - res.append(B58_DIGITS[r]) - res = ''.join(res[::-1]) - - # Encode leading zeros as base58 zeros - czero = b'\x00' - if sys.version_info >= (3,0): - # In Python3 indexing a bytes returns numbers, not characters. - czero = 0 - pad = 0 - for c in b: - if c == czero: - pad += 1 - else: - break - return B58_DIGITS[0] * pad + res - -def b58decode(s): - """Decode a base58-encoding string, returning bytes""" - if not s: - return b'' - - # Convert the string to an integer - n = 0 - for c in s: - n *= 58 - if c not in B58_DIGITS: - raise InvalidBase58Error('Character %r is not a valid base58 character' % c) - digit = B58_DIGITS.index(c) - n += digit - - # Convert the integer to bytes - h = '%x' % n - if len(h) % 2: - h = '0' + h - res = bytes(binascii.unhexlify(h.encode('utf8'))) - - # Add padding back. - pad = 0 - for c in s[:-1]: - if c == B58_DIGITS[0]: pad += 1 - else: break - return b'\x00' * pad + res - -def uint256encode(s): - """Convert bytes to uint256""" - r = 0 - t = struct.unpack(b"> (i * 32) & 0xffffffff) - return r - -def encode(val, base, minlen=0): - base, minlen = int(base), int(minlen) - code_string = get_code_string(base) - result_bytes = bytes() - while val > 0: - curcode = code_string[val % base] - result_bytes = bytes([ord(curcode)]) + result_bytes - val //= base - - pad_size = minlen - len(result_bytes) - - padding_element = b'\x00' if base == 256 else b'1' \ - if base == 58 else b'0' - if (pad_size > 0): - result_bytes = padding_element*pad_size + result_bytes - - result_string = ''.join([chr(y) for y in result_bytes]) - result = result_bytes if base == 256 else result_string - - return result - -def decode(string, base): - if base == 256 and isinstance(string, str): - string = bytes(bytearray.fromhex(string)) - base = int(base) - code_string = get_code_string(base) - result = 0 - if base == 256: - def extract(d, cs): - if isinstance(d, int): - return d - else: - return struct.unpack(b'B', d)[0] - else: - def extract(d, cs): - return cs.find(d if isinstance(d, str) else chr(d)) - - if base == 16: - string = string.lower() - while len(string) > 0: - result *= base - result += extract(string[0], code_string) - string = string[1:] - return result - """PoDLE related primitives """ def getG(compressed=True): @@ -237,165 +40,6 @@ def podle_PrivateKey(priv): """ return secp256k1.PrivateKey(priv) - -def privkey_to_address(priv, from_hex=True, magicbyte=0): - return pubkey_to_address(privkey_to_pubkey(priv, from_hex), magicbyte) - -privtoaddr = privkey_to_address - -# Hashes -def bin_hash160(string): - intermed = hashlib.sha256(string).digest() - return hashlib.new('ripemd160', intermed).digest() - -def hash160(string): - if not isinstance(string, bytes): - string = string.encode('utf-8') - return safe_hexlify(bin_hash160(string)) - -def bin_sha256(string): - binary_data = string if isinstance(string, bytes) else bytes(string, - 'utf-8') - return hashlib.sha256(binary_data).digest() - -def sha256(string): - return safe_hexlify(bin_sha256(string)) - -def bin_dbl_sha256(bytes_to_hash): - if not isinstance(bytes_to_hash, bytes): - bytes_to_hash = bytes_to_hash.encode('utf-8') - return hashlib.sha256(hashlib.sha256(bytes_to_hash).digest()).digest() - -def dbl_sha256(string): - return hashlib.sha256(hashlib.sha256(string).digest()).hexdigest() - -def hash_to_int(x): - if len(x) in [40, 64]: - return decode(x, 16) - return decode(x, 256) - -def num_to_var_int(x): - if not isinstance(x, int): - if len(x) == 0: - return b'\x00' - x = struct.unpack(b'B', x)[0] - if x < 253: return from_int_to_byte(x) - elif x < 65536: return from_int_to_byte(253) + struct.pack(b'= (3,0): - newpriv = secp256k1.PrivateKey(secret=native_bytes(priv)) - else: - newpriv = secp256k1.PrivateKey(secret=bytes_to_native_str(priv)) + newpriv = secp256k1.PrivateKey(secret=native_bytes(priv)) return newpriv.public_key.format(compressed) -def privkey_to_pubkey(priv, usehex=True): - '''To avoid changing the interface from the legacy system, - allow an *optional* hex argument here (called differently from - maker/taker code to how it's called in bip32 code), then - pass to the standard hexbin decorator under the hood. - ''' - return privkey_to_pubkey_inner(priv, usehex) +# b58check wrapper functions around bitcointx.base58 functions: +# (avoids complexity of key management structure) + +def bin_to_b58check(inp, magicbyte=b'\x00'): + """ The magic byte (prefix byte) should be passed either + as a single byte or an integer. What is returned is a string + in base58 encoding, with the prefix and the checksum. + """ + if not isinstance(magicbyte, int): + magicbyte = struct.unpack(b'B', magicbyte)[0] + assert(0 <= magicbyte <= 0xff) + if magicbyte == 0: + inp_fmtd = struct.pack(b'B', magicbyte) + inp + while magicbyte > 0: + inp_fmtd = struct.pack(b'B', magicbyte % 256) + inp + magicbyte //= 256 + checksum = Hash(inp_fmtd)[:4] + return base58.encode(inp_fmtd + checksum) + +def b58check_to_bin(s): + data = base58.decode(s) + assert Hash(data[:-4])[:4] == data[-4:] + return struct.pack(b"B", data[0]), data[1:-4] -privtopub = privkey_to_pubkey +def get_version_byte(s): + return b58check_to_bin(s)[0] -@hexbin -def is_valid_pubkey(pubkey, usehex, require_compressed=False): +def ecdsa_sign(msg, priv, formsg=False): + hashed_msg = BitcoinMessage(msg).GetHash() + sig = ecdsa_raw_sign(hashed_msg, priv, rawmsg=True, formsg=formsg) + return base64.b64encode(sig).decode('ascii') + +def ecdsa_verify(msg, sig, pub): + hashed_msg = BitcoinMessage(msg).GetHash() + sig = base64.b64decode(sig) + return ecdsa_raw_verify(hashed_msg, pub, sig, rawmsg=True) + +def is_valid_pubkey(pubkey, require_compressed=False): """ Returns True if the serialized pubkey is a valid secp256k1 pubkey serialization or False if not; returns False for an uncompressed encoding if require_compressed is True. @@ -458,8 +124,8 @@ def is_valid_pubkey(pubkey, usehex, require_compressed=False): return False return True -@hexbin -def multiply(s, pub, usehex, rawpub=True, return_serialized=True): + +def multiply(s, pub, return_serialized=True): '''Input binary compressed pubkey P(33 bytes) and scalar s(32 bytes), return s*P. The return value is a binary compressed public key, @@ -470,24 +136,19 @@ def multiply(s, pub, usehex, rawpub=True, return_serialized=True): ''' newpub = secp256k1.PublicKey(pub) #see note to "tweak_mul" function in podle.py - if sys.version_info >= (3,0): - res = newpub.multiply(native_bytes(s)) - else: - res = newpub.multiply(bytes_to_native_str(s)) + res = newpub.multiply(native_bytes(s)) if not return_serialized: return res return res.format() -@hexbin -def add_pubkeys(pubkeys, usehex): +def add_pubkeys(pubkeys): '''Input a list of binary compressed pubkeys and return their sum as a binary compressed pubkey.''' pubkey_list = [secp256k1.PublicKey(x) for x in pubkeys] r = secp256k1.PublicKey.combine_keys(pubkey_list) return r.format() -@hexbin -def add_privkeys(priv1, priv2, usehex): +def add_privkeys(priv1, priv2): '''Add privkey 1 to privkey 2. Input keys must be in binary either compressed or not. Returned key will have the same compression state. @@ -504,58 +165,34 @@ def add_privkeys(priv1, priv2, usehex): res += b'\x01' return res -@hexbin + def ecdsa_raw_sign(msg, priv, - usehex, - rawpriv=True, rawmsg=False, - usenonce=None, formsg=False): '''Take the binary message msg and sign it with the private key priv. - By default priv is just a 32 byte string, if rawpriv is false - it is assumed to be hex encoded (note only works if usehex=False). If rawmsg is True, no sha256 hash is applied to msg before signing. In this case, msg must be a precalculated hash (256 bit). If rawmsg is False, the secp256k1 lib will hash the message as part of the ECDSA-SHA256 signing algo. - If usenonce is not None, its value is passed to the secp256k1 library - sign() function as the ndata value, which is then used in conjunction - with a custom nonce generating function, such that the nonce used in the ECDSA - sign algorithm is exactly that value (ndata there, usenonce here). 32 bytes. Return value: the calculated signature.''' if rawmsg and len(msg) != 32: raise Exception("Invalid hash input to ECDSA raw sign.") - if rawpriv: - compressed, p = read_privkey(priv) - newpriv = secp256k1.PrivateKey(p) - else: - newpriv = secp256k1.PrivateKey.from_hex(priv) + + compressed, p = read_privkey(priv) + newpriv = secp256k1.PrivateKey(p) if formsg: sig = newpriv.sign_recoverable(msg) return sig - #Donations, thus custom nonce, currently disabled, hence not covered. - elif usenonce: #pragma: no cover - raise NotImplementedError - #if len(usenonce) != 32: - # raise ValueError("Invalid nonce passed to ecdsa_sign: " + str( - # usenonce)) - #nf = ffi.addressof(_noncefunc.lib, "nonce_function_rand") - #ndata = ffi.new("char [32]", usenonce) - #usenonce = (nf, ndata) - #sig = newpriv.ecdsa_sign(msg, raw=rawmsg, custom_nonce=usenonce) else: - #partial fix for secp256k1-transient not including customnonce; - #partial because donations will crash on windows in the "if". if rawmsg: sig = newpriv.sign(msg, hasher=None) else: sig = newpriv.sign(msg) return sig -@hexbin -def ecdsa_raw_verify(msg, pub, sig, usehex, rawmsg=False): +def ecdsa_raw_verify(msg, pub, sig, rawmsg=False): '''Take the binary message msg and binary signature sig, and verify it against the pubkey pub. If rawmsg is True, no sha256 hash is applied to msg before verifying. @@ -575,56 +212,6 @@ def ecdsa_raw_verify(msg, pub, sig, usehex, rawmsg=False): retval = newpub.verify(sig, msg, hasher=None) else: retval = newpub.verify(sig, msg) - except: + except Exception as e: return False return retval - -def estimate_tx_size(ins, outs, txtype='p2pkh'): - '''Estimate transaction size. - The txtype field as detailed below is used to distinguish - the type, but there is at least one source of meaningful roughness: - we assume the output types are the same as the input (to be fair, - outputs only contribute a little to the overall total). This combined - with a few bytes variation in signature sizes means we will expect, - say, 10% inaccuracy here. - - Assuming p2pkh: - out: 8+1+3+2+20=34, in: 1+32+4+1+1+~73+1+1+33=147, - ver:4,seq:4, +2 (len in,out) - total ~= 34*len_out + 147*len_in + 10 (sig sizes vary slightly) - Assuming p2sh M of N multisig: - "ins" must contain M, N so ins= (numins, M, N) (crude assuming all same) - 74*M + 34*N + 45 per input, so total ins ~ len_ins * (45+74M+34N) - so total ~ 34*len_out + (45+74M+34N)*len_in + 10 - Assuming p2sh-p2wpkh: - witness are roughly 3+~73+33 for each input - (txid, vin, 4+20 for witness program encoded as scriptsig, 4 for sequence) - non-witness input fields are roughly 32+4+4+20+4=64, so total becomes - n_in * 64 + 4(ver) + 4(locktime) + n_out*34 - Assuming p2wpkh native: - witness as previous case - non-witness loses the 24 witnessprogram, replaced with 1 zero, - in the scriptSig, so becomes: - n_in * 41 + 4(ver) + 4(locktime) +2 (len in, out) + n_out*34 - ''' - if txtype == 'p2pkh': - return 10 + ins * 147 + 34 * outs - elif txtype == 'p2sh-p2wpkh': - #return the estimate for the witness and non-witness - #portions of the transaction, assuming that all the inputs - #are of segwit type p2sh-p2wpkh - # Note as of Jan19: this misses 2 bytes (trivial) for len in, out - # and also overestimates output size by 2 bytes. - witness_estimate = ins*109 - non_witness_estimate = 4 + 4 + outs*34 + ins*64 - return (witness_estimate, non_witness_estimate) - elif txtype == 'p2wpkh': - witness_estimate = ins*109 - non_witness_estimate = 4 + 4 + 2 + outs*31 + ins*41 - return (witness_estimate, non_witness_estimate) - elif txtype == 'p2shMofN': - ins, M, N = ins - return 10 + (45 + 74*M + 34*N) * ins + 34 * outs - else: - raise NotImplementedError("Transaction size estimation not" + - "yet implemented for type: " + txtype) diff --git a/jmbitcoin/jmbitcoin/secp256k1_transaction.py b/jmbitcoin/jmbitcoin/secp256k1_transaction.py index b8982da..aca9903 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_transaction.py +++ b/jmbitcoin/jmbitcoin/secp256k1_transaction.py @@ -9,548 +9,95 @@ import struct # note, only used for non-cryptographic randomness: import random from jmbitcoin.secp256k1_main import * -from jmbitcoin.bech32 import * -import jmbitcoin as btc -P2PKH_PRE, P2PKH_POST = b'\x76\xa9\x14', b'\x88\xac' -P2SH_P2WPKH_PRE, P2SH_P2WPKH_POST = b'\xa9\x14', b'\x87' -P2WPKH_PRE = b'\x00\x14' -P2WSH_PRE = b'\x00\x20' - -# Transaction serialization and deserialization - -def deserialize(txinp): - if isinstance(txinp, basestring) and not isinstance(txinp, bytes): - tx = BytesIO(binascii.unhexlify(txinp)) - hexout = True - else: - tx = BytesIO(txinp) - hexout = False - - def hex_string(scriptbytes, hexout): - if hexout: - return binascii.hexlify(scriptbytes).decode('ascii') - else: - return scriptbytes - - def read_as_int(bytez): - if bytez == 2: - return struct.unpack(b' len(newtx["outs"]): - raise Exception( - "Transactions with sighash single should have len in <= len out") - newtx["outs"] = newtx["outs"][:i+1] - for out in newtx["outs"][:i]: - out['value'] = 2**64 - 1 - out['script'] = "" - for j, inp in enumerate(newtx["ins"]): - if j != i: - inp["sequence"] = 0 - if hashcode & SIGHASH_ANYONECANPAY: - newtx["ins"] = [newtx["ins"][i]] - else: - pass - return newtx - -def segwit_txid(tx, hashcode=None): - #An easy way to construct the old-style hash (which is the real txid, - #the one without witness or marker/flag, is to remove all txinwitness - #entries from the deserialized form of the full tx, then reserialize, - #because serialize uses that as a flag to decide which serialization - #style to apply. - dtx = deserialize(tx) - for vin in dtx["ins"]: - if "txinwitness" in vin: - del vin["txinwitness"] - reserialized_tx = serialize(dtx) - return txhash(reserialized_tx, hashcode) - -def txhash(tx, hashcode=None, check_sw=True): - """ Creates the appropriate sha256 hash as required - either for signing or calculating txids. - The hashcode argument is used to distinguish the case - where we are hashing for signing (sighashing); by default - it is None, and this indicates we are calculating a txid. - If check_sw is True it checks the serialized format for - segwit flag bytes, and produces the correct form for txid (not wtxid). - """ - if not isinstance(tx, basestring): - tx = serialize(tx) - if isinstance(tx, basestring) and not isinstance(tx, bytes): - tx = binascii.unhexlify(tx) - if check_sw and from_byte_to_int(tx[4:5]) == 0: - if not from_byte_to_int(tx[5:6]) == 1: - #This invalid, but a raise is a DOS vector in some contexts. - return None - return segwit_txid(tx, hashcode) - if hashcode: - if not isinstance(hashcode, int): - hashcode = struct.unpack(b'B', hashcode)[0] - return dbl_sha256(from_string_to_bytes(tx) + struct.pack(b'= 0 - if not isinstance(tx, bytes): - tx = binascii.unhexlify(tx) - if not isinstance(script, bytes): - script = binascii.unhexlify(script) - if scriptCode is not None and not isinstance(scriptCode, bytes): - scriptCode = binascii.unhexlify(scriptCode) - if isinstance(sig, bytes): - sig = binascii.hexlify(sig).decode('ascii') - if isinstance(pub, bytes): - pub = binascii.hexlify(pub).decode('ascii') - - hashcode = binascii.unhexlify(sig[-2:]) - - if amount: - modtx = segwit_signature_form(deserialize(tx), i, - scriptCode, amount, hashcode, decoder_func=lambda x: x) - else: - modtx = signature_form(tx, i, script, hashcode) - return ecdsa_tx_verify(modtx, sig, pub, hashcode) - - -def sign(tx, i, priv, hashcode=SIGHASH_ALL, usenonce=None, amount=None, - native=False): - """ - Given a serialized transaction tx, an input index i, and a privkey - in bytes or hex, returns a serialized transaction, in hex always, - into which the signature and/or witness has been inserted. The field - `amount` flags whether segwit signing is to be done, and the field - `native` flags that native segwit p2wpkh signing is to be done. Note - that signing multisig is to be done with the alternative functions - get_p2sh_signature or get_p2wsh_signature (and non N of N multisig - scripthash signing is not currently supported). - """ - if isinstance(tx, basestring) and not isinstance(tx, bytes): - tx = binascii.unhexlify(tx) - if len(priv) <= 33: - priv = safe_hexlify(priv) - if amount: - serobj = p2wpkh_sign(tx, i, priv, amount, hashcode=hashcode, - usenonce=usenonce, native=native) - else: - pub = privkey_to_pubkey(priv, True) - address = pubkey_to_address(pub) - signing_tx = signature_form(tx, i, mk_pubkey_script(address), hashcode) - sig = ecdsa_tx_sign(signing_tx, priv, hashcode, usenonce=usenonce) - txobj = deserialize(tx) - txobj["ins"][i]["script"] = serialize_script([sig, pub]) - serobj = serialize(txobj) - if isinstance(serobj, basestring) and isinstance(serobj, bytes): - return binascii.hexlify(serobj).decode('ascii') - else: - return serobj - -def p2wpkh_sign(tx, i, priv, amount, hashcode=SIGHASH_ALL, native=False, - usenonce=None): - """Given a serialized transaction, index, private key in hex, - amount in satoshis and optionally hashcode, return the serialized - transaction containing a signature and witness for this input; it's - assumed that the input is of type pay-to-witness-pubkey-hash. - If native is False, it's treated as p2sh nested. + Given a transaction tx of type CMutableTransaction, an input index i, + and a raw privkey in bytes, updates the CMutableTransaction to contain + the newly appended signature. + Only three scriptPubKey types supported: p2pkh, p2wpkh, p2sh-p2wpkh. + Note that signing multisig must be done outside this function, using + the wrapped library. + Returns: (signature, "signing succeeded") + or: (None, errormsg) in case of failure """ - pub = privkey_to_pubkey(priv) - # Convert the input tx and script to hex so that the deserialize() - # call creates hex-encoded fields - script = binascii.hexlify(pubkey_to_p2wpkh_script(pub)).decode('ascii') - scriptCode = binascii.hexlify(pubkey_to_p2pkh_script(pub)).decode('ascii') - if isinstance(tx, bytes): - tx = binascii.hexlify(tx).decode('ascii') - signing_tx = segwit_signature_form(deserialize(tx), i, scriptCode, amount, - hashcode=hashcode) - sig = ecdsa_tx_sign(signing_tx, priv, hashcode, usenonce=usenonce) - txobj = deserialize(tx) - if not native: - txobj["ins"][i]["script"] = "16" + script - else: - txobj["ins"][i]["script"] = "" - txobj["ins"][i]["txinwitness"] = [sig, pub] - return serialize(txobj) - -def signall(tx, priv): - # if priv is a dictionary, assume format is - # { 'txinhash:txinidx' : privkey } - if isinstance(priv, dict): - for e, i in enumerate(deserialize(tx)["ins"]): - k = priv["%s:%d" % (i["outpoint"]["hash"], i["outpoint"]["index"])] - tx = sign(tx, e, k) - else: - for i in range(len(deserialize(tx)["ins"])): - tx = sign(tx, i, priv) - return tx + # script verification flags + flags = set() -def get_p2sh_signature(tx, i, redeem_script, pk, amount=None, hashcode=SIGHASH_ALL): - """ - Tx is assumed to be serialized. redeem_script is for example the - output of mk_multisig_script. - pk is the private key, and must be passed in hex. - If amount is not None, the output of get_p2wsh_signature is returned. - What is returned is a single signature. - """ - if isinstance(tx, str): - tx = binascii.unhexlify(tx) - if isinstance(redeem_script, str): - redeem_script = binascii.unhexlify(redeem_script) - if amount: - return get_p2wsh_signature(tx, i, redeem_script, pk, amount, hashcode) - modtx = signature_form(tx, i, redeem_script, hashcode) - return ecdsa_tx_sign(modtx, pk, hashcode) + def return_err(e): + return None, "Error in signing: " + repr(e) -def get_p2wsh_signature(tx, i, redeem_script, pk, amount, hashcode=SIGHASH_ALL): - """ See note to multisign for the value to pass in as `script`. - Tx is assumed to be serialized. - """ - modtx = segwit_signature_form(deserialize(tx), i, redeem_script, amount, - hashcode, decoder_func=lambda x: x) - return ecdsa_tx_sign(modtx, pk, hashcode) + assert isinstance(tx, CMutableTransaction) + # using direct local access to libsecp256k1 binding, because + # python-bitcoinlib uses OpenSSL key management: + pub = privkey_to_pubkey(priv) -def apply_p2wsh_multisignatures(tx, i, script, sigs): - """Sigs must be passed in as a list, and must be a - complete list for this multisig, in the same order - as the list of pubkeys when creating the scriptPubKey. - """ - if isinstance(script, str): - script = binascii.unhexlify(script) - sigs = [binascii.unhexlify(x) if x[:2] == '30' else x for x in sigs] - if isinstance(tx, str): - return safe_hexlify(apply_p2wsh_multisignatures( - binascii.unhexlify(tx), i, script, sigs)) - txobj = deserialize(tx) - txobj["ins"][i]["script"] = "" - txobj["ins"][i]["txinwitness"] = [None] + sigs + [script] - return serialize(txobj) + if not amount: + # p2pkh only supported here: + input_scriptPubKey = pubkey_to_p2pkh_script(pub) + sighash = SignatureHash(input_scriptPubKey, tx, i, hashcode) + try: + sig = ecdsa_raw_sign(sighash, priv, rawmsg=True) + bytes([hashcode]) + except Exception as e: + return return_err(e) + tx.vin[i].scriptSig = CScript([sig, pub]) + # Verify the signature worked. + try: + VerifyScript(tx.vin[i].scriptSig, + input_scriptPubKey, tx, i, flags=flags) + except Exception as e: + return return_err(e) + return sig, "signing succeeded" -def apply_multisignatures(*args): - # tx,i,script,sigs OR tx,i,script,sig1,sig2...,sig[n] - tx, i, script = args[0], int(args[1]), args[2] - sigs = args[3] if isinstance(args[3], list) else list(args[3:]) + else: + # segwit case; we currently support p2wpkh native or under p2sh. + + # see line 1256 of bitcointx.core.scripteval.py: + flags.add(SCRIPT_VERIFY_P2SH) + + input_scriptPubKey = pubkey_to_p2wpkh_script(pub) + # only created for convenience access to scriptCode: + input_address = P2WPKHBitcoinAddress.from_scriptPubKey(input_scriptPubKey) + # function name is misleading here; redeemScript only applies to p2sh. + scriptCode = input_address.to_redeemScript() + + sighash = SignatureHash(scriptCode, tx, i, hashcode, amount=amount, + sigversion=SIGVERSION_WITNESS_V0) + try: + sig = ecdsa_raw_sign(sighash, priv, rawmsg=True) + bytes([hashcode]) + except Exception as e: + return return_err(e) + if native: + flags.add(SCRIPT_VERIFY_WITNESS) + else: + tx.vin[i].scriptSig = CScript([input_scriptPubKey]) - if isinstance(script, str): - script = binascii.unhexlify(script) - sigs = [binascii.unhexlify(x) if x[:2] == '30' else x for x in sigs] - if isinstance(tx, str): - return safe_hexlify(apply_multisignatures( - binascii.unhexlify(tx), i, script, sigs)) + witness = [sig, pub] + ctxwitness = CTxInWitness(CScriptWitness(witness)) + tx.wit.vtxinwit[i] = ctxwitness + # Verify the signature worked. + try: + VerifyScript(tx.vin[i].scriptSig, input_scriptPubKey, tx, i, + flags=flags, amount=amount, witness=tx.wit.vtxinwit[i].scriptWitness) + except ValidationError as e: + return return_err(e) - txobj = deserialize(tx) - txobj["ins"][i]["script"] = serialize_script([None] + sigs + [script]) - return serialize(txobj) + return sig, "signing succeeded" def apply_freeze_signature(tx, i, redeem_script, sig): if isinstance(redeem_script, str): @@ -863,14 +213,16 @@ def apply_freeze_signature(tx, i, redeem_script, sig): return serialize(txobj) def mktx(ins, outs, version=1, locktime=0): - """ Given a list of input dicts with key "output" - which are txid:n strings in hex, and a list of outputs - which are dicts with keys "address", "value", outputs - a hex serialized tranasction encoding this data. + """ Given a list of input tuples (txid(bytes), n(int)), + and a list of outputs which are dicts with + keys "address" (value should be *str* not CCoinAddress), + "value" (value should be integer satoshis), outputs a + CMutableTransaction object. Tx version and locktime are optionally set, for non-default locktimes, inputs are given nSequence as per below comment. """ - txobj = {"locktime": locktime, "version": version, "ins": [], "outs": []} + vin = [] + vout = [] # This does NOT trigger rbf and mimics Core's standard behaviour as of # Jan 2019. # Tx creators wishing to use rbf will need to set it explicitly outside @@ -880,48 +232,36 @@ def mktx(ins, outs, version=1, locktime=0): else: sequence = 0xffffffff for i in ins: - if isinstance(i, dict) and "outpoint" in i: - txobj["ins"].append(i) - else: - if isinstance(i, dict) and "output" in i: - i = i["output"] - txobj["ins"].append({ - "outpoint": {"hash": i[:64], - "index": int(i[65:])}, - "script": "", - "sequence": sequence - }) + outpoint = CMutableOutPoint((i[0][::-1]), i[1]) + inp = CMutableTxIn(prevout=outpoint, nSequence=sequence) + vin.append(inp) for o in outs: - if isinstance(o, str): - addr = o[:o.find(':')] - val = int(o[o.find(':') + 1:]) - o = {} - if re.match('^[0-9a-fA-F]*$', addr): - o["script"] = addr - else: - o["address"] = addr - o["value"] = val - - outobj = {} - if "address" in o: - outobj["script"] = address_to_script(o["address"]) - elif "script" in o: - outobj["script"] = o["script"] - else: - raise Exception("Could not find 'address' or 'script' in output.") - outobj["value"] = o["value"] - txobj["outs"].append(outobj) - return serialize(txobj) - -def make_shuffled_tx(ins, outs, deser=True, version=1, locktime=0): - """ Simple utility to ensure transaction + # note the to_scriptPubKey method is only available for standard + # address types + out = CMutableTxOut(o["value"], + CCoinAddress(o["address"]).to_scriptPubKey()) + vout.append(out) + return CMutableTransaction(vin, vout, nLockTime=locktime) + +def make_shuffled_tx(ins, outs, version=1, locktime=0): + """ Simple wrapper to ensure transaction inputs and outputs are randomly ordered. Can possibly be replaced by BIP69 in future """ random.shuffle(ins) random.shuffle(outs) - tx = mktx(ins, outs, version=version, locktime=locktime) - if deser: - return deserialize(tx) - else: - return tx + return mktx(ins, outs, version=version, locktime=locktime) + +def verify_tx_input(tx, i, scriptSig, scriptPubKey, amount=None, + witness=None, native=False): + flags = set() + if witness: + flags.add(SCRIPT_VERIFY_P2SH) + if native: + flags.add(SCRIPT_VERIFY_WITNESS) + try: + VerifyScript(scriptSig, scriptPubKey, tx, i, + flags=flags, amount=amount, witness=witness) + except ValidationError as e: + return False + return True diff --git a/jmbitcoin/setup.py b/jmbitcoin/setup.py index 5409382..018025f 100644 --- a/jmbitcoin/setup.py +++ b/jmbitcoin/setup.py @@ -9,6 +9,6 @@ setup(name='joinmarketbitcoin', author_email='', license='GPL', packages=['jmbitcoin'], - install_requires=['future', 'coincurve', 'urldecode'], + install_requires=['future', 'coincurve', 'python-bitcointx', 'urldecode'], python_requires='>=3.5', zip_safe=False) diff --git a/jmbitcoin/test/test_addresses.py b/jmbitcoin/test/test_addresses.py deleted file mode 100644 index d7723b1..0000000 --- a/jmbitcoin/test/test_addresses.py +++ /dev/null @@ -1,56 +0,0 @@ -import jmbitcoin as btc -import json -import pytest -import os -testdir = os.path.dirname(os.path.realpath(__file__)) - -def validate_address(addr, nettype): - """A mock of jmclient.validate_address - """ - BTC_P2PK_VBYTE = {"mainnet": b'\x00', "testnet": b'\x6f'} - BTC_P2SH_VBYTE = {"mainnet": b'\x05', "testnet": b'\xc4'} - try: - ver = btc.get_version_byte(addr) - except AssertionError as e: - return False, 'Checksum wrong. Typo in address?' - except Exception as e: - return False, "Invalid bitcoin address" - if ver not in [BTC_P2PK_VBYTE[nettype], BTC_P2SH_VBYTE[nettype]]: - return False, 'Wrong address version. Testnet/mainnet confused?' - if len(btc.b58check_to_bin(addr)) != 20: - return False, "Address has correct checksum but wrong length." - return True, 'address validated' - -@pytest.mark.parametrize( - "net", - [ - # 1 - ("mainnet"), - # 2 - ("testnet") - ]) -def test_b58_invalid_addresses(net): - #none of these are valid as any kind of key or address - with open(os.path.join(testdir,"base58_keys_invalid.json"), "r") as f: - json_data = f.read() - invalid_key_list = json.loads(json_data) - for k in invalid_key_list: - bad_key = k[0] - res, message = validate_address(bad_key, nettype=net) - assert res == False, "Incorrectly validated address: " + bad_key + " with message: " + message - -def test_b58_valid_addresses(): - with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f: - json_data = f.read() - valid_keys_list = json.loads(json_data) - for a in valid_keys_list: - addr, pubkey, prop_dict = a - if not prop_dict["isPrivkey"]: - if prop_dict["isTestnet"]: - net = "testnet" - else: - net = "mainnet" - #if using pytest -s ; sanity check to see what's actually being tested - res, message = validate_address(addr, net) - assert res == True, "Incorrectly failed to validate address: " + addr + " with message: " + message - diff --git a/jmbitcoin/test/test_bech32.py b/jmbitcoin/test/test_bech32.py deleted file mode 100644 index ec5de0d..0000000 --- a/jmbitcoin/test/test_bech32.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2017 Pieter Wuille -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - - -"""Reference tests for segwit adresses""" - -import binascii -import unittest -import jmbitcoin as btc - -VALID_CHECKSUM = [ - "A12UEL5L", - "an83characterlonghumanreadablepartthatcontainsthenumber1andtheexcludedcharactersbio1tt5tgs", - "abcdef1qpzry9x8gf2tvdw0s3jn54khce6mua7lmqqqxw", - "11qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqc8247j", - "split1checkupstagehandshakeupstreamerranterredcaperred2y9e3w", -] - -INVALID_CHECKSUM = [ - " 1nwldj5", - "\x7F" + "1axkwrx", - "an84characterslonghumanreadablepartthatcontainsthenumber1andtheexcludedcharactersbio1569pvx", - "pzry9x0s0muk", - "1pzry9x0s0muk", - "x1b4n0q5v", - "li1dgmt3", - "de1lg7wt\xff", -] - -VALID_ADDRESS = [ - ["BC1QW508D6QEJXTDG4Y5R3ZARVARY0C5XW7KV8F3T4", "0014751e76e8199196d454941c45d1b3a323f1433bd6"], - ["tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sl5k7", - "00201863143c14c5166804bd19203356da136c985678cd4d27a1b8c6329604903262"], - ["bc1pw508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7k7grplx", - "5128751e76e8199196d454941c45d1b3a323f1433bd6751e76e8199196d454941c45d1b3a323f1433bd6"], - ["BC1SW50QA3JX3S", "6002751e"], - ["bc1zw508d6qejxtdg4y5r3zarvaryvg6kdaj", "5210751e76e8199196d454941c45d1b3a323"], - ["tb1qqqqqp399et2xygdj5xreqhjjvcmzhxw4aywxecjdzew6hylgvsesrxh6hy", - "0020000000c4a5cad46221b2a187905e5266362b99d5e91c6ce24d165dab93e86433"], -] - -INVALID_ADDRESS = [ - "tc1qw508d6qejxtdg4y5r3zarvary0c5xw7kg3g4ty", - "bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t5", - "BC13W508D6QEJXTDG4Y5R3ZARVARY0C5XW7KN40WF2", - "bc1rw5uspcuh", - "bc10w508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7kw5rljs90", - "BC1QR508D6QEJXTDG4Y5R3ZARVARYV98GJ9P", - "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sL5k7", - "bc1zw508d6qejxtdg4y5r3zarvaryvqyzf3du", - "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3pjxtptv", - "bc1gmk9yu", - -] - -INVALID_ADDRESS_ENC = [ - ("BC", 0, 20), - ("bc", 0, 21), - ("bc", 17, 32), - ("bc", 1, 1), - ("bc", 16, 41), -] - -class TestSegwitAddress(unittest.TestCase): - """Unit test class for segwit addressess.""" - - def test_valid_checksum(self): - """Test checksum creation and validation.""" - for test in VALID_CHECKSUM: - hrp, _ = btc.bech32_decode(test) - self.assertIsNotNone(hrp) - pos = test.rfind('1') - test = test[:pos+1] + chr(ord(test[pos + 1]) ^ 1) + test[pos+2:] - hrp, _ = btc.bech32_decode(test) - self.assertIsNone(hrp) - - def test_invalid_checksum(self): - """Test validation of invalid checksums.""" - for test in INVALID_CHECKSUM: - hrp, _ = btc.bech32_decode(test) - self.assertIsNone(hrp) - - def test_valid_address(self): - """Test whether valid addresses decode to the correct output.""" - for (address, hexscript) in VALID_ADDRESS: - hrp = "bc" - witver, witprog = btc.bech32addr_decode(hrp, address) - if witver is None: - hrp = "tb" - witver, witprog = btc.bech32addr_decode(hrp, address) - self.assertIsNotNone(witver) - scriptpubkey = btc.segwit_scriptpubkey(witver, witprog) - self.assertEqual(scriptpubkey, binascii.unhexlify(hexscript)) - addr = btc.bech32addr_encode(hrp, witver, witprog) - self.assertEqual(address.lower(), addr) - - def test_invalid_address(self): - """Test whether invalid addresses fail to decode.""" - for test in INVALID_ADDRESS: - witver, _ = btc.bech32addr_decode("bc", test) - self.assertIsNone(witver) - witver, _ = btc.bech32addr_decode("tb", test) - self.assertIsNone(witver) - - def test_invalid_address_enc(self): - """Test whether address encoding fails on invalid input.""" - for hrp, version, length in INVALID_ADDRESS_ENC: - code = btc.bech32addr_encode(hrp, version, [0] * length) - self.assertIsNone(code) - -if __name__ == "__main__": - unittest.main() diff --git a/jmbitcoin/test/test_bip32.py b/jmbitcoin/test/test_bip32.py index d0eb5b1..f8b6c53 100644 --- a/jmbitcoin/test/test_bip32.py +++ b/jmbitcoin/test/test_bip32.py @@ -103,9 +103,9 @@ def test_ckd_pubkeys(): def test_bip32_descend(): master = btc.bip32_master_key(b'\x07'*32) end_key = btc.bip32_descend(master, [2, 3, 10000]) - assert end_key=="6856ef965940a1a7b1311dc041050ac0013e326c7ff4e2c677a7694b4f0405c901" + assert end_key==binascii.unhexlify("6856ef965940a1a7b1311dc041050ac0013e326c7ff4e2c677a7694b4f0405c901") end_key = btc.bip32_descend(master, 2, 5, 4, 5) - assert end_key=="d2d816b6485103c0d7ff95482788f0e8e73fa11817079e006d47979d8196c4b101" + assert end_key==binascii.unhexlify("d2d816b6485103c0d7ff95482788f0e8e73fa11817079e006d47979d8196c4b101") diff --git a/jmbitcoin/test/test_btc_formatting.py b/jmbitcoin/test/test_btc_formatting.py deleted file mode 100644 index 66475d5..0000000 --- a/jmbitcoin/test/test_btc_formatting.py +++ /dev/null @@ -1,57 +0,0 @@ -#! /usr/bin/env python -'''Test bitcoin module data handling''' - -import jmbitcoin as btc -import pytest -import binascii - -#used in p2sh addresses -def test_hash160(): - assert '0e3397b4abc7a382b3ea2365883c3c7ca5f07600' == \ - btc.hash160('The quick brown fox jumps over the lazy dog') - -def test_bad_code_string(): - for i in [1,9,257,-3,"256"]: - with pytest.raises(ValueError) as e_info: - btc.get_code_string(i) - -#Tests of compactsize encoding, see: -#https://bitcoin.org/en/developer-reference#compactsize-unsigned-integers -#note that little endian is used. -@pytest.mark.parametrize( - "num, compactsize", - [ - (252, "fc"), - (253, "fdfd00"), - (254, "fdfe00"), - (515, "fd0302"), - (65535, "fdffff"), - (65536, "fe00000100"), - (65537, "fe01000100"), - (4294967295, "feffffffff"), - (4294967296, "ff0000000001000000"), - - ]) -def test_compact_size(num, compactsize): - assert btc.num_to_var_int(num) == binascii.unhexlify(compactsize) - -@pytest.mark.parametrize("frm, to", [ - (("16001405481b7f1d90c5a167a15b00e8af76eb6984ea59"), - ["001405481b7f1d90c5a167a15b00e8af76eb6984ea59"]), - (("483045022100ad0dda327945e581a5effd83d75d76a9f07c3128f4dc6d25a54" - "e5ad5dd629bd00220487a992959bd540dbc335c655e6485ebfb394129eb48038f" - "0a2d319782f7cb690121039319452b6abafb5fcf06096196d0c141b8bd18a3de7" - "e9b9352da800d671ccd84"), - ["3045022100ad0dda327945e581a5effd83d75d76a9f07c3128f4dc6d25a54e5ad5dd629bd00220487a992959bd540dbc335c655e6485ebfb394129eb48038f0a2d319782f7cb6901", - "039319452b6abafb5fcf06096196d0c141b8bd18a3de7e9b9352da800d671ccd84"]), - (("51"), [1]), - (("00"), [None]), - (("510000"), [1, None, None]), - (("636505aaaaaaaaaa53"), [99, 101, "aaaaaaaaaa", 3]), - (("51" + "4d0101" + "aa"*257), [1, "aa"*257]), - (("4e" + "03000100" + "aa"*65539), ["aa"*65539]), -]) -def test_deserialize_script(frm, to): - #print(len(btc.deserialize_script(frm)[0])) - assert btc.deserialize_script(frm) == to - assert btc.serialize_script(to) == frm diff --git a/jmbitcoin/test/test_ecc_signing.py b/jmbitcoin/test/test_ecc_signing.py index 570a292..e3e9a73 100644 --- a/jmbitcoin/test/test_ecc_signing.py +++ b/jmbitcoin/test/test_ecc_signing.py @@ -12,25 +12,23 @@ vectors = None def test_valid_sigs(setup_ecc): for v in vectors['vectors']: - msg = v['msg'] - sig = v['sig'] - priv = v['privkey'] - assert sig == btc.ecdsa_raw_sign(msg, priv, True, rawmsg=True)+'01' - #check that the signature verifies against the key(pair) - pubkey = btc.privtopub(priv) - assert btc.ecdsa_raw_verify(msg, pubkey, sig[:-2], True, rawmsg=True) - #check that it fails to verify against corrupted signatures + msg, sig, priv = (binascii.unhexlify( + v[a]) for a in ["msg", "sig", "privkey"]) + assert sig == btc.ecdsa_raw_sign(msg, priv, rawmsg=True)+ b'\x01' + # check that the signature verifies against the key(pair) + pubkey = btc.privkey_to_pubkey(priv) + assert btc.ecdsa_raw_verify(msg, pubkey, sig[:-1], rawmsg=True) + # check that it fails to verify against corrupted signatures for i in [0,1,2,4,7,25,55]: - #corrupt one byte - binsig = binascii.unhexlify(sig) - checksig = binascii.hexlify(binsig[:i] + btc.from_string_to_bytes(chr( - (ord(binsig[i:i+1])+1) %256)) + binsig[i+1:-1]).decode('ascii') + # corrupt one byte + checksig = sig[:i] + chr( + (ord(sig[i:i+1])+1) %256).encode() + sig[i+1:-1] - #this kind of corruption will sometimes lead to an assert - #failure (if the DER format is corrupted) and sometimes lead - #to a signature verification failure. + # this kind of corruption will sometimes lead to an assert + # failure (if the DER format is corrupted) and sometimes lead + # to a signature verification failure. try: - res = btc.ecdsa_raw_verify(msg, pubkey, checksig, True, rawmsg=True) + res = btc.ecdsa_raw_verify(msg, pubkey, checksig, rawmsg=True) except: continue assert res==False diff --git a/jmbitcoin/test/test_main.py b/jmbitcoin/test/test_main.py deleted file mode 100644 index 5f2bf10..0000000 --- a/jmbitcoin/test/test_main.py +++ /dev/null @@ -1,61 +0,0 @@ -#! /usr/bin/env python -'''Testing mostly exceptional cases in secp256k1_main. - Some of these may represent code that should be removed, TODO.''' - -import jmbitcoin as btc -import binascii -import pytest -import os -testdir = os.path.dirname(os.path.realpath(__file__)) - -def test_hex2b58check(): - assert btc.hex_to_b58check("aa"*32) == "12JAT9y2EcnV6DPUGikLJYjWwk5UmUEFXRiQVmTbfSLbL3njFzp" - -def test_bindblsha(): - assert btc.bin_dbl_sha256("abc") == binascii.unhexlify( - "4f8b42c22dd3729b519ba6f68d2da7cc5b2d606d05daed5ad5128cc03e6c6358") - -def test_lpad(): - assert btc.lpad("aaaa", "b", 5) == "baaaa" - assert btc.lpad("aaaa", "b", 4) == "aaaa" - assert btc.lpad("aaaa", "b", 3) == "aaaa" - -def test_safe_from_hex(): - assert btc.safe_from_hex('ff0100') == b'\xff\x01\x00' - -def test_hash2int(): - assert btc.hash_to_int("aa"*32) == \ - 77194726158210796949047323339125271902179989777093709359638389338608753093290 - -@btc.hexbin -def dummyforwrap(a, b, c, d="foo", e="bar"): - newa = a+b"\x01" - x, y = b - newb = [x+b"\x02", y+b"\x03"] - if d == "foo": - return newb[1] - else: - return newb[0] - -def test_hexbin(): - assert dummyforwrap("aa", ["bb", "cc"], True) == "cc03" - assert dummyforwrap("aa", ["bb", "cc"], True, d="baz") == "bb02" - assert dummyforwrap(b"\xaa", [b"\xbb", b"\xcc"], False) == b"\xcc\x03" - -def test_add_privkeys(): - with pytest.raises(Exception) as e_info: - btc.add_privkeys("aa"*32, "bb"*32+"01", True) - -def test_ecdsa_raw_sign(): - msg = "aa"*31 - with pytest.raises(Exception) as e_info: - btc.ecdsa_raw_sign(msg, None, None, rawmsg=True) - assert e_info.match("Invalid hash input") - #build non-raw priv object as input - privraw = "aa"*32 - msghash = b"\xbb"*32 - sig = binascii.hexlify(btc.ecdsa_raw_sign(msghash, privraw, False, rawpriv=False, rawmsg=True)).decode('ascii') - assert sig == "3045022100b81960b4969b423199dea555f562a66b7f49dea5836a0168361f1a5f8a3c8298022003eea7d7ee4462e3e9d6d59220f950564caeb77f7b1cdb42af3c83b013ff3b2f" - - - \ No newline at end of file diff --git a/jmbitcoin/test/test_tx_serialize.py b/jmbitcoin/test/test_tx_serialize.py deleted file mode 100644 index 225ee5d..0000000 --- a/jmbitcoin/test/test_tx_serialize.py +++ /dev/null @@ -1,234 +0,0 @@ -import jmbitcoin as btc -import pytest -import json -import os -testdir = os.path.dirname(os.path.realpath(__file__)) - -#TODO: fold these examples into the tx_valid.json file -@pytest.mark.parametrize( - "tx_type, tx_id, tx_hex", - [("simple-tx", - "f31916a1d398a4ec18d56a311c942bb6db934cee6aa8ac30af0b30aad9efb841", - "0100000001c74265f31fc5e24895fdc83f7157cc40045235f3a71ae326a219de9de873" + - "0d8b010000006a473044022076055917470b7ec4f4bb008096266cf816ebb089ad983e" + - "6a0f63340ba0e6a6cb022059ec938b996a75db10504e46830e13d399f28191b9832bd5" + - "f61df097b9e0d47801210291941334a00959af4aa5757abf81d2a7d1aca8adb3431c67" + - "e89419271ba71cb4feffffff023cdeeb03000000001976a914a2426748f14eba44b3f6" + - "abba3e8bce216ea233f388acf4ebf303000000001976a914bfa366464a464005ba0df8" + - "6024a6c3ed859f03ac88ac33280600"), - ("3partycoinjoin", - "d91278e125673f5b9201456b0c36efac3b2b6700fdd04fc2227352a63f941170", - "010000000cedf1756d1d679de268a533c2d009f1d06ba8918908369c5e3b818678603a" + - "4bcb030000006b483045022100a775815dcfaf706ee9b6e6015f162d20d227eb44cf88" + - "93fb3ac95f910058fbd50220500f1f3de75b3bdf7f6a0949528f836c694a5bd1effedb" + - "b02156bcb292a78e5d012102a726879b9d663467ab71726ff5469c8b0a4213a93643f6" + - "ed739fdbb1e81ca5b9ffffffff3e0ebec9b42f30657ef187bd263c57849541014700cc" + - "5480268dc63ce6997552070000006b4830450221008ec354dcb5d661a60651e988356a" + - "215ccaaa688be1e78861b3b89f35e0c1bb7b02203d2117791b74ad2aa594e4dc806d03" + - "ec2a1241c4955b013dc365b6eacc4ee6c9012102632c8b13d9b94f2226c28295b52686" + - "4bbece7ea283a90ca001e498e2a08dbebcffffffff841e4ed900a6592687d7684ec0dd" + - "0893d7a3b86df07bb2626a1f51ae74ee0ba6010000006a473044022017f96effaf9812" + - "0745323859e1b52699baf0993675d028882e42310deac6921102200daaab3008e9bce1" + - "2a3e136a7e0472a12da31121237ac382a6a64e4ab5ed2e5201210365a1491d9c10f866" + - "0a02f47a54e5c0285f186a9f5b4cb75405548cef3427c4eeffffffff7e131188b66001" + - "c0f5a4b45bd3503d2ce17f40e43e3803a7f30c20a8214abc95080000006a4730440220" + - "6c21a02e4bfcce3f9a5775ce7b9033bebb354b15167dec1a1b3f0c7255633234022071" + - "994111c455b1ddd4cd093d0d4f2ecbbf1283a0a1f982962127768e3c4e251301210394" + - "1f5e1834e8b1c1503f2d20b3f3e06db2b80de47f7f31b4cb39f75525753822ffffffff" + - "955544430628ea06493caa541d59374d7708dbef0e59a6653ac025f341c888c8040000" + - "006a473044022022c2afb23a7d401cdfff735ba390fac8dd2cbde2dcce4c3a8f988dfd" + - "a08b956c02204ba4ca6dc72993da6bf3d020530f01d7fc931ef28c78c2fa9b70657e50" + - "8873c801210240dc515cf540c43575615f18670d3f511f6f47b719650be60183ad9f61" + - "f805cdffffffffedf1756d1d679de268a533c2d009f1d06ba8918908369c5e3b818678" + - "603a4bcb010000006b483045022100ee3b322aab4c77debc6ebf48967f7677c03cd5fb" + - "df775885691f1162cca1f04f0220031a8c4017618562f4a2f4f0ece98bf13063c9e03b" + - "720b2eb3ead0ef024bee1f012103ce920bc0f19adf76e0f8b36eb62873d3cba991ee3e" + - "6504e0dbf114a0d5865827ffffffffa8b4085018a3e8848531eecf3164ec2b89ed5ed2" + - "172e22a19e3343f29faa3c03050000006a47304402202ff247fb0ec1eb9fa24903957d" + - "8973eb9a3a8e2dc4d6e6dbbb366c7563b49e680220301a0df3f1dbff1f939f14b3bf78" + - "687d8a2da26dafcb310ade85efd9cd37bd050121038cfc01875ccaacd4863515259e52" + - "b96188e83ca30358bce4111c4021bfc66343fffffffff5e7b4d41b53a1fce37b349d02" + - "33aed80be2b5da0ff45c59014ed5e4ff09b0a5030000006a473044022015035665d467" + - "19031a1e09fffb92484365a38f8c51171046bedc0eea1385cb7402200a296a8315f9a6" + - "3fe9a7020d4fa3be20a2c0587f98ae0d8d0560985d9cfb0748012103d9537b8ccb7206" + - "383edff317fcb1e4188036a18a84a6727104dd9b4d00638037ffffffffedf1756d1d67" + - "9de268a533c2d009f1d06ba8918908369c5e3b818678603a4bcb000000006a47304402" + - "200742b54bb0d8e62dbb87eaa00823152ff46106824d791bcb538e47b9f52e9df90220" + - "705de19cdc96c9dcb9f8dfff070e780c129524a56f369dc3633f69b1472c41d2012102" + - "90428148e18a34190de8a0b9a7ab7e9353feac7bd7bb57dde4acb2a03d1dfb94ffffff" + - "ff841e4ed900a6592687d7684ec0dd0893d7a3b86df07bb2626a1f51ae74ee0ba60c00" + - "00006a4730440220719b2baf0422e98d7462d120c3406e7e3657cceb8fabe84ddece55" + - "d34affb4620220168f87c8bcd57f903104b5effc0feffe3bd776dce6c09bfda8a3a36c" + - "95788ec701210263c387f7b141cf649a5296096976b6b622cd44a611c0e485724f9432" + - "9cf22b79ffffffff3e0ebec9b42f30657ef187bd263c57849541014700cc5480268dc6" + - "3ce6997552010000006a47304402206e62849a47494e9294bd7e47d87b9d32a6b69097" + - "067184b551350b7b34291d46022017ca461f6bcceb97432e9890700a68a4df4f4c5cfd" + - "272926af00a05dfdb3c07b0121024decad000f3aabfdac5ea8f132e457f0056b1f2cab" + - "0c946884d2ff0afbdd68b8ffffffff7e131188b66001c0f5a4b45bd3503d2ce17f40e4" + - "3e3803a7f30c20a8214abc95090000006b483045022100fa9c274892db835066eeac6c" + - "8f25d9507b43c93f4557b16560745e627387d7780220461025024a97816ff0567dcdca" + - "4702b5238894ee7028d04af48a7eb11bd3520301210391c94422cd25ee416178827513" + - "365ee49e4b2f17cdf0b07aed04353c10e34f6dffffffff06008c8647000000001976a9" + - "14e924c7d01f0201df5b1465aed50012c4dffa0ff988ac89ac7800000000001976a914" + - "3dc5ae7471acbba55107b651ce20177b3b44721388ac9f258509000000001976a91400" + - "656d12c4b48359f8637bcf96f9f02ceb2fad1c88ac4de9ec04000000001976a914f7de" + - "454e8b402cbde90f4a325de69236040f144088ac008c8647000000001976a914b7d9f8" + - "29f1bd6919728f91f8a10ce0ca87bde1aa88ac008c8647000000001976a914b877256a" + - "588a315e30b2c5aec38ff70cea71dd0c88ac00000000"), - ("4partycoinjoin", - "55eac9d4a4159d4ba355122c6c18f85293c19ae358306a3773ec3a5d053e2f1b", - "01000000108048bbbc26c394d45b514835d998ae2679c31cecd7623041372703f469aa" + - "ff8e040000006b4830450220293480c5975676ddbb8b66fc6e4bc53668ae4cc59d0da5" + - "377e520f3f360ad15e022100fe6aaf3d890570d66728210afb13e221f0405bbbf43262" + - "16371459877531a2d80121033392a42f2a8e4e93a5d7b50b4d26eabda42e04e110e77e" + - "22b514982a832dde92ffffffff587af588bbe3f911f8234a858cdc62a1c04c12cf09f5" + - "95f92a7292fd3f54d17e080000006b483045022100c47a07c3dd537e1e11216dcf3440" + - "619ec80a88d25f91f8b9de74b348947ba47902205e5a335e2c75b202b75ff8a50a2560" + - "f49542fe4e96598be6fe77b00bc7c33484012103e21d83d80666a020c3e4657f7155ac" + - "4572edf9fc79d59b5b69e27f23ff5f8df5ffffffff8048bbbc26c394d45b514835d998" + - "ae2679c31cecd7623041372703f469aaff8e0b0000006b48304502202a5403bfed1826" + - "c0d5b4143229518d6c0dc6b0881ad04e26039cef4981aa9487022100cecad7bd8fd476" + - "2257f83080bf2f56249bb36554137e9af9c1588f8f2ed170a9012103fd52e4f1e64cdd" + - "003528cf5f388ad455d001ea691bd02125d694117a1d481bbcffffffff8048bbbc26c3" + - "94d45b514835d998ae2679c31cecd7623041372703f469aaff8e110000006b48304502" + - "204e024b69b4a78ea9daa63ad1d8dba5e8729fbb61749dacc49be20053db8c02710221" + - "00c24555b43e6e838fd7cb797cbf96c278531af29493aaf719bdfe3354941d0b320121" + - "02a322a78564a976271153b0e289d631e5c6a217542d540c94262c40a83e48281affff" + - "ffff8048bbbc26c394d45b514835d998ae2679c31cecd7623041372703f469aaff8e05" + - "0000006b483045022100d3c7d326e233b2238576aecf53d5c3dfb88a3b41d815ff2cfd" + - "c3e2c5664aa52102204f0dc1cd110e4317ab435533a4440bbdabee75f360551bb46206" + - "79043058288801210310950ae84cfb02e24960c7227e40dedbc8804397a19e277233b1" + - "f3ecc797fdc2ffffffff9dd9d7a38ab7f29f17c4f32ae5be1426f80ff58c158fc7c74a" + - "afb9d53659cd05050000006b483045022100cc46ea21d4a1622cf7507ab9c96863b5fa" + - "3744c8c4b34a74c7bfdb92af41f40702200def9598dfcae25abaac047969f231c565c0" + - "6df6f7cb8f3c588e390049b0217801210202d1c96a5b459ca4fd430afc4d1b0e468a2a" + - "ba84c48ab5f5ad906ec09f55cfb3ffffffff84dc724e49f71d2a82cc46d781b1f47a21" + - "18c4056ab6dad2a83f03c7ed4a08c6040000006b483045022100ee5db129b8264b8a12" + - "e85fbf29566ad7edb84704214ced4dbea3e628149f08af02201149ab3c4e0aa6fa48ec" + - "4816ecefda6105a6df5838e9a6dbe235d156898d3b40012102da5240065576c7567c3e" + - "cc8dc09f3fda64a0c6b89c3974a7506bdeb7fb440704ffffffff68e787d07a48d3ba00" + - "d8160311ff0533684dd66c7ddba8f7d74af3862d98bd8a020000006b483045022100a8" + - "0edec0b7a6946fbcb877f95c48e22e26710f3f7a7ec9e5dd0e7622832fca3002204a8f" + - "ecc644f674cfa8c5ba0682318f5b6cd8cc5f0c15b70e5e423e3e0888e6550121032708" + - "198b4c046db76f9e0ff7350d05243950738b39c431f55d85d5ec886a1389ffffffff21" + - "57aecaef0b44114b1035d6630a436ee3a6ecbeebfca48b5ce8fa6dbc5e100700000000" + - "6b483045022100df2085a721c026328ea6dec9307c45d5991d51f696c15896ed14d40c" + - "dc69a3d302205a598476fa84a37c5eb70bab35d4a414fce63bc8bedec7883a724ea1f7" + - "d4b98401210271f768f5b16806df11ee81619c1371864707de25b27df94bd4516b6680" + - "2f0709ffffffff7cabc836092178f7325571627777476d1796e446ac8039dd706754b6" + - "824ee723040000006a473044022036eedb5259b74f8a99cba9f4866fafce94b5b7ebf7" + - "d248e973bdba27ac1161b202203d58951892c1680fafc77dd6ee1a9c27754018b96030" + - "a51c8891c1f5273c077f01210271f768f5b16806df11ee81619c1371864707de25b27d" + - "f94bd4516b66802f0709ffffffff84dc724e49f71d2a82cc46d781b1f47a2118c4056a" + - "b6dad2a83f03c7ed4a08c6000000006a473044022014a918294f28157288ab1e66e22b" + - "fe5e5cb92dde076e133b5d1d807bfd252048022020f8ef9d0c094b476b31f177fb083c" + - "b6e65f5167a086fd25bebe5012e46ae8b601210271f768f5b16806df11ee81619c1371" + - "864707de25b27df94bd4516b66802f0709ffffffff4c8670675a981c70935cbd0a0de4" + - "98c9b28c41784d590f1b9e59930e64dd53f5020000006a473044022067b08694963116" + - "f7d53522c2327b46150084b7b9cf5b279daa1461492dffd7010220453b1fd98cdc26fe" + - "d9385ec578da1dd11d1ca7367ebfd5586f4ddb913feabb6c01210397db984e478086ca" + - "f0bf90181c3c597905a52416e152bff007dffd8edf06e2deffffffff8048bbbc26c394" + - "d45b514835d998ae2679c31cecd7623041372703f469aaff8e070000006c4930460221" + - "0094e5123e70426ff6732bebbcaaac2d871b873880d77c71a97507a074fe1046000221" + - "00c67b7d7663817507dd03f00fd2b690d33daa645a13b6966ec1c76e5a8df0b0a40121" + - "035a9e5cdd37824ad390d73abf2c7bc902de6dd7e4ab92fcf4e84506747e8c6cdaffff" + - "ffff8048bbbc26c394d45b514835d998ae2679c31cecd7623041372703f469aaff8e01" + - "0000006b483045022100cfab7cb7b645e201515b756439ba943fecb77e68c76c482de5" + - "225bf7a80ca94a022061600e67eaf33b59c5863de617bdb79f63e69b310710cb2b7f4f" + - "63d9d64fb3bd012102dabd952ef6ebfe396dfd66436089f5df731dd8300ecb6d88eefe" + - "458bc73a4c8fffffffff11565b2030be339919eaa5d1c1b47f7d265485c2e230fe44d9" + - "5ce41f3f6d5bdc030000006b483045022021d350b215077c4b1aee83c747a0cabbe148" + - "2fc4ccc97657f07d481df3f89d45022100b16d629e35e857506af6e8dd74373f370f4b" + - "965f332a488878dc983f554378c40121022cde2859fb62f3b671a887fcce054626cdb2" + - "8ed4d20d0dfcf8e0b090e2d7fe4effffffff8048bbbc26c394d45b514835d998ae2679" + - "c31cecd7623041372703f469aaff8e0c0000006a47304402203adf6153783989ca7ec1" + - "fa12785e5abae5c6bd8a4e26bb0c2ae0cbc4635af981022056faa34a713bede42d0efe" + - "af84c35506f47d487bed59fba2de53167fdad64c570121025d3368bbb24f5980fc6081" + - "d2a1e0e04675d11751a53d13d896cff7b5940ac205ffffffff1cb00400000000000019" + - "76a9146418ea7a9cbcfbd9dfb4f552c00b834072ac833a88ac40420f00000000001976" + - "a91492f8428eae5228083b588b330f9dfc1cccd300fc88ac40420f00000000001976a9" + - "14871e08ec7dd94ef87fd7722ee703007badb7020d88ace8030000000000001976a914" + - "2ced0ea9992d7e5492b9053437702c83ea0750c088ac40420f00000000001976a91449" + - "464457f02609ba1d54153eaba614ab84d593cb88ac877fc30f000000001976a9141c20" + - "8af75037715bdca6c42f29e104af62204e4788ac540b0000000000001976a914cfccd5" + - "185a3c4ced7514fa584eee6ebd895c984888ac73037000000000001976a9149f0ddeb2" + - "473d013452bce71973d7515c0b4cd5cf88ac40420f00000000001976a9142cb4e342c4" + - "398a08397a8cedfd270e432e977da488ac09c2e005000000001976a914ca9ec96314a5" + - "f4842e3fe33a0cf6d047a4398d0a88ac40420f00000000001976a914cd0124b0969f16" + - "4132301032a0f99de5c8d8471e88ace9320400000000001976a9145c5f87677c7b50bc" + - "4fe88bdec9f2221a0ca3c35088ac40420f00000000001976a914fa40beb4e557e14d5e" + - "461c7aa66d674ea6b7ce5988ac40420f00000000001976a914564bdac0f31da4f35ce5" + - "3c9aaef7e796ed7acdcd88ace33c2b00000000001976a9142ef77132983b843b07396b" + - "731d213e3a60b4c6e488ac40420f00000000001976a9144aa2c6a81b4ad8c9414b1039" + - "d5b2c51d3416233888ac40420f00000000001976a914713667aa801d3d7cbf9c3d1667" + - "626131a9937a6288ac40420f00000000001976a9142fa24ea7272cbb3ee06beeb274b3" + - "486c3d83f60188ac40420f00000000001976a9147c2e0ee7484234b0a20fab40855dff" + - "93c3b195da88ac40420f00000000001976a914123d15de9b64aea49a9cf81d781fdfad" + - "00287acf88ac40420f00000000001976a91463f456082de5dbd63c7ab9cd511813d836" + - "a2a7cd88ac306a2e00000000001976a9144103b28ac18a4ad0ce075fe323ee3d38a95c" + - "4d2188acc0541d00000000001976a91484d1b7dadb4638895fc5eb79145dca39daea86" + - "ca88acb80b0000000000001976a9146baff4a7a65316aed38c032c6eded2e48111fb62" + - "88ac34080000000000001976a914a5e54586ea1626fc90359f55fa4a291c99cffae488" + - "accdc30100000000001976a914d547e4e5b7eb382f9309622bdf0271836083936988ac" + - "40420f00000000001976a914a0b17374b3085de303202b29439957ccd71ba13088acde" + - "af2600000000001976a914af4d27ced320ae73068d0e2bb2c3abfe6fbe2c6988ac0000" + - "0000"), - ("p2sh-in-to-p2sh-out", - "355a6090bfdb77d1d7fd7c67ba6a711b9f493344e372928fe79ca0206b34796b", - "0100000001b5bbd7adb1f9f6f599d8e547f7f789ba9b63da9f036e8b739477bb8295fd" + - "4a2600000000fdfd0000473044022062a3ca46f976d4719d94fa67c9360c65b0946e3a" + - "7877f7f2c4cc37b8c77dede302203af47e6c6ccd804726295eb99f90dc198c86414d85" + - "7e21bc617f26903d380b0901483045022100af39dc96cf32aed7165a7644dcdfa939cb" + - "7e6af8236126644be937ff8a10b3a10220204b03b1a6b8acc7748e3c76ee968f9e9997" + - "a81c6e944910d2dc626133aa82bf014c695221021de4261f30b149c3f14f93b1a06a5e" + - "96aa24780c789343f80c341043d46d700a21036daab402e66c56470eda26d981f80a1b" + - "8f224f21fdbe64329b0481b1b37d8a0b2103c72347824e9099dbdfad5a2dac56e5d056" + - "8c9c100caa7490f62f2731c23af27353aeffffffff028a7faf00000000001976a91472" + - "2a53477336bd4960bed86be7fc47b8ece07ba688acbda438010000000017a9148a6425" + - "ccc0cb64097c29579a40e733819d4e07c58700000000"), - ("op-return-output", - "5be05925b8ef04f326b19e2c57f9ce1f3e2024aa992dd0d744f7942e92a200a5", - "0100000001067c0bd822cdcfabf0978bdede3a7b395a5c175219fd704b1ee47f137858" + - "6362000000006b483045022100dffb89e46c734b6429b6322bce9b6343c98d0d9eed12" + - "059ac2b9dfcffaa5577702204350bc86aea9a7e3b2868a643d27c5a36bd3105c8a2a2c" + - "5daf6e8441c83c870f012102a1260627f8845765759454a2ee47603312bbd7fdad0fa0" + - "0f3ad0cea0c5602d2cffffffff03e9690700000000001976a91497d34e7a0c8082f180" + - "546040030afb925c0f2dd888ac0000000000000000166a146f6d6e6900000000000000" + - "0300000000000186a0aa0a0000000000001976a91488d924f51033b74a895863a5fb57" + - "fd545529df7d88ac00000000")]) -def test_serialization_roundtrip(tx_type, tx_id, tx_hex): - assert tx_hex == btc.serialize(btc.deserialize(tx_hex)) - -@pytest.mark.parametrize( - "ins, outs, txtype, valid", - [ - (4, 3, "p2pkh", True), - (4, 3, "p2sh", False), - ]) -def test_estimate_tx_size(ins, outs, txtype, valid): - #TODO: this function should throw on invalid number of ins or outs - if valid: - assert btc.estimate_tx_size(ins, outs, txtype)== 10 + 147*ins + 34*outs - else: - with pytest.raises(NotImplementedError) as e_info: - btc.estimate_tx_size(ins, outs, txtype) - - -def test_serialization_roundtrip2(): - #Data extracted from: - #https://github.com/bitcoin/bitcoin/blob/master/src/test/data/tx_valid.json - #These are a variety of rather strange edge case transactions, which are - #still valid. - #Note that of course this is only a serialization, not validity test, so - #only currently of very limited significance - with open(os.path.join(testdir,"tx_valid.json"), "r") as f: - json_data = f.read() - valid_txs = json.loads(json_data) - for j in valid_txs: - #ignore comment entries - if len(j) < 2: - continue - print(j) - deserialized = btc.deserialize(str(j[0])) - print(deserialized) - assert j[0] == btc.serialize(deserialized) diff --git a/jmbitcoin/test/test_tx_signing.py b/jmbitcoin/test/test_tx_signing.py new file mode 100644 index 0000000..4db7f3c --- /dev/null +++ b/jmbitcoin/test/test_tx_signing.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 + +import sys +import pytest + +import binascii +import hashlib +import jmbitcoin as btc + +@pytest.mark.parametrize( + "addrtype", + [("p2wpkh"), + ("p2sh-p2wpkh"), + ("p2pkh"), + ]) +def test_sign_standard_txs(addrtype): + # liberally copied from python-bitcoinlib tests, + # in particular see: + # https://github.com/petertodd/python-bitcoinlib/pull/227 + + # Create the (in)famous correct brainwallet secret key. + priv = hashlib.sha256(b'correct horse battery staple').digest() + b"\x01" + pub = btc.privkey_to_pubkey(priv) + + # Create an address from that private key. + # (note that the input utxo is fake so we are really only creating + # a destination here). + scriptPubKey = btc.CScript([btc.OP_0, btc.Hash160(pub)]) + address = btc.P2WPKHBitcoinAddress.from_scriptPubKey(scriptPubKey) + + # Create a dummy outpoint; use same 32 bytes for convenience + txid = priv[:32] + vout = 2 + amount = btc.coins_to_satoshi(float('0.12345')) + + # Calculate an amount for the upcoming new UTXO. Set a high fee to bypass + # bitcoind minfee setting. + amount_less_fee = int(amount - btc.coins_to_satoshi(0.01)) + + # Create a destination to send the coins. + destination_address = address + target_scriptPubKey = scriptPubKey + + # Create the unsigned transaction. + txin = btc.CTxIn(btc.COutPoint(txid[::-1], vout)) + txout = btc.CTxOut(amount_less_fee, target_scriptPubKey) + tx = btc.CMutableTransaction([txin], [txout]) + + # Calculate the signature hash for the transaction. This is then signed by the + # private key that controls the UTXO being spent here at this txin_index. + if addrtype == "p2wpkh": + sig, msg = btc.sign(tx, 0, priv, amount=amount, native=True) + elif addrtype == "p2sh-p2wpkh": + sig, msg = btc.sign(tx, 0, priv, amount=amount, native=False) + elif addrtype == "p2pkh": + sig, msg = btc.sign(tx, 0, priv) + else: + assert False + if not sig: + print(msg) + raise + print("created signature: ", binascii.hexlify(sig)) + print("serialized transaction: {}".format(btc.b2x(tx.serialize()))) + +def test_mk_shuffled_tx(): + # prepare two addresses for the outputs + pub = btc.privkey_to_pubkey(btc.Hash(b"priv") + b"\x01") + scriptPubKey = btc.CScript([btc.OP_0, btc.Hash160(pub)]) + addr1 = btc.P2WPKHBitcoinAddress.from_scriptPubKey(scriptPubKey) + scriptPubKey_p2sh = scriptPubKey.to_p2sh_scriptPubKey() + addr2 = btc.CCoinAddress.from_scriptPubKey(scriptPubKey_p2sh) + + ins = [(btc.Hash(b"blah"), 7), (btc.Hash(b"foo"), 15)] + # note the casts str() ; most calls to mktx will have addresses fed + # as strings, so this is enforced for simplicity. + outs = [{"address": str(addr1), "value": btc.coins_to_satoshi(float("0.1"))}, + {"address": str(addr2), "value": btc.coins_to_satoshi(float("45981.23331234"))}] + tx = btc.make_shuffled_tx(ins, outs, version=2, locktime=500000) + +def test_bip143_tv(): + # p2sh-p2wpkh case: + rawtx_hex = "0100000001db6b1b20aa0fd7b23880be2ecbd4a98130974cf4748fb66092ac4d3ceb1a54770100000000feffffff02b8b4eb0b000000001976a914a457b684d7f0d539a46a45bbc043f35b59d0d96388ac0008af2f000000001976a914fd270b1ee6abcaea97fea7ad0402e8bd8ad6d77c88ac92040000" + inp_spk_hex = "a9144733f37cf4db86fbc2efed2500b4f4e49f31202387" + value = 10 + redeemScript = "001479091972186c449eb1ded22b78e40d009bdf0089" + privkey_hex = "eb696a065ef48a2192da5b28b694f87544b30fae8327c4510137a922f32c6dcf01" + pubkey_hex = "03ad1d8e89212f0b92c74d23bb710c00662ad1470198ac48c43f7d6f93a2a26873" + tx = btc.CMutableTransaction.deserialize(btc.x(rawtx_hex)) + btc.sign(tx, 0, btc.x(privkey_hex), amount=btc.coins_to_satoshi(10), native=False) + expectedsignedtx = "01000000000101db6b1b20aa0fd7b23880be2ecbd4a98130974cf4748fb66092ac4d3ceb1a5477010000001716001479091972186c449eb1ded22b78e40d009bdf0089feffffff02b8b4eb0b000000001976a914a457b684d7f0d539a46a45bbc043f35b59d0d96388ac0008af2f000000001976a914fd270b1ee6abcaea97fea7ad0402e8bd8ad6d77c88ac02473044022047ac8e878352d3ebbde1c94ce3a10d057c24175747116f8288e5d794d12d482f0220217f36a485cae903c713331d877c1f64677e3622ad4010726870540656fe9dcb012103ad1d8e89212f0b92c74d23bb710c00662ad1470198ac48c43f7d6f93a2a2687392040000" + assert btc.b2x(tx.serialize()) == expectedsignedtx \ No newline at end of file diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index f4dd264..0b73d68 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -18,7 +18,8 @@ from .wallet import (Mnemonic, estimate_tx_fee, WalletError, BaseWallet, ImportW UTXOManager, WALLET_IMPLEMENTATIONS, compute_tx_locktime) from .storage import (Argon2Hash, Storage, StorageError, RetryableStorageError, StoragePasswordError, VolatileStorage) -from .cryptoengine import BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, EngineError +from .cryptoengine import (BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, EngineError, + TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH) from .configure import (load_test_config, load_program_config, get_p2pk_vbyte, jm_single, get_network, update_persist_config, validate_address, is_burn_destination, get_irc_mchannels, diff --git a/jmclient/jmclient/blockchaininterface.py b/jmclient/jmclient/blockchaininterface.py index 777530a..56ddff0 100644 --- a/jmclient/jmclient/blockchaininterface.py +++ b/jmclient/jmclient/blockchaininterface.py @@ -6,7 +6,7 @@ import time from decimal import Decimal import binascii from twisted.internet import reactor, task - +from jmbase import bintohex, hextobin import jmbitcoin as btc from jmclient.jsonrpc import JsonRpcConnectionError, JsonRpcError @@ -297,9 +297,8 @@ class BitcoinCoreInterface(BlockchainInterface): if not "hex" in rpcretval: log.info("Malformed gettransaction output") return None - #str cast for unicode - hexval = str(rpcretval["hex"]) - return btc.deserialize(hexval) + return btc.CMutableTransaction.deserialize( + hextobin(rpcretval["hex"])) def list_transactions(self, num, skip=0): """ Return a list of the last `num` transactions seen @@ -309,20 +308,22 @@ class BitcoinCoreInterface(BlockchainInterface): return self.rpc("listtransactions", ["*", num, skip, True]) def get_transaction(self, txid): - """ Returns a serialized transaction for txid txid, + """ Argument txid is passed in binary. + Returns a serialized transaction for txid txid, in hex as returned by Bitcoin Core rpc, or None if no transaction can be retrieved. Works also for watch-only wallets. """ + htxid = bintohex(txid) #changed syntax in 0.14.0; allow both syntaxes try: - res = self.rpc("gettransaction", [txid, True]) - except: + res = self.rpc("gettransaction", [htxid, True]) + except Exception as e: try: - res = self.rpc("gettransaction", [txid, 1]) + res = self.rpc("gettransaction", [htxid, 1]) except JsonRpcError as e: #This should never happen (gettransaction is a wallet rpc). - log.warn("Failed gettransaction call; JsonRpcError") + log.warn("Failed gettransaction call; JsonRpcError: " + repr(e)) return None except Exception as e: log.warn("Failed gettransaction call; unexpected error:") @@ -333,7 +334,11 @@ class BitcoinCoreInterface(BlockchainInterface): return None return res - def pushtx(self, txhex): + def pushtx(self, txbin): + """ Given a binary serialized valid bitcoin transaction, + broadcasts it to the network. + """ + txhex = bintohex(txbin) try: txid = self.rpc('sendrawtransaction', [txhex]) except JsonRpcConnectionError as e: @@ -345,9 +350,9 @@ class BitcoinCoreInterface(BlockchainInterface): return True def query_utxo_set(self, txout, includeconf=False, includeunconf=False): - """If txout is either (a) a single string in hex encoded txid:n form, + """If txout is either (a) a single utxo in (txidbin, n) form, or a list of the same, returns, as a list for each txout item, - the result of gettxout from the bitcoind rpc for those utxs; + the result of gettxout from the bitcoind rpc for those utxos; if any utxo is invalid, None is returned. includeconf: if this is True, the current number of confirmations of the prescribed utxo is included in the returned result dict. @@ -360,16 +365,18 @@ class BitcoinCoreInterface(BlockchainInterface): txout = [txout] result = [] for txo in txout: - if len(txo) < 66: + txo_hex = bintohex(txo[0]) + if len(txo_hex) != 64: + log.warn("Invalid utxo format, ignoring: {}".format(txo)) result.append(None) continue try: - txo_idx = int(txo[65:]) + txo_idx = int(txo[1]) except ValueError: log.warn("Invalid utxo format, ignoring: {}".format(txo)) result.append(None) continue - ret = self.rpc('gettxout', [txo[:64], txo_idx, includeunconf]) + ret = self.rpc('gettxout', [txo_hex, txo_idx, includeunconf]) if ret is None: result.append(None) else: @@ -380,7 +387,7 @@ class BitcoinCoreInterface(BlockchainInterface): result_dict = {'value': int(Decimal(str(ret['value'])) * Decimal('1e8')), 'address': address, - 'script': ret['scriptPubKey']['hex']} + 'script': hextobin(ret['scriptPubKey']['hex'])} if includeconf: result_dict['confirms'] = int(ret['confirmations']) result.append(result_dict) diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index 8783cfc..9711d9b 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -9,16 +9,17 @@ try: except ImportError: pass from jmbase import commands - +import binascii import json import hashlib import os import sys -from jmbase import get_log +from jmbase import (get_log, EXIT_FAILURE, hextobin, bintohex, + utxo_to_utxostr, dictchanger) from jmclient import (jm_single, get_irc_mchannels, RegtestBitcoinCoreInterface) import jmbitcoin as btc -from jmbase.support import EXIT_FAILURE + jlog = get_log() @@ -30,7 +31,8 @@ class JMClientProtocol(amp.AMP): self.client = client self.factory = factory if not nick_priv: - self.nick_priv = hashlib.sha256(os.urandom(16)).hexdigest() + '01' + self.nick_priv = hashlib.sha256( + os.urandom(16)).digest() + b"\x01" else: self.nick_priv = nick_priv @@ -59,10 +61,18 @@ class JMClientProtocol(amp.AMP): self.clientStart() def set_nick(self): - self.nick_pubkey = btc.privtopub(self.nick_priv) - self.nick_pkh_raw = btc.bin_sha256(self.nick_pubkey)[ - :self.nick_hashlen] - self.nick_pkh = btc.b58encode(self.nick_pkh_raw) + """ Algorithm: take pubkey and hex-serialized it; + then SHA2(hexpub) but truncate output to nick_hashlen. + Then encode to a base58 string (no check). + Then prepend J and version char (e.g. '5'). + Finally append padding to nick_maxencoded (+2). + """ + self.nick_pubkey = btc.privkey_to_pubkey(self.nick_priv) + # note we use binascii hexlify directly here because input + # to hashing must be encoded. + self.nick_pkh_raw = hashlib.sha256(binascii.hexlify( + self.nick_pubkey)).digest()[:self.nick_hashlen] + self.nick_pkh = btc.base58.encode(self.nick_pkh_raw) #right pad to maximum possible; b58 is not fixed length. #Use 'O' as one of the 4 not included chars in base58. self.nick_pkh += 'O' * (self.nick_maxencoded - len(self.nick_pkh)) @@ -93,7 +103,7 @@ class JMClientProtocol(amp.AMP): @commands.JMRequestMsgSig.responder def on_JM_REQUEST_MSGSIG(self, nick, cmd, msg, msg_to_be_signed, hostid): sig = btc.ecdsa_sign(str(msg_to_be_signed), self.nick_priv) - msg_to_return = str(msg) + " " + self.nick_pubkey + " " + sig + msg_to_return = str(msg) + " " + bintohex(self.nick_pubkey) + " " + sig d = self.callRemote(commands.JMMsgSignature, nick=nick, cmd=cmd, @@ -105,20 +115,21 @@ class JMClientProtocol(amp.AMP): @commands.JMRequestMsgSigVerify.responder def on_JM_REQUEST_MSGSIG_VERIFY(self, msg, fullmsg, sig, pubkey, nick, hashlen, max_encoded, hostid): + pubkey_bin = hextobin(pubkey) verif_result = True - if not btc.ecdsa_verify(str(msg), sig, pubkey): + if not btc.ecdsa_verify(str(msg), sig, pubkey_bin): # workaround for hostid, which sometimes is lowercase-only for some IRC connections - if not btc.ecdsa_verify(str(msg[:-len(hostid)] + hostid.lower()), sig, pubkey): + if not btc.ecdsa_verify(str(msg[:-len(hostid)] + hostid.lower()), sig, pubkey_bin): jlog.debug("nick signature verification failed, ignoring: " + str(nick)) verif_result = False #check that nick matches hash of pubkey - nick_pkh_raw = btc.bin_sha256(pubkey)[:hashlen] + nick_pkh_raw = hashlib.sha256(pubkey.encode("ascii")).digest()[:hashlen] nick_stripped = nick[2:2 + max_encoded] #strip right padding nick_unpadded = ''.join([x for x in nick_stripped if x != 'O']) - if not nick_unpadded == btc.b58encode(nick_pkh_raw): + if not nick_unpadded == btc.base58.encode(nick_pkh_raw): jlog.debug("Nick hash check failed, expected: " + str(nick_unpadded) - + ", got: " + str(btc.b58encode(nick_pkh_raw))) + + ", got: " + str(btc.base58.encode(nick_pkh_raw))) verif_result = False d = self.callRemote(commands.JMMsgSignatureVerify, verif_result=verif_result, @@ -234,10 +245,18 @@ class JMMakerClientProtocol(JMClientProtocol): jlog.info("Maker refuses to continue on receiving auth.") else: utxos, auth_pub, cj_addr, change_addr, btc_sig = retval[1:] + # json does not allow non-string keys: + utxos_strkeyed = {} + for k in utxos: + success, u = utxo_to_utxostr(k) + assert success + utxos_strkeyed[u] = {"value": utxos[k]["value"], + "address": utxos[k]["address"]} + auth_pub_hex = bintohex(auth_pub) d = self.callRemote(commands.JMIOAuth, nick=nick, - utxolist=json.dumps(utxos), - pubkey=auth_pub, + utxolist=json.dumps(utxos_strkeyed), + pubkey=auth_pub_hex, cjaddr=cj_addr, changeaddr=change_addr, pubkeysig=btc_sig) @@ -258,14 +277,14 @@ class JMMakerClientProtocol(JMClientProtocol): else: sigs = retval[1] self.finalized_offers[nick] = offer - tx = btc.deserialize(txhex) + tx = btc.CMutableTransaction.deserialize(hextobin(txhex)) self.finalized_offers[nick]["txd"] = tx - txid = btc.txhash(btc.serialize(tx)) + txid = tx.GetTxid()[::-1] # we index the callback by the out-set of the transaction, # because the txid is not known until all scriptSigs collected # (hence this is required for Makers, but not Takers). # For more info see WalletService.transaction_monitor(): - txinfo = tuple((x["script"], x["value"]) for x in tx["outs"]) + txinfo = tuple((x.scriptPubKey, x.nValue) for x in tx.vout) self.client.wallet_service.register_callbacks([self.unconfirm_callback], txinfo, "unconfirmed") self.client.wallet_service.register_callbacks([self.confirm_callback], @@ -285,8 +304,8 @@ class JMMakerClientProtocol(JMClientProtocol): def tx_match(self, txd): for k,v in iteritems(self.finalized_offers): - #Tx considered defined by its output set - if v["txd"]["outs"] == txd["outs"]: + # Tx considered defined by its output set + if v["txd"].vout == txd.vout: offerinfo = v break else: @@ -302,7 +321,7 @@ class JMMakerClientProtocol(JMClientProtocol): txid) self.client.modify_orders(to_cancel, to_announce) - txinfo = tuple((x["script"], x["value"]) for x in txd["outs"]) + txinfo = tuple((x.scriptPubKey, x.nValue) for x in txd.vout) confirm_timeout_sec = float(jm_single().config.get( "TIMEOUT", "confirm_timeout_hours")) * 3600 task.deferLater(reactor, confirm_timeout_sec, diff --git a/jmclient/jmclient/commitment_utils.py b/jmclient/jmclient/commitment_utils.py index 64a80c4..ab598ad 100644 --- a/jmclient/jmclient/commitment_utils.py +++ b/jmclient/jmclient/commitment_utils.py @@ -2,8 +2,10 @@ import sys import jmbitcoin as btc from jmbase import jmprint -from jmclient import jm_single, get_p2pk_vbyte, get_p2sh_vbyte -from jmbase.support import EXIT_FAILURE +from jmclient import (jm_single, get_p2pk_vbyte, get_p2sh_vbyte, + BTCEngine, TYPE_P2PKH, TYPE_P2SH_P2WPKH, + BTC_P2PKH, BTC_P2SH_P2WPKH) +from jmbase.support import EXIT_FAILURE, utxostr_to_utxo def quit(parser, errmsg): #pragma: no cover @@ -18,17 +20,16 @@ def get_utxo_info(upriv): u, priv = upriv.split(',') u = u.strip() priv = priv.strip() - txid, n = u.split(':') - assert len(txid)==64 - assert len(n) in range(1, 4) - n = int(n) - assert n in range(256) + success, utxo = utxostr_to_utxo(u) + assert success, utxo except: #not sending data to stdout in case privkey info jmprint("Failed to parse utxo information for utxo", "error") raise try: - hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte()) + # see note below for why keytype is ignored, and note that + # this calls read_privkey to validate. + raw, _ = BTCEngine.wif_to_privkey(priv) except: jmprint("failed to parse privkey, make sure it's WIF compressed format.", "error") raise @@ -40,16 +41,20 @@ def validate_utxo_data(utxo_datas, retrieve=False, segwit=False): then use the blockchain instance to look up the utxo and check that its address field matches. If retrieve is True, return the set of utxos and their values. + If segwit is true, assumes a p2sh wrapped p2wpkh, i.e. + native segwit is NOT currently supported here. If segwit + is false, p2pkh is assumed. """ results = [] for u, priv in utxo_datas: jmprint('validating this utxo: ' + str(u), "info") - hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte()) - if segwit: - addr = btc.pubkey_to_p2sh_p2wpkh_address( - btc.privkey_to_pubkey(hexpriv), get_p2sh_vbyte()) - else: - addr = btc.privkey_to_address(hexpriv, magicbyte=get_p2pk_vbyte()) + # as noted in `ImportWalletMixin` code comments, there is not + # yet a functional auto-detection of key type from WIF, so the + # second argument is ignored; we assume p2sh-p2wpkh if segwit, + # else we assume p2pkh. + engine = BTC_P2SH_P2WPKH if segwit else BTC_P2PKH + rawpriv, _ = BTCEngine.wif_to_privkey(priv) + addr = engine.privkey_to_address(rawpriv) jmprint('claimed address: ' + addr, "info") res = jm_single().bc_interface.query_utxo_set([u]) if len(res) != 1 or None in res: diff --git a/jmclient/jmclient/configure.py b/jmclient/jmclient/configure.py index 48372b2..45f5df2 100644 --- a/jmclient/jmclient/configure.py +++ b/jmclient/jmclient/configure.py @@ -377,33 +377,18 @@ def get_p2pk_vbyte(): def validate_address(addr): try: - assert len(addr) > 2 - if addr[:2].lower() in ['bc', 'tb']: - # Regtest special case - if addr[:4] == 'bcrt': - if btc.bech32addr_decode('bcrt', addr)[1]: - return True, 'address validated' - return False, 'Invalid bech32 regtest address' - #Else, enforce testnet/mainnet per config - if get_network() == "testnet": - hrpreq = 'tb' - else: - hrpreq = 'bc' - if btc.bech32addr_decode(hrpreq, addr)[1]: - return True, 'address validated' - return False, 'Invalid bech32 address' - #Not bech32; assume b58 from here - ver = btc.get_version_byte(addr) - except AssertionError: - return False, 'Checksum wrong. Typo in address?' - except Exception: - return False, "Invalid bitcoin address" - if ver != get_p2pk_vbyte() and ver != get_p2sh_vbyte(): - return False, 'Wrong address version. Testnet/mainnet confused?' - if len(btc.b58check_to_bin(addr)) != 20: - return False, "Address has correct checksum but wrong length." - return True, 'address validated' - + # automatically respects the network + # as set in btc.select_chain_params(...) + x = btc.CCoinAddress(addr) + except Exception as e: + return False, repr(e) + # additional check necessary because python-bitcointx + # does not check hash length on p2sh construction. + try: + x.to_scriptPubKey() + except Exception as e: + return False, repr(e) + return True, "address validated" _BURN_DESTINATION = "BURN" @@ -565,6 +550,7 @@ def get_blockchain_interface_instance(_config): source = _config.get("BLOCKCHAIN", "blockchain_source") network = get_network() testnet = network == 'testnet' + if source in ('bitcoin-rpc', 'regtest', 'bitcoin-rpc-no-history'): rpc_host = _config.get("BLOCKCHAIN", "rpc_host") rpc_port = _config.get("BLOCKCHAIN", "rpc_port") @@ -574,10 +560,20 @@ def get_blockchain_interface_instance(_config): rpc_wallet_file) if source == 'bitcoin-rpc': #pragma: no cover bc_interface = BitcoinCoreInterface(rpc, network) + if testnet: + btc.select_chain_params("bitcoin/testnet") + else: + btc.select_chain_params("bitcoin") elif source == 'regtest': bc_interface = RegtestBitcoinCoreInterface(rpc) + btc.select_chain_params("bitcoin/regtest") elif source == "bitcoin-rpc-no-history": bc_interface = BitcoinCoreNoHistoryInterface(rpc, network) + if testnet or network == "regtest": + # TODO will not work for bech32 regtest addresses: + btc.select_chain_params("bitcoin/testnet") + else: + btc.select_chain_params("bitcoin") else: assert 0 elif source == 'electrum': diff --git a/jmclient/jmclient/cryptoengine.py b/jmclient/jmclient/cryptoengine.py index 9b5d9f1..c24d108 100644 --- a/jmclient/jmclient/cryptoengine.py +++ b/jmclient/jmclient/cryptoengine.py @@ -19,20 +19,30 @@ NET_MAP = {'mainnet': NET_MAINNET, 'testnet': NET_TESTNET} WIF_PREFIX_MAP = {'mainnet': b'\x80', 'testnet': b'\xef'} BIP44_COIN_MAP = {'mainnet': 2**31, 'testnet': 2**31 + 1} -def detect_script_type(script): - if script.startswith(btc.P2PKH_PRE) and script.endswith(btc.P2PKH_POST) and\ - len(script) == 0x14 + len(btc.P2PKH_PRE) + len(btc.P2PKH_POST): +def detect_script_type(script_str): + """ Given a scriptPubKey, decide which engine + to use, one of: p2pkh, p2sh-p2wpkh, p2wpkh. + Note that for the p2sh case, we are assuming the nature + of the redeem script (p2wpkh wrapped) because that is what + we support; but we can't know for sure, from the sPK only. + Raises EngineError if the type cannot be detected, so + callers MUST handle this exception to avoid crashes. + """ + script = btc.CScript(script_str) + if not script.is_valid(): + raise EngineError("Unknown script type for script '{}'" + .format(hexlify(script_str))) + if script.is_p2pkh(): return TYPE_P2PKH - elif (script.startswith(btc.P2SH_P2WPKH_PRE) and - script.endswith(btc.P2SH_P2WPKH_POST) and - len(script) == 0x14 + len(btc.P2SH_P2WPKH_PRE) + len( - btc.P2SH_P2WPKH_POST)): + elif script.is_p2sh(): + # see note above. + # note that is_witness_v0_nested_keyhash does not apply, + # since that picks up scriptSigs not scriptPubKeys. return TYPE_P2SH_P2WPKH - elif script.startswith(btc.P2WPKH_PRE) and\ - len(script) == 0x14 + len(btc.P2WPKH_PRE): + elif script.is_witness_v0_keyhash(): return TYPE_P2WPKH raise EngineError("Unknown script type for script '{}'" - .format(hexlify(script))) + .format(hexlify(script_str))) class classproperty(object): """ @@ -97,28 +107,38 @@ class BTCEngine(object): @staticmethod def privkey_to_pubkey(privkey): - return btc.privkey_to_pubkey(privkey, False) + return btc.privkey_to_pubkey(privkey) @staticmethod def address_to_script(addr): - return unhexlify(btc.address_to_script(addr)) + return btc.CCoinAddress(addr).to_scriptPubKey() @classmethod def wif_to_privkey(cls, wif): - raw = btc.b58check_to_bin(wif) + raw = btc.b58check_to_bin(wif)[1] + # see note to `privkey_to_wif`; same applies here. + # We only handle valid private keys, not any byte string. + btc.read_privkey(raw) + vbyte = struct.unpack('B', btc.get_version_byte(wif))[0] - if (struct.unpack('B', btc.BTC_P2PK_VBYTE[get_network()])[0] + struct.unpack('B', cls.WIF_PREFIX)[0]) & 0xff == vbyte: + if (struct.unpack('B', btc.BTC_P2PK_VBYTE[get_network()])[0] + \ + struct.unpack('B', cls.WIF_PREFIX)[0]) & 0xff == vbyte: key_type = TYPE_P2PKH - elif (struct.unpack('B', btc.BTC_P2SH_VBYTE[get_network()])[0] + struct.unpack('B', cls.WIF_PREFIX)[0]) & 0xff == vbyte: + elif (struct.unpack('B', btc.BTC_P2SH_VBYTE[get_network()])[0] + \ + struct.unpack('B', cls.WIF_PREFIX)[0]) & 0xff == vbyte: key_type = TYPE_P2SH_P2WPKH else: key_type = None - return raw, key_type @classmethod def privkey_to_wif(cls, priv): + # refuse to WIF-ify something that we don't recognize + # as a private key; ignoring the return value of this + # function as we only want to raise whatever Exception + # it does: + btc.read_privkey(priv) return btc.bin_to_b58check(priv, cls.WIF_PREFIX) @classmethod @@ -166,12 +186,12 @@ class BTCEngine(object): @classmethod def privkey_to_address(cls, privkey): script = cls.key_to_script(privkey) - return btc.script_to_address(script, cls.VBYTE) + return str(btc.CCoinAddress.from_scriptPubKey(script)) @classmethod def pubkey_to_address(cls, pubkey): script = cls.pubkey_to_script(pubkey) - return btc.script_to_address(script, cls.VBYTE) + return str(btc.script_to_address(script, cls.VBYTE)) @classmethod def pubkey_has_address(cls, pubkey, addr): @@ -203,7 +223,12 @@ class BTCEngine(object): @classmethod def script_to_address(cls, script): - return btc.script_to_address(script, vbyte=cls.VBYTE) + """ a script passed in as binary converted to a + Bitcoin address of the appropriate type. + """ + s = btc.CScript(script) + assert s.is_valid() + return str(btc.CCoinAddress.from_scriptPubKey(s)) class BTC_P2PKH(BTCEngine): @@ -213,6 +238,7 @@ class BTC_P2PKH(BTCEngine): @classmethod def pubkey_to_script(cls, pubkey): + # this call does not enforce compressed: return btc.pubkey_to_p2pkh_script(pubkey) @classmethod @@ -222,7 +248,7 @@ class BTC_P2PKH(BTCEngine): @classmethod def sign_transaction(cls, tx, index, privkey, *args, **kwargs): hashcode = kwargs.get('hashcode') or btc.SIGHASH_ALL - return btc.sign(btc.serialize(tx), index, privkey, + return btc.sign(tx, index, privkey, hashcode=hashcode, amount=None, native=False) @@ -250,8 +276,9 @@ class BTC_P2SH_P2WPKH(BTCEngine): def sign_transaction(cls, tx, index, privkey, amount, hashcode=btc.SIGHASH_ALL, **kwargs): assert amount is not None - return btc.sign(btc.serialize(tx), index, privkey, + a, b = btc.sign(tx, index, privkey, hashcode=hashcode, amount=amount, native=False) + return a, b class BTC_P2WPKH(BTCEngine): @@ -286,7 +313,7 @@ class BTC_P2WPKH(BTCEngine): def sign_transaction(cls, tx, index, privkey, amount, hashcode=btc.SIGHASH_ALL, **kwargs): assert amount is not None - return btc.sign(btc.serialize(tx), index, privkey, + return btc.sign(tx, index, privkey, hashcode=hashcode, amount=amount, native=True) class BTC_Timelocked_P2WSH(BTCEngine): diff --git a/jmclient/jmclient/maker.py b/jmclient/jmclient/maker.py index c19c5a4..8ae4834 100644 --- a/jmclient/jmclient/maker.py +++ b/jmclient/jmclient/maker.py @@ -5,15 +5,13 @@ import pprint import random import sys import abc -from binascii import unhexlify - -from jmbitcoin import SerializationError, SerializationTruncationError import jmbitcoin as btc +from jmbase import (bintohex, hextobin, hexbin, + get_log, EXIT_SUCCESS, EXIT_FAILURE) from jmclient.wallet import estimate_tx_fee, compute_tx_locktime from jmclient.wallet_service import WalletService from jmclient.configure import jm_single -from jmbase.support import get_log, EXIT_SUCCESS, EXIT_FAILURE from jmclient.support import calc_cj_fee, select_one_utxo from jmclient.podle import verify_podle, PoDLE, PoDLEError from twisted.internet import task, reactor @@ -48,10 +46,15 @@ class Maker(object): sys.exit(EXIT_FAILURE) jlog.info('offerlist={}'.format(self.offerlist)) + @hexbin def on_auth_received(self, nick, offer, commitment, cr, amount, kphex): """Receives data on proposed transaction offer from daemon, verifies commitment, returns necessary data to send ioauth message (utxos etc) """ + # special case due to cjfee passed as string: it can accidentally parse + # as hex: + if not isinstance(offer["cjfee"], str): + offer["cjfee"] = bintohex(offer["cjfee"]) #check the validity of the proof of discrete log equivalence tries = jm_single().config.getint("POLICY", "taker_utxo_retries") def reject(msg): @@ -65,9 +68,8 @@ class Maker(object): reason = repr(e) return reject(reason) - if not verify_podle(str(cr_dict['P']), str(cr_dict['P2']), str(cr_dict['sig']), - str(cr_dict['e']), str(commitment), - index_range=range(tries)): + if not verify_podle(cr_dict['P'], cr_dict['P2'], cr_dict['sig'], + cr_dict['e'], commitment, index_range=range(tries)): reason = "verify_podle failed" return reject(reason) #finally, check that the proffered utxo is real, old enough, large enough, @@ -89,14 +91,14 @@ class Maker(object): try: if not self.wallet_service.pubkey_has_script( - unhexlify(cr_dict['P']), unhexlify(res[0]['script'])): + cr_dict['P'], res[0]['script']): raise EngineError() except EngineError: reason = "Invalid podle pubkey: " + str(cr_dict['P']) return reject(reason) # authorisation of taker passed - #Find utxos for the transaction now: + # Find utxos for the transaction now: utxos, cj_addr, change_addr = self.oid_to_order(offer, amount) if not utxos: #could not find funds @@ -109,21 +111,28 @@ class Maker(object): # Just choose the first utxo in self.utxos and retrieve key from wallet. auth_address = utxos[list(utxos.keys())[0]]['address'] auth_key = self.wallet_service.get_key_from_addr(auth_address) - auth_pub = btc.privtopub(auth_key) - btc_sig = btc.ecdsa_sign(kphex, auth_key) + auth_pub = btc.privkey_to_pubkey(auth_key) + # kphex was auto-converted by @hexbin but we actually need to sign the + # hex version to comply with pre-existing JM protocol: + btc_sig = btc.ecdsa_sign(bintohex(kphex), auth_key) return (True, utxos, auth_pub, cj_addr, change_addr, btc_sig) - def on_tx_received(self, nick, txhex, offerinfo): + @hexbin + def on_tx_received(self, nick, tx_from_taker, offerinfo): """Called when the counterparty has sent an unsigned transaction. Sigs are created and returned if and only if the transaction passes verification checks (see verify_unsigned_tx()). """ + # special case due to cjfee passed as string: it can accidentally parse + # as hex: + if not isinstance(offerinfo["offer"]["cjfee"], str): + offerinfo["offer"]["cjfee"] = bintohex(offerinfo["offer"]["cjfee"]) try: - tx = btc.deserialize(txhex) - except (IndexError, SerializationError, SerializationTruncationError) as e: + tx = btc.CMutableTransaction.deserialize(tx_from_taker) + except Exception as e: return (False, 'malformed txhex. ' + repr(e)) - jlog.info('obtained tx\n' + pprint.pformat(tx)) + jlog.info('obtained tx\n' + bintohex(tx.serialize())) goodtx, errmsg = self.verify_unsigned_tx(tx, offerinfo) if not goodtx: jlog.info('not a good tx, reason=' + errmsg) @@ -133,25 +142,26 @@ class Maker(object): utxos = offerinfo["utxos"] our_inputs = {} - for index, ins in enumerate(tx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + for index, ins in enumerate(tx.vin): + utxo = (ins.prevout.hash[::-1], ins.prevout.n) if utxo not in utxos: continue script = self.wallet_service.addr_to_script(utxos[utxo]['address']) amount = utxos[utxo]['value'] our_inputs[index] = (script, amount) - txs = self.wallet_service.sign_tx(btc.deserialize(unhexlify(txhex)), our_inputs) + success, msg = self.wallet_service.sign_tx(tx, our_inputs) + assert success, msg for index in our_inputs: - sigmsg = unhexlify(txs['ins'][index]['script']) - if 'txinwitness' in txs['ins'][index]: + sigmsg = tx.vin[index].scriptSig + if tx.has_witness(): # Note that this flag only implies that the transaction # *as a whole* is using segwit serialization; it doesn't # imply that this specific input is segwit type (to be # fully general, we allow that even our own wallet's # inputs might be of mixed type). So, we catch the EngineError # which is thrown by non-segwit types. This way the sigmsg - # will only contain the scriptSig field if the wallet object + # will only contain the scriptCode field if the wallet object # decides it's necessary/appropriate for this specific input # If it is segwit, we prepend the witness data since we want # (sig, pub, witnessprogram=scriptSig - note we could, better, @@ -160,16 +170,16 @@ class Maker(object): # transaction scriptSig), else (non-sw) the !sig message remains # unchanged as (sig, pub). try: - scriptSig = btc.pubkey_to_p2wpkh_script(txs['ins'][index]['txinwitness'][1]) - sigmsg = b''.join(btc.serialize_script_unit( - x) for x in txs['ins'][index]['txinwitness'] + [scriptSig]) - except IndexError: + sig, pub = [a for a in iter(tx.wit.vtxinwit[index].scriptWitness)] + scriptCode = btc.pubkey_to_p2wpkh_script(pub) + sigmsg = btc.CScript([sig]) + btc.CScript(pub) + scriptCode + except Exception as e: #the sigmsg was already set before the segwit check pass sigs.append(base64.b64encode(sigmsg).decode('ascii')) return (True, sigs) - def verify_unsigned_tx(self, txd, offerinfo): + def verify_unsigned_tx(self, tx, offerinfo): """This code is security-critical. Before signing the transaction the Maker must ensure that all details are as expected, and most importantly @@ -177,14 +187,13 @@ class Maker(object): in total. The data is taken from the offerinfo dict and compared with the serialized txhex. """ - tx_utxo_set = set(ins['outpoint']['hash'] + ':' + str( - ins['outpoint']['index']) for ins in txd['ins']) + tx_utxo_set = set((x.prevout.hash[::-1], x.prevout.n) for x in tx.vin) utxos = offerinfo["utxos"] cjaddr = offerinfo["cjaddr"] - cjaddr_script = btc.address_to_script(cjaddr) + cjaddr_script = btc.CCoinAddress(cjaddr).to_scriptPubKey() changeaddr = offerinfo["changeaddr"] - changeaddr_script = btc.address_to_script(changeaddr) + changeaddr_script = btc.CCoinAddress(changeaddr).to_scriptPubKey() #Note: this value is under the control of the Taker, #see comment below. amount = offerinfo["amount"] @@ -216,14 +225,14 @@ class Maker(object): #exactly once with the required amts, in the output. times_seen_cj_addr = 0 times_seen_change_addr = 0 - for outs in txd['outs']: - if outs['script'] == cjaddr_script: + for outs in tx.vout: + if outs.scriptPubKey == cjaddr_script: times_seen_cj_addr += 1 - if outs['value'] != amount: + if outs.nValue != amount: return (False, 'Wrong cj_amount. I expect ' + str(amount)) - if outs['script'] == changeaddr_script: + if outs.scriptPubKey == changeaddr_script: times_seen_change_addr += 1 - if outs['value'] != expected_change_value: + if outs.nValue != expected_change_value: return (False, 'wrong change, i expect ' + str( expected_change_value)) if times_seen_cj_addr != 1 or times_seen_change_addr != 1: @@ -392,7 +401,8 @@ class P2EPMaker(Maker): # will not be reached except in testing self.on_tx_unconfirmed(txd, txid) - def on_tx_received(self, nick, txhex): + @hexbin + def on_tx_received(self, nick, txser): """ Called when the sender-counterparty has sent a transaction proposal. 1. First we check for the expected destination and amount (this is sufficient to identify our cp, as this info was presumably passed @@ -418,29 +428,30 @@ class P2EPMaker(Maker): we broadcast the non-coinjoin fallback tx instead. """ try: - tx = btc.deserialize(txhex) - except (IndexError, SerializationError, SerializationTruncationError) as e: + tx = btc.CMutableTransaction.deserialize(txser) + except Exception as e: return (False, 'malformed txhex. ' + repr(e)) self.user_info('obtained proposed fallback (non-coinjoin) ' +\ - 'transaction from sender:\n' + pprint.pformat(tx)) + 'transaction from sender:\n' + str(tx)) - if len(tx["outs"]) != 2: + if len(tx.vout) != 2: return (False, "Transaction has more than 2 outputs; not supported.") dest_found = False destination_index = -1 change_index = -1 proposed_change_value = 0 - for index, out in enumerate(tx["outs"]): - if out["script"] == btc.address_to_script(self.destination_addr): + for index, out in enumerate(tx.vout): + if out.scriptPubKey == btc.CCoinAddress( + self.destination_addr).to_scriptPubKey(): # we found the expected destination; is the amount correct? - if not out["value"] == self.receiving_amount: + if not out.nValue == self.receiving_amount: return (False, "Wrong payout value in proposal from sender.") dest_found = True destination_index = index else: change_found = True - proposed_change_out = out["script"] - proposed_change_value = out["value"] + proposed_change_out = out.scriptPubKey + proposed_change_value = out.nValue change_index = index if not dest_found: @@ -450,9 +461,8 @@ class P2EPMaker(Maker): # batch retrieval of utxo data utxo = {} ctr = 0 - for index, ins in enumerate(tx['ins']): - utxo_for_checking = ins['outpoint']['hash'] + ':' + str(ins[ - 'outpoint']['index']) + for index, ins in enumerate(tx.vin): + utxo_for_checking = (ins.prevout.hash[::-1], ins.prevout.n) utxo[ctr] = [index, utxo_for_checking] ctr += 1 @@ -471,7 +481,7 @@ class P2EPMaker(Maker): btc_fee = total_sender_input - self.receiving_amount - proposed_change_value self.user_info("Network transaction fee of fallback tx is: " + str( btc_fee) + " satoshis.") - fee_est = estimate_tx_fee(len(tx['ins']), len(tx['outs']), + fee_est = estimate_tx_fee(len(tx.vin), len(tx.vout), txtype=self.wallet_service.get_txtype()) fee_ok = False if btc_fee > 0.3 * fee_est and btc_fee < 3 * fee_est: @@ -488,51 +498,28 @@ class P2EPMaker(Maker): # It has the advantage of (a) being simpler and (b) allowing for any # non standard coins. # - #res = jm_single().bc_interface.rpc('testmempoolaccept', [txhex]) + #res = jm_single().bc_interface.rpc('testmempoolaccept', [txser]) #print("Got this result from rpc call: ", res) #if not res["accepted"]: # return (False, "Proposed transaction was rejected from mempool.") - # Manual verification of the transaction signatures. Passing this - # test does imply that the transaction is valid (unless there is - # a double spend during the process), but is restricted to standard - # types: p2pkh, p2wpkh, p2sh-p2wpkh only. Double spend is not counted - # as a risk as this is a payment. + # Manual verification of the transaction signatures. + # TODO handle native segwit properly for i, u in iteritems(utxo): - if "txinwitness" in tx["ins"][u[0]]: - ver_amt = utxo_data[i]["value"] - try: - ver_sig, ver_pub = tx["ins"][u[0]]["txinwitness"] - except Exception as e: - self.user_info("Segwit error: " + repr(e)) - return (False, "Segwit input not of expected type, " - "either p2sh-p2wpkh or p2wpkh") - # note that the scriptCode is the same whether nested or not - # also note that the scriptCode has to be inferred if we are - # only given a transaction serialization. - scriptCode = "76a914" + btc.hash160(unhexlify(ver_pub)) + "88ac" - else: - scriptCode = None - ver_amt = None - scriptSig = btc.deserialize_script(tx["ins"][u[0]]["script"]) - if len(scriptSig) != 2: - return (False, - "Proposed transaction contains unsupported input type") - ver_sig, ver_pub = scriptSig - if not btc.verify_tx_input(txhex, u[0], - utxo_data[i]['script'], - ver_sig, ver_pub, - scriptCode=scriptCode, - amount=ver_amt): + if not btc.verify_tx_input(tx, i, + tx.vin[i].scriptSig, + btc.CScript(utxo_data[i]["script"]), + amount=utxo_data[i]["value"], + witness=tx.wit.vtxinwit[i].scriptWitness): return (False, "Proposed transaction is not correctly signed.") # At this point we are satisfied with the proposal. Record the fallback # in case the sender disappears and the payjoin tx doesn't happen: self.user_info("We'll use this serialized transaction to broadcast if your" " counterparty fails to broadcast the payjoin version:") - self.user_info(txhex) + self.user_info(bintohex(txser)) # Keep a local copy for broadcast fallback: - self.fallback_tx = txhex + self.fallback_tx = tx # Now we add our own inputs: # See the gist comment here: @@ -558,7 +545,7 @@ class P2EPMaker(Maker): self.user_info("Choosing one coin at random") try: my_utxos = self.wallet_service.select_utxos( - self.mixdepth, jm_single().DUST_THRESHOLD, + self.mixdepth, jm_single().DUST_THRESHOLD, select_fn=select_one_utxo) except: return self.no_coins_fallback() @@ -566,7 +553,7 @@ class P2EPMaker(Maker): else: # get an approximate required amount assuming 4 inputs, which is # fairly conservative (but guess by necessity). - fee_for_select = estimate_tx_fee(len(tx['ins']) + 4, 2, + fee_for_select = estimate_tx_fee(len(tx.vin) + 4, 2, txtype=self.wallet_service.get_txtype()) approx_sum = max_sender_amt - self.receiving_amount + fee_for_select try: @@ -592,7 +579,7 @@ class P2EPMaker(Maker): # adjust the output amount at the destination based on our contribution new_destination_amount = self.receiving_amount + my_total_in # estimate the required fee for the new version of the transaction - total_ins = len(tx["ins"]) + len(my_utxos.keys()) + total_ins = len(tx.vin) + len(my_utxos.keys()) est_fee = estimate_tx_fee(total_ins, 2, txtype=self.wallet_service.get_txtype()) self.user_info("We estimated a fee of: " + str(est_fee)) new_change_amount = total_sender_input + my_total_in - \ @@ -604,25 +591,27 @@ class P2EPMaker(Maker): new_outs = [{"address": self.destination_addr, "value": new_destination_amount}] if new_change_amount >= jm_single().BITCOIN_DUST_THRESHOLD: - new_outs.append({"script": proposed_change_out, - "value": new_change_amount}) + new_outs.append({"address": str(btc.CCoinAddress.from_scriptPubKey( + proposed_change_out)), "value": new_change_amount}) new_ins = [x[1] for x in utxo.values()] new_ins.extend(my_utxos.keys()) - new_tx = btc.make_shuffled_tx(new_ins, new_outs, False, 2, compute_tx_locktime()) - new_tx_deser = btc.deserialize(new_tx) + new_tx = btc.make_shuffled_tx(new_ins, new_outs, 2, compute_tx_locktime()) + # sign our inputs before transfer our_inputs = {} - for index, ins in enumerate(new_tx_deser['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + for index, ins in enumerate(new_tx.vin): + utxo = (ins.prevout.hash[::-1], ins.prevout.n) if utxo not in my_utxos: continue - script = self.wallet_service.addr_to_script(my_utxos[utxo]['address']) - amount = my_utxos[utxo]['value'] + script = my_utxos[utxo]["script"] + amount = my_utxos[utxo]["value"] our_inputs[index] = (script, amount) - txs = self.wallet_service.sign_tx(btc.deserialize(new_tx), our_inputs) - txinfo = tuple((x["script"], x["value"]) for x in txs["outs"]) + success, msg = self.wallet_service.sign_tx(new_tx, our_inputs) + if not success: + return (False, "Failed to sign new transaction, error: " + msg) + txinfo = tuple((x.scriptPubKey, x.nValue) for x in new_tx.vout) self.wallet_service.register_callbacks([self.on_tx_unconfirmed], txinfo, "unconfirmed") self.wallet_service.register_callbacks([self.on_tx_confirmed], txinfo, "confirmed") # The blockchain interface just abandons monitoring if the transaction @@ -630,7 +619,7 @@ class P2EPMaker(Maker): # action in this case, so we add an additional callback to the reactor: reactor.callLater(jm_single().config.getint("TIMEOUT", "unconfirm_timeout_sec"), self.broadcast_fallback) - return (True, nick, btc.serialize(txs)) + return (True, nick, bintohex(new_tx.serialize())) def no_coins_fallback(self): """ Broadcast, optionally, the fallback non-coinjoin transaction @@ -646,8 +635,8 @@ class P2EPMaker(Maker): def broadcast_fallback(self): self.user_info("Broadcasting non-coinjoin fallback transaction.") - txid = btc.txhash(self.fallback_tx) - success = jm_single().bc_interface.pushtx(self.fallback_tx) + txid = self.fallback_tx.GetTxid()[::-1] + success = jm_single().bc_interface.pushtx(self.fallback_tx.serialize()) if not success: self.user_info("ERROR: the fallback transaction did not broadcast. " "The payment has NOT been made.") diff --git a/jmclient/jmclient/output.py b/jmclient/jmclient/output.py index 7242aec..a6ffc4f 100644 --- a/jmclient/jmclient/output.py +++ b/jmclient/jmclient/output.py @@ -61,7 +61,7 @@ def generate_podle_error_string(priv_utxo_pairs, to, ts, wallet_service, cjamoun "with 'python add-utxo.py --help'\n\n") errmsg += ("***\nFor reference, here are the utxos in your wallet:\n") - for md, utxos in wallet_service.get_utxos_by_mixdepth(hexfmt=False).items(): + for md, utxos in wallet_service.get_utxos_by_mixdepth().items(): if not utxos: continue errmsg += ("\nmixdepth {}:\n{}".format( diff --git a/jmclient/jmclient/podle.py b/jmclient/jmclient/podle.py index eff5dcc..d7dd26f 100644 --- a/jmclient/jmclient/podle.py +++ b/jmclient/jmclient/podle.py @@ -7,11 +7,12 @@ import hashlib import json import binascii import struct +from pprint import pformat from jmbase import jmprint from jmbitcoin import multiply, add_pubkeys, getG, podle_PublicKey,\ - podle_PrivateKey, encode, decode, N, podle_PublicKey_class -from jmbase.support import EXIT_FAILURE - + podle_PrivateKey, N, podle_PublicKey_class +from jmbase import (EXIT_FAILURE, utxostr_to_utxo, + utxo_to_utxostr, hextobin, bintohex) PODLE_COMMIT_FILE = None @@ -45,33 +46,31 @@ class PoDLE(object): used=False): #This class allows storing of utxo in format "txid:n" only for #convenience of storage/access; it doesn't check or use the data. - #Arguments must be provided in hex. + #Arguments must be provided in binary not hex. self.u = u if not priv: if P: - #Construct a pubkey from raw hex - self.P = podle_PublicKey(binascii.unhexlify(P)) + self.P = podle_PublicKey(P) else: self.P = None else: if P: raise PoDLEError("Pubkey should not be provided with privkey") #any other formatting abnormality will just throw in PrivateKey - if len(priv) == 66 and priv[-2:] == '01': - priv = priv[:-2] - self.priv = podle_PrivateKey(binascii.unhexlify(priv)) + if len(priv) == 33 and priv[-1:] == b"\x01": + priv = priv[:-1] + self.priv = podle_PrivateKey(priv) self.P = self.priv.public_key if P2: - self.P2 = podle_PublicKey(binascii.unhexlify(P2)) + self.P2 = podle_PublicKey(P2) else: self.P2 = None - #These sig values should be passed in hex. self.s = None self.e = None if s: - self.s = binascii.unhexlify(s) + self.s = s if e: - self.e = binascii.unhexlify(e) + self.e = e #Optionally maintain usage state (boolean) self.used = used #the H(P2) value @@ -79,14 +78,13 @@ class PoDLE(object): def get_commitment(self): """Set the commitment to sha256(serialization of public key P2) - Return in hex to calling function """ if not self.P2: raise PoDLEError("Cannot construct commitment, no P2 available") if not isinstance(self.P2, podle_PublicKey_class): raise PoDLEError("Cannot construct commitment, P2 is not a pubkey") self.commitment = hashlib.sha256(self.P2.format()).digest() - return binascii.hexlify(self.commitment).decode('ascii') + return self.commitment def generate_podle(self, index=0, k=None): """Given a raw private key, in hex format, @@ -117,79 +115,85 @@ class PoDLE(object): and the associated signature data that can be used to open the commitment. """ + self.i = index #TODO nonce could be rfc6979? if not k: k = os.urandom(32) - J = getNUMS(index) + J = getNUMS(self.i) KG = podle_PrivateKey(k).public_key - KJ = multiply(k, J.format(), False, return_serialized=False) + KJ = multiply(k, J.format(), return_serialized=False) self.P2 = getP2(self.priv, J) self.get_commitment() self.e = hashlib.sha256(b''.join([x.format( ) for x in [KG, KJ, self.P, self.P2]])).digest() - k_int = decode(k, 256) - priv_int = decode(self.priv.secret, 256) - e_int = decode(self.e, 256) + k_int, priv_int, e_int = (int.from_bytes(x, + byteorder="big") for x in [k, self.priv.secret, self.e]) sig_int = (k_int + priv_int * e_int) % N - self.s = encode(sig_int, 256, minlen=32) + self.s = (sig_int).to_bytes(32, byteorder="big") return self.reveal() def reveal(self): """Encapsulate all the data representing the proof - in a dict for client functions. Data output in hex. + in a dict for client functions. """ if not all([self.u, self.P, self.P2, self.s, self.e]): raise PoDLEError("Cannot generate proof, data is missing") if not self.commitment: self.get_commitment() - Phex, P2hex, shex, ehex, commit = [ - binascii.hexlify(x).decode('ascii') - for x in [self.P.format(), self.P2.format(), self.s, self.e, - self.commitment] - ] - return {'used': str(self.used), + return {'used': self.used, 'utxo': self.u, - 'P': Phex, - 'P2': P2hex, - 'commit': commit, - 'sig': shex, - 'e': ehex} + 'P': self.P.format(), + 'P2': self.P2.format(), + 'commit': self.commitment, + 'sig': self.s, + 'e': self.e} def serialize_revelation(self, separator='|'): + """ Outputs the over-the-wire format as used in + Joinmarket communication protocol. + """ state_dict = self.reveal() - ser_list = [] - for k in ['utxo', 'P', 'P2', 'sig', 'e']: - ser_list += [state_dict[k]] + success, utxo = utxo_to_utxostr(state_dict["utxo"]) + assert success, "invalid utxo in PoDLE" + ser_list = [utxo] + ser_list += [bintohex(state_dict[x]) for x in ["P", "P2", "sig", "e"]] ser_string = separator.join(ser_list) return ser_string @classmethod def deserialize_revelation(cls, ser_rev, separator='|'): + """ Reads the over-the-wire format as used in + Joinmarket communication protocol. + """ ser_list = ser_rev.split(separator) if len(ser_list) != 5: raise PoDLEError("Failed to deserialize, wrong format") - utxo, P, P2, s, e = ser_list - return {'utxo': utxo, 'P': P, 'P2': P2, 'sig': s, 'e': e} + utxostr, P, P2, s, e = ser_list + success, utxo = utxostr_to_utxo(utxostr) + assert success, "invalid utxo format in PoDLE." + return {'utxo': utxo, 'P': hextobin(P), + 'P2': hextobin(P2), 'sig': hextobin(s), 'e': hextobin(e)} def verify(self, commitment, index_range): """For an object created without a private key, check that the opened commitment verifies for at least - one NUMS point as defined by the range in index_range + one NUMS point as defined by the range in index_range. """ if not all([self.P, self.P2, self.s, self.e]): raise PoDLEError("Verify called without sufficient data") if not self.get_commitment() == commitment: return False + for J in [getNUMS(i) for i in index_range]: sig_priv = podle_PrivateKey(self.s) sG = sig_priv.public_key - sJ = multiply(self.s, J.format(), False) - e_int = decode(self.e, 256) - minus_e = encode(-e_int % N, 256, minlen=32) - minus_e_P = multiply(minus_e, self.P.format(), False) - minus_e_P2 = multiply(minus_e, self.P2.format(), False) - KGser = add_pubkeys([sG.format(), minus_e_P], False) - KJser = add_pubkeys([sJ, minus_e_P2], False) + sJ = multiply(self.s, J.format()) + e_int = int.from_bytes(self.e, byteorder="big") + minus_e = (-e_int % N).to_bytes(32, byteorder="big") + minus_e_P = multiply(minus_e, self.P.format()) + minus_e_P2 = multiply(minus_e, self.P2.format()) + KGser = add_pubkeys([sG.format(), minus_e_P]) + KJser = add_pubkeys([sJ, minus_e_P2]) #check 2: e =?= H(K_G || K_J || P || P2) e_check = hashlib.sha256(KGser + KJser + self.P.format() + self.P2.format()).digest() @@ -198,6 +202,20 @@ class PoDLE(object): #commitment fails for any NUMS in the provided range return False + def __repr__(self): + """ Specified here to allow logging. + """ + # note: will throw if not fully initalised + r = self.reveal() + success, utxo = utxo_to_utxostr(r["utxo"]) + assert success, "invalid utxo in PoDLE." + return pformat({'used': r["used"], + 'utxo': utxo, + 'P': bintohex(r["P"]), + 'P2': bintohex(r["P2"]), + 'commit': bintohex(r["commit"]), + 'sig': bintohex(r["sig"]), + 'e': bintohex(r["e"])}) def getNUMS(index=0): """Taking secp256k1's G as a seed, @@ -244,7 +262,7 @@ def verify_all_NUMS(write=False): """ nums_points = {} for i in range(256): - nums_points[i] = binascii.hexlify(getNUMS(i).format()).decode('ascii') + nums_points[i] = bintohex(getNUMS(i).format()) if write: with open("nums_basepoints.txt", "wb") as f: from pprint import pformat @@ -263,13 +281,82 @@ def getP2(priv, nums_pt): priv_raw = priv.secret return multiply(priv_raw, nums_pt.format(), - False, return_serialized=False) +# functions which interact with the external persistence of podle data: + +def switch_external_dict_format(ed, utxo_converter, hexbinconverter): + """External dict has structure: + {txid:N:{'P':pubkey, 'reveal':{1:{'P2':P2,'s':s,'e':e}, 2:{..},..}}} + This function switches between readable/writable in file (strings, hex) + and that used internally. + """ + retval = {} + for u in ed: + success, u2 = utxo_converter(u) + assert success, "invalid utxo format in external dict parsing." + retval[u2] = {"P": hexbinconverter(ed[u]["P"])} + retval[u2]["reveal"] = {} + for i in ed[u]["reveal"]: + # hack: python json does not allow int dict keys, they must + # be str, so when reading from file, we convert back to int: + if not isinstance(i, int): + j = int(i) + else: + j = i + retval[u2]["reveal"][j] = { + "P2": hexbinconverter(ed[u]["reveal"][i]["P2"]), + "s": hexbinconverter(ed[u]["reveal"][i]["s"]), + "e": hexbinconverter(ed[u]["reveal"][i]["e"])} + return retval + +def external_dict_to_file(ed): + """ Converts internal format of dict to one writable/readable + in file. + """ + return switch_external_dict_format(ed, utxo_to_utxostr, bintohex) + +def external_dict_from_file(ed): + """ Takes the external dict extracted through json deserialization + from a file and converts it to internal format: + {txid:N:{'P':pubkey, 'reveal':{1:{'P2':P2,'s':s,'e':e}, 2:{..},..}}} + """ + return switch_external_dict_format(ed, utxostr_to_utxo, hextobin) + +def write_to_podle_file(used, external): + """ Update persisted commitment data in PODLE_COMMIT_FILE. + """ + to_write = {} + to_write['used'] = [bintohex(x) for x in used] + externalfmt = external_dict_to_file(external) + to_write['external'] = externalfmt + with open(PODLE_COMMIT_FILE, "wb") as f: + f.write(json.dumps(to_write, indent=4).encode('utf-8')) + +def read_from_podle_file(): + """ Returns used commitment list and external commitments dict + struct currently stored in PODLE_COMMIT_FILE. + """ + if os.path.isfile(PODLE_COMMIT_FILE): + with open(PODLE_COMMIT_FILE, "rb") as f: + try: + c = json.loads(f.read().decode('utf-8')) + except ValueError: #pragma: no cover + #Exit conditions cannot be included in tests. + jmprint("the file: " + PODLE_COMMIT_FILE + " is not valid json.", + "error") + sys.exit(EXIT_FAILURE) + if 'used' not in c.keys() or 'external' not in c.keys(): + raise PoDLEError("Incorrectly formatted file: " + PODLE_COMMIT_FILE) + + used = [hextobin(x) for x in c["used"]] + external = external_dict_from_file(c["external"]) + return (used, external) + return ([], {}) def get_podle_commitments(): """Returns set of commitments used as a list: - [H(P2),..] (hex) and a dict of all existing external commitments. + [H(P2),..] and a dict of all existing external commitments. It is presumed that each H(P2) can be used only once (this may not literally be true, but represents good joinmarket "citizenship"). @@ -280,18 +367,11 @@ def get_podle_commitments(): """ if not os.path.isfile(PODLE_COMMIT_FILE): return ([], {}) - with open(PODLE_COMMIT_FILE, "rb") as f: - c = json.loads(f.read().decode('utf-8')) - if 'used' not in c.keys() or 'external' not in c.keys(): - raise PoDLEError("Incorrectly formatted file: " + PODLE_COMMIT_FILE) - return (c['used'], c['external']) - + return read_from_podle_file() def add_external_commitments(ecs): """To allow external functions to add - PoDLE commitments that were calculated elsewhere; - the format of each entry in ecs must be: - {txid:N:{'P':pubkey, 'reveal':{1:{'P2':P2,'s':s,'e':e}, 2:{..},..}}} + PoDLE commitments that were calculated elsewhere. """ update_commitments(external_to_add=ecs) @@ -304,25 +384,7 @@ def update_commitments(commitment=None, whose key value is the utxo in external_to_remove, persist updated entries to disk. """ - c = {} - if os.path.isfile(PODLE_COMMIT_FILE): - with open(PODLE_COMMIT_FILE, "rb") as f: - try: - c = json.loads(f.read().decode('utf-8')) - except ValueError: #pragma: no cover - #Exit conditions cannot be included in tests. - jmprint("the file: " + PODLE_COMMIT_FILE + " is not valid json.", - "error") - sys.exit(EXIT_FAILURE) - - if 'used' in c: - commitments = c['used'] - else: - commitments = [] - if 'external' in c: - external = c['external'] - else: - external = {} + commitments, external = read_from_podle_file() if commitment: commitments.append(commitment) #remove repeats @@ -334,11 +396,7 @@ def update_commitments(commitment=None, } if external_to_add: external.update(external_to_add) - to_write = {} - to_write['used'] = commitments - to_write['external'] = external - with open(PODLE_COMMIT_FILE, "wb") as f: - f.write(json.dumps(to_write, indent=4).encode('utf-8')) + write_to_podle_file(commitments, external) def get_podle_tries(utxo, priv=None, max_tries=1, external=False): used_commitments, external_commitments = get_podle_commitments() @@ -349,9 +407,8 @@ def get_podle_tries(utxo, priv=None, max_tries=1, external=False): #use as many as were provided in the file, up to a max of max_tries m = min([len(ec['reveal'].keys()), max_tries]) for i in reversed(range(m)): - key = str(i) - p = PoDLE(u=utxo,P=ec['P'],P2=ec['reveal'][key]['P2'], - s=ec['reveal'][key]['s'], e=ec['reveal'][key]['e']) + p = PoDLE(u=utxo, P=ec["P"], P2=ec["reveal"][i]["P2"], + s=ec["reveal"][i]["s"], e=ec["reveal"][i]["e"]) if p.get_commitment() in used_commitments: return i+1 else: @@ -387,10 +444,10 @@ def generate_podle(priv_utxo_pairs, max_tries=1, allow_external=None, k=None): #which is still available. index = tries p = PoDLE(u=utxo, priv=priv) - c = p.generate_podle(index) + p.generate_podle(index) #persist for future checks - update_commitments(commitment=c['commit']) - return c + update_commitments(commitment=p.commitment) + return p if allow_external: for u in allow_external: tries = get_podle_tries(utxo=u, max_tries=max_tries, external=True) @@ -400,19 +457,17 @@ def generate_podle(priv_utxo_pairs, max_tries=1, allow_external=None, k=None): #remove this entry update_commitments(external_to_remove=u) continue - index = str(tries) ec = external_commitments[u] - p = PoDLE(u=u,P=ec['P'],P2=ec['reveal'][index]['P2'], - s=ec['reveal'][index]['s'], e=ec['reveal'][index]['e']) + ecri = ec["reveal"][tries] + p = PoDLE(u=u, P=ec["P"], P2=ecri["P2"], s=ecri["s"], e=ecri["e"]) update_commitments(commitment=p.get_commitment()) - return p.reveal() + return p #Failed to find any non-used valid commitment: return None def verify_podle(Pser, P2ser, sig, e, commitment, index_range=range(10)): verifying_podle = PoDLE(P=Pser, P2=P2ser, s=sig, e=e) - #check 1: Hash(P2ser) =?= commitment if not verifying_podle.verify(commitment, index_range): return False return True diff --git a/jmclient/jmclient/taker.py b/jmclient/jmclient/taker.py index 22d1e44..9ee185a 100644 --- a/jmclient/jmclient/taker.py +++ b/jmclient/jmclient/taker.py @@ -5,12 +5,10 @@ import base64 import pprint import random from twisted.internet import reactor, task -from binascii import hexlify, unhexlify -from jmbitcoin import SerializationError, SerializationTruncationError import jmbitcoin as btc from jmclient.configure import jm_single, validate_address -from jmbase.support import get_log +from jmbase import get_log, hextobin, bintohex, hexbin from jmclient.support import (calc_cj_fee, weighted_order_choose, choose_orders, choose_sweep_orders) from jmclient.wallet import estimate_tx_fee, compute_tx_locktime @@ -238,7 +236,6 @@ class Taker(object): #Initialization has been successful. We must set the nonrespondants #now to keep track of what changed when we receive the utxo data self.nonrespondants = list(self.orderbook.keys()) - return (True, self.cjamount, commitment, revelation, self.orderbook) def filter_orderbook(self, orderbook, sweep=False): @@ -344,6 +341,7 @@ class Taker(object): self.utxos = {None: list(self.input_utxos.keys())} return True + @hexbin def receive_utxos(self, ioauth_data): """Triggered when the daemon returns utxo data from makers who responded; this is the completion of phase 1 @@ -354,6 +352,7 @@ class Taker(object): #Temporary list used to aggregate all ioauth data that must be removed rejected_counterparties = [] + #Need to authorize against the btc pubkey first. for nick, nickdata in iteritems(ioauth_data): utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata @@ -376,10 +375,9 @@ class Taker(object): self.maker_utxo_data = {} for nick, nickdata in iteritems(ioauth_data): - utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata + utxo_list, auth_pub, cj_addr, change_addr, _, _ = nickdata + utxo_data = jm_single().bc_interface.query_utxo_set(utxo_list) self.utxos[nick] = utxo_list - utxo_data = jm_single().bc_interface.query_utxo_set(self.utxos[ - nick]) if None in utxo_data: jlog.warn(('ERROR outputs unconfirmed or already spent. ' 'utxo_data={}').format(pprint.pformat(utxo_data))) @@ -391,13 +389,12 @@ class Taker(object): #Extract the address fields from the utxos #Construct the Bitcoin address for the auth_pub field #Ensure that at least one address from utxos corresponds. - auth_pub_bin = unhexlify(auth_pub) for inp in utxo_data: try: if self.wallet_service.pubkey_has_script( - auth_pub_bin, unhexlify(inp['script'])): + auth_pub, inp['script']): break - except EngineError: + except EngineError as e: pass else: jlog.warn("ERROR maker's (" + nick + ")" @@ -500,81 +497,85 @@ class Taker(object): else: self.outputs.append({'address': self.my_change_addr, 'value': my_change_value}) - self.utxo_tx = [dict([('output', u)]) - for u in sum(self.utxos.values(), [])] + self.utxo_tx = [u for u in sum(self.utxos.values(), [])] self.outputs.append({'address': self.coinjoin_address(), 'value': self.cjamount}) - tx = btc.make_shuffled_tx(self.utxo_tx, self.outputs, False) - jlog.info('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) + self.latest_tx = btc.make_shuffled_tx(self.utxo_tx, self.outputs) + jlog.info('obtained tx\n' + bintohex(self.latest_tx.serialize())) - self.latest_tx = btc.deserialize(tx) - for index, ins in enumerate(self.latest_tx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + for index, ins in enumerate(self.latest_tx.vin): + utxo = (ins.prevout.hash[::-1], ins.prevout.n) if utxo not in self.input_utxos.keys(): continue # placeholders required - ins['script'] = 'deadbeef' + ins.scriptSig = btc.CScript.fromhex("deadbeef") self.taker_info_callback("INFO", "Built tx, sending to counterparties.") - return (True, list(self.maker_utxo_data.keys()), tx) + return (True, list(self.maker_utxo_data.keys()), + bintohex(self.latest_tx.serialize())) + @hexbin def auth_counterparty(self, btc_sig, auth_pub, maker_pk): """Validate the counterpartys claim to own the btc address/pubkey that will be used for coinjoining with an ecdsa verification. """ try: - if not btc.ecdsa_verify(maker_pk, btc_sig, auth_pub): + # maker pubkey as message is in hex format: + if not btc.ecdsa_verify(bintohex(maker_pk), btc_sig, auth_pub): jlog.debug('signature didnt match pubkey and message') return False except Exception as e: - jlog.info("Failed ecdsa verify for maker pubkey: " + str(maker_pk)) + jlog.info("Failed ecdsa verify for maker pubkey: " + bintohex(maker_pk)) jlog.info("Exception was: " + repr(e)) return False return True def on_sig(self, nick, sigb64): """Processes transaction signatures from counterparties. - Returns True if all signatures received correctly, else - returns False + If all signatures received correctly, returns the result + of self.self_sign_and_push() (i.e. we complete the signing + and broadcast); else returns False (thus returns False for + all but last signature). """ if self.aborted: return False if nick not in self.nonrespondants: jlog.debug(('add_signature => nick={} ' 'not in nonrespondants {}').format(nick, self.nonrespondants)) - return - sig = hexlify(base64.b64decode(sigb64)).decode('ascii') + return False + sig = base64.b64decode(sigb64) inserted_sig = False - txhex = btc.serialize(self.latest_tx) # batch retrieval of utxo data utxo = {} ctr = 0 - for index, ins in enumerate(self.latest_tx['ins']): - utxo_for_checking = ins['outpoint']['hash'] + ':' + str(ins[ - 'outpoint']['index']) - #'deadbeef' markers mean our own input scripts are not '' - if (ins['script'] != ''): + for index, ins in enumerate(self.latest_tx.vin): + utxo_for_checking = (ins.prevout.hash[::-1], ins.prevout.n) + # 'deadbeef' markers mean our own input scripts are not queried + if ins.scriptSig != b"": continue utxo[ctr] = [index, utxo_for_checking] ctr += 1 utxo_data = jm_single().bc_interface.query_utxo_set([x[ 1] for x in utxo.values()]) - # insert signatures for i, u in iteritems(utxo): if utxo_data[i] is None: continue - #Check if the sender serialize_scripted the scriptCode - #item into the sig message; if so, also pick up the amount - #from the utxo data retrieved from the blockchain to verify - #the segwit-style signature. Note that this allows a mixed - #SW/non-SW transaction as each utxo is interpreted separately. - sig_deserialized = btc.deserialize_script(sig) - #verify_tx_input will not even parse the script if it has integers or None, - #so abort in case we were given a junk sig: - if not all([not isinstance(x, int) and x for x in sig_deserialized]): - jlog.warn("Junk signature: " + str(sig_deserialized) + \ + # Check if the sender included the scriptCode in the sig message; + # if so, also pick up the amount from the utxo data retrieved + # from the blockchain to verify the segwit-style signature. + # Note that this allows a mixed SW/non-SW transaction as each utxo + # is interpreted separately. + try: + sig_deserialized = [a for a in iter(btc.CScript(sig))] + except Exception as e: + jlog.debug("Failed to parse junk sig message, ignoring.") + break + # abort in case we were given a junk sig (note this previously had + # to check to avoid crashes in verify_tx_input, no longer (Feb 2020)): + if not all([x for x in sig_deserialized]): + jlog.debug("Junk signature: " + str(sig_deserialized) + \ ", not attempting to verify") break if len(sig_deserialized) == 2: @@ -583,44 +584,48 @@ class Taker(object): elif len(sig_deserialized) == 3: ver_sig, ver_pub, scriptCode = sig_deserialized else: - jlog.debug("Invalid signature message - more than 3 items") + jlog.debug("Invalid signature message - not 2 or 3 items") break + ver_amt = utxo_data[i]['value'] if scriptCode else None - sig_good = btc.verify_tx_input(txhex, u[0], utxo_data[i]['script'], - ver_sig, ver_pub, scriptCode=scriptCode, amount=ver_amt) - - if ver_amt is not None and not sig_good: - # Special case to deal with legacy bots 0.5.0 or lower: - # the third field in the sigmessage was originally *not* the - # scriptCode, but the contents of tx['ins'][index]['script'], - # i.e. the witness program 0014... ; for this we can verify - # implicitly, as verify_tx_input used to, by reconstructing - # from the public key. For these cases, we can *assume* that - # the input is of type p2sh-p2wpkh; we call the jmbitcoin method - # directly, as we cannot assume that *our* wallet handles this. - scriptCode = hexlify(btc.pubkey_to_p2pkh_script( - ver_pub, True)).decode('ascii') - sig_good = btc.verify_tx_input(txhex, u[0], utxo_data[i]['script'], - ver_sig, ver_pub, scriptCode=scriptCode, amount=ver_amt) + witness = btc.CScriptWitness( + [ver_sig, ver_pub]) if scriptCode else None + + # don't attempt to parse `pub` as pubkey unless it's valid. + if scriptCode: + try: + s = btc.pubkey_to_p2wpkh_script(ver_pub) + except: + jlog.debug("Junk signature message, invalid pubkey, ignoring.") + break + scriptSig = btc.CScript([ver_sig, ver_pub]) if not scriptCode else btc.CScript([s]) + + # Pre-Feb 2020, we used the third field scriptCode differently in + # pre- and post-0.5.0; now the scriptCode is implicit (i.e. calculated + # by underlying library, so that exceptional case is covered. + sig_good = btc.verify_tx_input(self.latest_tx, u[0], scriptSig, + btc.CScript(utxo_data[i]['script']), amount=ver_amt, witness=witness) + + # verification for the native case is functionally identical but + # adds another flag; so we can allow it here: + if not sig_good: + sig_good = btc.verify_tx_input(self.latest_tx, u[0], scriptSig, + btc.CScript(utxo_data[i]['script']), amount=ver_amt, + witness=witness, native=True) + # if passes, below code executes, and we should change for native: + scriptSig = btc.CScript([b""]) if sig_good: jlog.debug('found good sig at index=%d' % (u[0])) + + # Note that, due to the complexity of handling multisig or other + # arbitrary script (considering sending multiple signatures OTW), + # there is an assumption of p2sh-p2wpkh or p2wpkh, for the segwit + # case. + self.latest_tx.vin[u[0]].scriptSig = scriptSig if ver_amt: - # Note that, due to the complexity of handling multisig or other - # arbitrary script (considering sending multiple signatures OTW), - # there is an assumption of p2sh-p2wpkh or p2wpkh, for the segwit - # case. - self.latest_tx["ins"][u[0]]["txinwitness"] = [ver_sig, ver_pub] - if btc.is_segwit_native_script(utxo_data[i]['script']): - scriptSig = "" - else: - scriptSig = btc.serialize_script_unit( - btc.pubkey_to_p2wpkh_script(ver_pub)) - self.latest_tx["ins"][u[0]]["script"] = scriptSig - else: - # Non segwit (as per above comments) is limited only to single key, - # p2pkh case. - self.latest_tx["ins"][u[0]]["script"] = sig + self.latest_tx.wit.vtxinwit[u[0]] = btc.CTxInWitness( + btc.CScriptWitness(witness)) inserted_sig = True # check if maker has sent everything possible @@ -643,8 +648,8 @@ class Taker(object): # other guy sent a failed signature tx_signed = True - for ins in self.latest_tx['ins']: - if ins['script'] == '': + for ins in self.latest_tx.vin: + if ins.scriptSig == b"": tx_signed = False if not tx_signed: return False @@ -667,7 +672,7 @@ class Taker(object): def filter_by_coin_age_amt(utxos, age, amt): results = jm_single().bc_interface.query_utxo_set(utxos, - includeconf=True) + includeconf=True) newresults = [] too_old = [] too_small = [] @@ -683,7 +688,6 @@ class Taker(object): too_small.append(utxos[i]) if valid_age and valid_amt: newresults.append(utxos[i]) - return newresults, too_old, too_small def priv_utxo_pairs_from_utxos(utxos, age, amt): @@ -695,7 +699,7 @@ class Taker(object): age, amt) new_utxos_dict = {k: v for k, v in utxos.items() if k in new_utxos} for k, v in iteritems(new_utxos_dict): - addr = v['address'] + addr = self.wallet_service.script_to_addr(v["script"]) priv = self.wallet_service.get_key_from_addr(addr) if priv: #can be null from create-unsigned priv_utxo_pairs.append((priv, k)) @@ -723,7 +727,7 @@ class Taker(object): #in the transaction, about to be consumed, rather than use #random utxos that will persist after. At this step we also #allow use of external utxos in the json file. - if any(self.wallet_service.get_utxos_by_mixdepth(hexfmt=False).values()): + if any(self.wallet_service.get_utxos_by_mixdepth().values()): utxos = {} for mdutxo in self.wallet_service.get_utxos_by_mixdepth().values(): utxos.update(mdutxo) @@ -739,13 +743,9 @@ class Taker(object): ext_valid = None podle_data = generate_podle(priv_utxo_pairs, tries, ext_valid) if podle_data: - jlog.debug("Generated PoDLE: " + pprint.pformat(podle_data)) - revelation = PoDLE(u=podle_data['utxo'], - P=podle_data['P'], - P2=podle_data['P2'], - s=podle_data['sig'], - e=podle_data['e']).serialize_revelation() - return (commit_type_byte + podle_data["commit"], revelation, + jlog.debug("Generated PoDLE: " + repr(podle_data)) + return (commit_type_byte + bintohex(podle_data.commitment), + podle_data.serialize_revelation(), "Commitment sourced OK") else: errmsgheader, errmsg = generate_podle_error_string(priv_utxo_pairs, @@ -771,25 +771,26 @@ class Taker(object): def self_sign(self): # now sign it ourselves our_inputs = {} - for index, ins in enumerate(self.latest_tx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + for index, ins in enumerate(self.latest_tx.vin): + utxo = (ins.prevout.hash[::-1], ins.prevout.n) if utxo not in self.input_utxos.keys(): continue - script = self.wallet_service.addr_to_script(self.input_utxos[utxo]['address']) + script = self.input_utxos[utxo]["script"] amount = self.input_utxos[utxo]['value'] our_inputs[index] = (script, amount) - self.latest_tx = self.wallet_service.sign_tx(self.latest_tx, our_inputs) + success, msg = self.wallet_service.sign_tx(self.latest_tx, our_inputs) + if not success: + jlog.error("Failed to sign transaction: " + msg) def push(self): - tx = btc.serialize(self.latest_tx) - jlog.debug('\n' + tx) - self.txid = btc.txhash(tx) + jlog.debug('\n' + bintohex(self.latest_tx.serialize())) + self.txid = bintohex(self.latest_tx.GetTxid()[::-1]) jlog.info('txid = ' + self.txid) #If we are sending to a bech32 address, in case of sweep, will #need to use that bech32 for address import, which requires #converting to script (Core does not allow import of bech32) if self.my_cj_addr.lower()[:2] in ['bc', 'tb']: - notify_addr = btc.address_to_script(self.my_cj_addr) + notify_addr = btc.CCoinAddress(self.my_cj_addr).to_scriptPubKey() else: notify_addr = self.my_cj_addr #add the callbacks *before* pushing to ensure triggering; @@ -810,7 +811,7 @@ class Taker(object): tx_broadcast = jm_single().config.get('POLICY', 'tx_broadcast') nick_to_use = None if tx_broadcast == 'self': - pushed = jm_single().bc_interface.pushtx(tx) + pushed = jm_single().bc_interface.pushtx(self.latest_tx.serialize()) elif tx_broadcast in ['random-peer', 'not-self']: n = len(self.maker_utxo_data) if tx_broadcast == 'random-peer': @@ -818,14 +819,14 @@ class Taker(object): else: i = random.randrange(n) if i == n: - pushed = jm_single().bc_interface.pushtx(tx) + pushed = jm_single().bc_interface.pushtx(self.latest_tx.serialize()) else: nick_to_use = list(self.maker_utxo_data.keys())[i] pushed = True else: jlog.info("Only self, random-peer and not-self broadcast " "methods supported. Reverting to self-broadcast.") - pushed = jm_single().bc_interface.pushtx(tx) + pushed = jm_single().bc_interface.pushtx(self.latest_tx.serialize()) if not pushed: self.on_finished_callback(False, fromtx=True) else: @@ -841,7 +842,7 @@ class Taker(object): # Takers process only in series, so this should not occur: assert self.latest_tx is not None # check if the transaction matches our created tx: - if txd['outs'] != self.latest_tx['outs']: + if txd.vout != self.latest_tx.vout: return False return True @@ -928,12 +929,12 @@ class P2EPTaker(Taker): # For the p2ep taker, the variable 'my_cj_addr' is the destination: self.my_cj_addr = si[3] if isinstance(self.cjamount, float): - raise JMTakerError("P2EP coinjoin must use amount in satoshis") + raise JMTakerError("Payjoin must use amount in satoshis") if self.cjamount == 0: # Note that we don't allow sweep, currently, since the coin # choosing algo would not apply in that case (we'd have to rewrite # prepare_my_bitcoin_data for that case). - raise JMTakerError("P2EP coinjoin does not currently support sweep") + raise JMTakerError("Payjoin does not currently support sweep") # Next we prepare our coins with the inherited method # for this purpose; for this we must set the @@ -1004,27 +1005,27 @@ class P2EPTaker(Taker): if self.my_change_addr is not None: self.outputs.append({'address': self.my_change_addr, 'value': my_change_value}) - # As for JM coinjoins, the `None` key is used for our own inputs - # to the transaction; this preparatory version contains only those. - tx = btc.make_shuffled_tx(self.utxos[None], self.outputs, - False, 2, compute_tx_locktime()) - jlog.info('Created proposed fallback tx\n' + pprint.pformat( - btc.deserialize(tx))) + # Oour own inputs to the transaction; this preparatory version + # contains only those. + tx = btc.make_shuffled_tx(self.input_utxos, self.outputs, + version=2, locktime=compute_tx_locktime()) + jlog.info('Created proposed fallback tx\n' + pprint.pformat(str(tx))) # We now sign as a courtesy, because if we disappear the recipient # can still claim his coins with this. # sign our inputs before transfer our_inputs = {} - dtx = btc.deserialize(tx) - for index, ins in enumerate(dtx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - script = self.wallet_service.addr_to_script(self.input_utxos[utxo]['address']) - amount = self.input_utxos[utxo]['value'] - our_inputs[index] = (script, amount) - self.signed_noncj_tx = btc.serialize(self.wallet_service.sign_tx(dtx, our_inputs)) + for index, ins in enumerate(tx.vin): + utxo = (ins.prevout.hash[::-1], ins.prevout.n) + our_inputs[index] = (self.input_utxos[utxo]["script"], + self.input_utxos[utxo]['value']) + success, msg = self.wallet_service.sign_tx(tx, our_inputs) + if not success: + jlog.error("Failed to create backup transaction; error: " + msg) self.taker_info_callback("INFO", "Built tx proposal, sending to receiver.") - return (True, [self.p2ep_receiver_nick], self.signed_noncj_tx) + return (True, [self.p2ep_receiver_nick], bintohex(tx.serialize())) - def on_tx_received(self, nick, txhex): + @hexbin + def on_tx_received(self, nick, txser): """ Here the taker (payer) retrieves a version of the transaction from the maker (receiver) which should have the following properties: @@ -1041,12 +1042,14 @@ class P2EPTaker(Taker): and then broadcast (TODO broadcast delay or don't broadcast). """ try: - tx = btc.deserialize(txhex) - except (IndexError, SerializationError, SerializationTruncationError) as e: + tx = btc.CMutableTransaction.deserialize(txser) + except Exception as e: return (False, "malformed txhex. " + repr(e)) - jlog.info("Obtained tx from receiver:\n" + pprint.pformat(tx)) - cjaddr_script = btc.address_to_script(self.my_cj_addr) - changeaddr_script = btc.address_to_script(self.my_change_addr) + jlog.info("Obtained tx from receiver:\n" + pprint.pformat(str(tx))) + cjaddr_script = btc.CCoinAddress( + self.my_cj_addr).to_scriptPubKey() + changeaddr_script = btc.CCoinAddress( + self.my_change_addr).to_scriptPubKey() # We ensure that the coinjoin address and our expected change # address are still in the outputs, once (with the caveat that @@ -1054,19 +1057,19 @@ class P2EPTaker(Taker): # of dust change, which we assess after). times_seen_cj_addr = 0 times_seen_change_addr = 0 - for outs in tx['outs']: - if outs['script'] == cjaddr_script: + for outs in tx.vout: + if outs.scriptPubKey == cjaddr_script: times_seen_cj_addr += 1 - new_cj_amount = outs['value'] + new_cj_amount = outs.nValue if new_cj_amount < self.cjamount: # This is a violation of protocol; # receiver must be providing extra bitcoin # as input, so his receiving amount should have increased. return (False, 'Wrong cj_amount. I expect at least' + str(self.cjamount)) - if outs['script'] == changeaddr_script: + if outs.scriptPubKey == changeaddr_script: times_seen_change_addr += 1 - new_change_amount = outs['value'] + new_change_amount = outs.nValue if times_seen_cj_addr != 1: fmt = ('cj addr not in tx outputs once, #cjaddr={}').format return (False, (fmt(times_seen_cj_addr))) @@ -1077,8 +1080,7 @@ class P2EPTaker(Taker): new_change_amount = 0 # Check that our inputs are present. - tx_utxo_set = set(ins['outpoint']['hash'] + ':' + str( - ins['outpoint']['index']) for ins in tx['ins']) + tx_utxo_set = set((ins.prevout.hash[::-1], ins.prevout.n) for ins in tx.vin) if not tx_utxo_set.issuperset(set(self.utxos[None])): return (False, "my utxos are not contained") # Check that the sequence numbers of all inputs are unaltered @@ -1087,10 +1089,10 @@ class P2EPTaker(Taker): # Note that this is hacky and is most elegantly addressed by # use of PSBT (although any object encapsulation of tx input # would serve the same purpose). - if tx["locktime"] == 0: + if tx.nLockTime == 0: return (False, "Invalid PayJoin v0 transaction: locktime 0") - for i in tx["ins"]: - if i["sequence"] != 0xffffffff - 1: + for i in tx.vin: + if i.nSequence != 0xffffffff - 1: return (False, "Invalid PayJoin v0 transaction: "+\ "sequence is not 0xffffffff -1") @@ -1099,7 +1101,7 @@ class P2EPTaker(Taker): # not) of PayJoin to sweep utxos at no cost. # (TODO This is very kludgy, more sophisticated approach # should be used in future): - if len(tx["ins"]) - len (self.utxos[None]) > 5: + if len(tx.vin) - len (self.utxos[None]) > 5: return (False, "proposed tx has more than 5 inputs from " "the recipient, which is too expensive.") @@ -1122,9 +1124,8 @@ class P2EPTaker(Taker): # checking input validity and transaction balance. retrieve_utxos = {} ctr = 0 - for index, ins in enumerate(tx['ins']): - utxo_for_checking = ins['outpoint']['hash'] + ':' + str( - ins['outpoint']['index']) + for index, ins in enumerate(tx.vin): + utxo_for_checking = (ins.prevout.hash[::-1], ins.prevout.n) if utxo_for_checking in self.utxos[None]: continue retrieve_utxos[ctr] = [index, utxo_for_checking] @@ -1140,32 +1141,11 @@ class P2EPTaker(Taker): if utxo_data[i] is None: return (False, "Proposed transaction contains invalid utxos") total_receiver_input += utxo_data[i]["value"] - scriptCode = None - ver_amt = None idx = retrieve_utxos[i][0] - if "txinwitness" in tx["ins"][idx]: - ver_amt = utxo_data[i]["value"] - try: - ver_sig, ver_pub = tx["ins"][idx]["txinwitness"] - except Exception as e: - print("Segwit error: ", repr(e)) - return (False, "Segwit input not of expected type, " - "either p2sh-p2wpkh or p2wpkh") - # note that the scriptCode is the same whether nested or not - # also note that the scriptCode has to be inferred if we are - # only given a transaction serialization. - scriptCode = "76a914" + btc.hash160(unhexlify(ver_pub)) + "88ac" - else: - scriptSig = btc.deserialize_script(tx["ins"][idx]["script"]) - if len(scriptSig) != 2: - return (False, - "Proposed transaction contains unsupported input type") - ver_sig, ver_pub = scriptSig - if not btc.verify_tx_input(txhex, idx, - utxo_data[i]['script'], - ver_sig, ver_pub, - scriptCode=scriptCode, - amount=ver_amt): + if not btc.verify_tx_input(tx, idx, tx.vin[idx].scriptSig, + btc.CScript(utxo_data[i]['script']), + amount=utxo_data[i]["value"], + witness=tx.wit.vtxinwit[idx].scriptWitness): return (False, "Proposed transaction is not correctly signed.") payment = new_cj_amount - total_receiver_input @@ -1185,7 +1165,7 @@ class P2EPTaker(Taker): # our fee estimator. Its return value will be governed by our own fee settings # in joinmarket.cfg; allow either (a) automatic agreement for any value within # a range of 0.3 to 3x this figure, or (b) user to agree on prompt. - fee_est = estimate_tx_fee(len(tx['ins']), len(tx['outs']), + fee_est = estimate_tx_fee(len(tx.vin), len(tx.vout), txtype=self.wallet_service.get_txtype()) fee_ok = False if btc_fee > 0.3 * fee_est and btc_fee < 3 * fee_est: diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index 8524018..4f32544 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -5,15 +5,13 @@ import os import sys import time import numbers - -from jmbase import get_log, jmprint +from jmbase import get_log, jmprint, bintohex from .configure import jm_single, validate_address, is_burn_destination from .schedule import human_readable_schedule_entry, tweak_tumble_schedule,\ schedule_to_text from .wallet import BaseWallet, estimate_tx_fee, compute_tx_locktime, \ FidelityBondMixin -from jmbitcoin import deserialize, make_shuffled_tx, serialize, txhash,\ - amount_to_str, mk_burn_script, bin_hash160 +from jmbitcoin import make_shuffled_tx, amount_to_str, mk_burn_script from jmbase.support import EXIT_SUCCESS log = get_log() @@ -136,13 +134,20 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, log.info("Using a fee of : " + amount_to_str(fee_est) + ".") if amount != 0: log.info("Using a change value of: " + amount_to_str(changeval) + ".") - txsigned = sign_tx(wallet_service, make_shuffled_tx( - list(utxos.keys()), outs, False, 2, tx_locktime), utxos) + tx = make_shuffled_tx(list(utxos.keys()), outs, 2, compute_tx_locktime()) + list(utxos.keys()), outs, 2, tx_locktime), utxos) + inscripts = {} + for i, txinp in enumerate(tx.vin): + u = (txinp.prevout.hash[::-1], txinp.prevout.n) + inscripts[i] = (utxos[u]["script"], utxos[u]["value"]) + success, msg = wallet_service.sign_tx(tx, inscripts) + if not success: + log.error("Failed to sign transaction, quitting. Error msg: " + msg) + return log.info("Got signed transaction:\n") - log.info(pformat(txsigned)) - tx = serialize(txsigned) + log.info(pformat(str(tx))) log.info("In serialized form (for copy-paste):") - log.info(tx) + log.info(bintohex(tx.serialize())) actual_amount = amount if amount != 0 else total_inputs_val - fee_est log.info("Sends: " + amount_to_str(actual_amount) + " to destination: " + destination) if not answeryes: @@ -151,12 +156,12 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, log.info("You chose not to broadcast the transaction, quitting.") return False else: - accepted = accept_callback(pformat(txsigned), destination, actual_amount, + accepted = accept_callback(pformat(str(tx)), destination, actual_amount, fee_est) if not accepted: return False - jm_single().bc_interface.pushtx(tx) - txid = txhash(tx) + jm_single().bc_interface.pushtx(tx.serialize()) + txid = bintohex(tx.GetTxid()[::-1]) successmsg = "Transaction sent: " + txid cb = log.info if not info_callback else info_callback cb(successmsg) diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index 5159a0d..2a62a28 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -27,7 +27,7 @@ from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH,\ from .support import get_random_bytes from . import mn_encode, mn_decode import jmbitcoin as btc -from jmbase import JM_WALLET_NAME_PREFIX +from jmbase import JM_WALLET_NAME_PREFIX, bintohex """ @@ -198,7 +198,8 @@ class UTXOManager(object): assert isinstance(index, numbers.Integral) assert isinstance(mixdepth, numbers.Integral) - return self._utxo[mixdepth].pop((txid, index)) + x = self._utxo[mixdepth].pop((txid, index)) + return x def add_utxo(self, txid, index, path, value, mixdepth, height=None): # Assumed: that we add a utxo only if we want it enabled, @@ -433,19 +434,22 @@ class BaseWallet(object): Add signatures to transaction for inputs referenced by scripts. args: - tx: transaction dict + tx: CMutableTransaction object scripts: {input_index: (output_script, amount)} kwargs: additional arguments for engine.sign_transaction returns: - input transaction dict with added signatures, hex-encoded. + True, None if success. + False, msg if signing failed, with error msg. """ for index, (script, amount) in scripts.items(): assert amount > 0 path = self.script_to_path(script) privkey, engine = self._get_key_from_path(path) - tx = btc.deserialize(engine.sign_transaction(tx, index, privkey, - amount, **kwargs)) - return tx + sig, msg = engine.sign_transaction(tx, index, privkey, + amount, **kwargs) + if not sig: + return False, msg + return True, None @deprecated def get_key_from_addr(self, addr): @@ -455,7 +459,7 @@ class BaseWallet(object): script = self._ENGINE.address_to_script(addr) path = self.script_to_path(script) privkey = self._get_key_from_path(path)[0] - return hexlify(privkey).decode('ascii') + return privkey def _get_addr_int_ext(self, address_type, mixdepth): if address_type == self.ADDRESS_TYPE_EXTERNAL: @@ -578,32 +582,19 @@ class BaseWallet(object): """ self.save() - @deprecated def remove_old_utxos(self, tx): - tx = deepcopy(tx) - for inp in tx['ins']: - inp['outpoint']['hash'] = unhexlify(inp['outpoint']['hash']) - - ret = self.remove_old_utxos_(tx) - - removed_utxos = {} - for (txid, index), val in ret.items(): - val['address'] = self.get_address_from_path(val['path']) - removed_utxos[hexlify(txid).decode('ascii') + ':' + str(index)] = val - return removed_utxos - - def remove_old_utxos_(self, tx): """ Remove all own inputs of tx from internal utxo list. args: - tx: transaction dict + tx: CMutableTransaction returns: {(txid, index): {'script': bytes, 'path': str, 'value': int} for all removed utxos """ removed_utxos = {} - for inp in tx['ins']: - txid, index = inp['outpoint']['hash'], inp['outpoint']['index'] + for inp in tx.vin: + txid = inp.prevout.hash[::-1] + index = inp.prevout.n md = self._utxos.have_utxo(txid, index) if md is False: continue @@ -614,46 +605,32 @@ class BaseWallet(object): 'value': value} return removed_utxos - @deprecated - def add_new_utxos(self, tx, txid, height=None): - tx = deepcopy(tx) - for out in tx['outs']: - out['script'] = unhexlify(out['script']) - - ret = self.add_new_utxos_(tx, unhexlify(txid), height=height) - - added_utxos = {} - for (txid_bin, index), val in ret.items(): - addr = self.get_address_from_path(val['path']) - val['address'] = addr - added_utxos[txid + ':' + str(index)] = val - return added_utxos - - def add_new_utxos_(self, tx, txid, height=None): + def add_new_utxos(self, tx, height=None): """ Add all outputs of tx for this wallet to internal utxo list. - + They are also returned in standard dict form. args: - tx: transaction dict + tx: CMutableTransaction height: blockheight in which tx was included, or None if unconfirmed. returns: - {(txid, index): {'script': bytes, 'path': tuple, 'value': int} + {(txid, index): {'script': bytes, 'path': tuple, 'value': int, + 'address': str} for all added utxos """ - assert isinstance(txid, bytes) and len(txid) == self._utxos.TXID_LEN added_utxos = {} - for index, outs in enumerate(tx['outs']): + txid = tx.GetTxid()[::-1] + for index, outs in enumerate(tx.vout): + spk = outs.scriptPubKey + val = outs.nValue try: - self.add_utxo(txid, index, outs['script'], outs['value'], - height=height) + self.add_utxo(txid, index, spk, val, height=height) except WalletError: continue - path = self.script_to_path(outs['script']) - added_utxos[(txid, index)] = {'script': outs['script'], - 'path': path, - 'value': outs['value']} + path = self.script_to_path(spk) + added_utxos[(txid, index)] = {'script': spk, 'path': path, 'value': val, + 'address': self._ENGINE.script_to_address(spk)} return added_utxos def add_utxo(self, txid, index, script, value, height=None): @@ -669,9 +646,10 @@ class BaseWallet(object): mixdepth = self._get_mixdepth_from_path(path) self._utxos.add_utxo(txid, index, path, value, mixdepth, height=height) - def process_new_tx(self, txd, txid, height=None): - """ Given a newly seen transaction, deserialized as txd and - with transaction id, process its inputs and outputs and update + def process_new_tx(self, txd, height=None): + """ Given a newly seen transaction, deserialized as + CMutableTransaction txd, + process its inputs and outputs and update the utxo contents of this wallet accordingly. NOTE: this should correctly handle transactions that are not actually related to the wallet; it will not add (or remove, @@ -679,30 +657,15 @@ class BaseWallet(object): functions check this condition. """ removed_utxos = self.remove_old_utxos(txd) - added_utxos = self.add_new_utxos(txd, txid, height=height) + added_utxos = self.add_new_utxos(txd, height=height) return (removed_utxos, added_utxos) - @deprecated - def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None, - maxheight=None): - utxo_filter_new = None - if utxo_filter: - utxo_filter_new = [(unhexlify(utxo[:64]), int(utxo[65:])) - for utxo in utxo_filter] - ret = self.select_utxos_(mixdepth, amount, utxo_filter_new, select_fn, - maxheight=maxheight) - ret_conv = {} - for utxo, data in ret.items(): - addr = self.get_address_from_path(data['path']) - utxo_txt = hexlify(utxo[0]).decode('ascii') + ':' + str(utxo[1]) - ret_conv[utxo_txt] = {'address': addr, 'value': data['value']} - return ret_conv - - def select_utxos_(self, mixdepth, amount, utxo_filter=None, - select_fn=None, maxheight=None): + def select_utxos(self, mixdepth, amount, utxo_filter=None, + select_fn=None, maxheight=None, includeaddr=False): """ Select a subset of available UTXOS for a given mixdepth whose value is - greater or equal to amount. + greater or equal to amount. If `includeaddr` is True, adds an `address` + key to the returned dict. args: mixdepth: int, mixdepth to select utxos from, must be smaller or @@ -713,6 +676,7 @@ class BaseWallet(object): returns: {(txid, index): {'script': bytes, 'path': tuple, 'value': int}} + """ assert isinstance(mixdepth, numbers.Integral) assert isinstance(amount, numbers.Integral) @@ -728,7 +692,8 @@ class BaseWallet(object): for data in ret.values(): data['script'] = self.get_script_from_path(data['path']) - + if includeaddr: + data["address"] = self.get_address_from_path(data["path"]) return ret def disable_utxo(self, txid, index, disable=True): @@ -758,21 +723,7 @@ class BaseWallet(object): include_disabled=include_disabled, maxheight=maxheight) - @deprecated - def get_utxos_by_mixdepth(self, verbose=True, includeheight=False): - # TODO: verbose - ret = self.get_utxos_by_mixdepth_(includeheight=includeheight) - - utxos_conv = collections.defaultdict(dict) - for md, utxos in ret.items(): - for utxo, data in utxos.items(): - utxo_str = hexlify(utxo[0]).decode('ascii') + ':' + str(utxo[1]) - addr = self.get_address_from_path(data['path']) - data['address'] = addr - utxos_conv[md][utxo_str] = data - return utxos_conv - - def get_utxos_by_mixdepth_(self, include_disabled=False, includeheight=False): + def get_utxos_by_mixdepth(self, include_disabled=False, includeheight=False): """ Get all UTXOs for active mixdepths. @@ -791,9 +742,11 @@ class BaseWallet(object): if not include_disabled and self._utxos.is_disabled(*utxo): continue script = self.get_script_from_path(path) + addr = self.get_address_from_path(path) script_utxos[md][utxo] = {'script': script, 'path': path, - 'value': value} + 'value': value, + 'address': addr} if includeheight: script_utxos[md][utxo]['height'] = height return script_utxos @@ -1101,7 +1054,6 @@ class ImportWalletMixin(object): raise WalletError("Unsupported key type for imported keys.") privkey, key_type_wif = self._ENGINE.wif_to_privkey(wif) - # FIXME: there is no established standard for encoding key type in wif #if key_type is not None and key_type_wif is not None and \ # key_type != key_type_wif: diff --git a/jmclient/jmclient/wallet_service.py b/jmclient/jmclient/wallet_service.py index ef186b2..3552105 100644 --- a/jmclient/jmclient/wallet_service.py +++ b/jmclient/jmclient/wallet_service.py @@ -16,8 +16,9 @@ from jmclient.output import fmt_tx_data from jmclient.blockchaininterface import (INF_HEIGHT, BitcoinCoreInterface, BitcoinCoreNoHistoryInterface) from jmclient.wallet import FidelityBondMixin -from jmbase.support import jmprint, EXIT_SUCCESS -import jmbitcoin as btc +from jmbase.support import jmprint, EXIT_SUCCESS, utxo_to_utxostr, bintohex, hextobin +from jmbitcoin import lx + """Wallet service The purpose of this independent service is to allow @@ -173,9 +174,9 @@ class WalletService(Service): and return type boolean. `txinfo` - either a txid expected for the transaction, if known, or a tuple of the ordered output set, of the form - (('script': script), ('value': value), ..). This can be - constructed from jmbitcoin.deserialize output, key "outs", - using tuple(). See WalletService.transaction_monitor(). + ((CScript, int), ..). This is be constructed from the + CMutableTransaction vout list. + See WalletService.transaction_monitor(). `cb_type` - must be one of "all", "unconfirmed", "confirmed"; the first type will be called back once for every new transaction, the second only once when the number of @@ -223,7 +224,9 @@ class WalletService(Service): self.bci.import_addresses([address], self.EXTERNAL_WALLET_LABEL, restart_cb=self.restart_callback) - def default_autofreeze_warning_cb(self, utxostr): + def default_autofreeze_warning_cb(self, utxo): + success, utxostr = utxo_to_utxostr(utxo) + assert success, "Autofreeze warning cb called with invalid utxo." jlog.warning("WARNING: new utxo has been automatically " "frozen to prevent forced address reuse: ") jlog.warning(utxostr) @@ -267,9 +270,7 @@ class WalletService(Service): utxo]["value"] <= freeze_threshold: # freezing of coins must be communicated to user: self.autofreeze_warning_cb(utxo) - # process_new_tx returns added utxos in str format: - txidstr, idx = utxo.split(":") - self.disable_utxo(binascii.unhexlify(txidstr), int(idx)) + self.disable_utxo(*utxo) def transaction_monitor(self): """Keeps track of any changes in the wallet (new transactions). @@ -298,7 +299,7 @@ class WalletService(Service): for tx in new_txs: txid = tx["txid"] - res = self.bci.get_transaction(txid) + res = self.bci.get_transaction(hextobin(txid)) if not res: continue confs = res["confirmations"] @@ -317,7 +318,7 @@ class WalletService(Service): txd = self.bci.get_deser_from_gettransaction(res) if txd is None: continue - removed_utxos, added_utxos = self.wallet.process_new_tx(txd, txid, height) + removed_utxos, added_utxos = self.wallet.process_new_tx(txd, height) if txid not in self.processed_txids: # apply checks to disable/freeze utxos to reused addrs if needed: self.check_for_reuse(added_utxos) @@ -339,16 +340,9 @@ class WalletService(Service): f(txd, txid) # The tuple given as the second possible key for the dict - # is such because dict keys must be hashable types, so a simple - # replication of the entries in the list tx["outs"], where tx - # was generated via jmbitcoin.deserialize, is unacceptable to - # Python, since they are dicts. However their keyset is deterministic - # so it is sufficient to convert these dicts to tuples with fixed - # ordering, thus it can be used as a key into the self.callbacks - # dicts. (This is needed because txid is not always available + # is such because txid is not always available # at the time of callback registration). - possible_keys = [txid, tuple( - (x["script"], x["value"]) for x in txd["outs"])] + possible_keys = [txid, tuple((x.scriptPubKey, x.nValue) for x in txd.vout)] # note that len(added_utxos) > 0 is not a sufficient condition for # the tx being new, since wallet.add_new_utxos will happily re-add @@ -406,8 +400,9 @@ class WalletService(Service): def report_changed(x, utxos): if len(utxos.keys()) > 0: jlog.info(x + ' utxos=\n{}'.format('\n'.join( - '{} - {}'.format(u, fmt_tx_data(tx_data, self)) - for u, tx_data in utxos.items()))) + '{} - {}'.format(utxo_to_utxostr(u)[1], + fmt_tx_data(tx_data, self)) for u, + tx_data in utxos.items()))) report_changed("Removed", removed_utxos) report_changed("Added", added_utxos) @@ -752,14 +747,15 @@ class WalletService(Service): def _add_unspent_txo(self, utxo, height): """ Add a UTXO as returned by rpc's listunspent call to the wallet. - + Note that these are returned as little endian outpoint txids, so + are converted. params: utxo: single utxo dict as returned by listunspent current_blockheight: blockheight as integer, used to set the block in which a confirmed utxo is included. """ - txid = binascii.unhexlify(utxo['txid']) - script = binascii.unhexlify(utxo['scriptPubKey']) + txid = hextobin(utxo['txid']) + script = hextobin(utxo['scriptPubKey']) value = int(Decimal(str(utxo['amount'])) * Decimal('1e8')) self.add_utxo(txid, int(utxo['vout']), script, value, height) @@ -774,11 +770,9 @@ class WalletService(Service): self.wallet.save() def get_utxos_by_mixdepth(self, include_disabled=False, - verbose=False, hexfmt=True, includeconfs=False): + verbose=False, includeconfs=False): """ Returns utxos by mixdepth in a dict, optionally including information about how many confirmations each utxo has. - TODO clean up underlying wallet.get_utxos_by_mixdepth (including verbosity - and formatting options) to make this less confusing. """ def height_to_confs(x): # convert height entries to confirmations: @@ -793,36 +787,30 @@ class WalletService(Service): confs = self.current_blockheight - h + 1 ubym_conv[m][u]["confs"] = confs return ubym_conv + ubym = self.wallet.get_utxos_by_mixdepth( + include_disabled=include_disabled, includeheight=includeconfs) + if not includeconfs: + return ubym + else: + return height_to_confs(ubym) - if hexfmt: - ubym = self.wallet.get_utxos_by_mixdepth(verbose=verbose, - includeheight=includeconfs) - if not includeconfs: - return ubym - else: - return height_to_confs(ubym) + def minconfs_to_maxheight(self, minconfs): + if minconfs is None: + return None else: - ubym = self.wallet.get_utxos_by_mixdepth_( - include_disabled=include_disabled, includeheight=includeconfs) - if not includeconfs: - return ubym - else: - return height_to_confs(ubym) + return self.current_blockheight - minconfs + 1 def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None, - minconfs=None): + minconfs=None, includeaddr=False): """ Request utxos from the wallet in a particular mixdepth to satisfy a certain total amount, optionally set the selector function (or use the currently configured function set by the wallet, and optionally require a minimum of minconfs confirmations (default none means unconfirmed are allowed). """ - if minconfs is None: - maxheight = None - else: - maxheight = self.current_blockheight - minconfs + 1 return self.wallet.select_utxos(mixdepth, amount, utxo_filter=utxo_filter, - select_fn=select_fn, maxheight=maxheight) + select_fn=select_fn, maxheight=self.minconfs_to_maxheight(minconfs), + includeaddr=includeaddr) def get_balance_by_mixdepth(self, verbose=True, include_disabled=False, diff --git a/jmclient/jmclient/wallet_utils.py b/jmclient/jmclient/wallet_utils.py index 6770a9e..8aa3818 100644 --- a/jmclient/jmclient/wallet_utils.py +++ b/jmclient/jmclient/wallet_utils.py @@ -16,7 +16,8 @@ from jmclient import (get_network, WALLET_IMPLEMENTATIONS, Storage, podle, LegacyWallet, SegwitWallet, FidelityBondMixin, FidelityBondWatchonlyWallet, is_native_segwit_mode, load_program_config, add_base_options, check_regtest) from jmclient.wallet_service import WalletService -from jmbase.support import get_password, jmprint, EXIT_FAILURE, EXIT_ARGERROR +from jmbase.support import (get_password, jmprint, EXIT_FAILURE, + EXIT_ARGERROR, utxo_to_utxostr) from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH, \ TYPE_SEGWIT_LEGACY_WALLET_FIDELITY_BONDS @@ -369,22 +370,26 @@ def wallet_showutxos(wallet, showprivkey): utxos = wallet.get_utxos_by_mixdepth(includeconfs=True) for md in utxos: for u, av in utxos[md].items(): + success, us = utxo_to_utxostr(u) + assert success key = wallet.get_key_from_addr(av['address']) tries = podle.get_podle_tries(u, key, max_tries) tries_remaining = max(0, max_tries - tries) - unsp[u] = {'address': av['address'], 'value': av['value'], + unsp[us] = {'address': av['address'], 'value': av['value'], 'tries': tries, 'tries_remaining': tries_remaining, 'external': False, 'confirmations': av['confs']} if showprivkey: - unsp[u]['privkey'] = wallet.get_wif_path(av['path']) + unsp[us]['privkey'] = wallet.get_wif_path(av['path']) used_commitments, external_commitments = podle.get_podle_commitments() for u, ec in iteritems(external_commitments): + success, us = utxo_to_utxostr(u) + assert success tries = podle.get_podle_tries(utxo=u, max_tries=max_tries, external=True) tries_remaining = max(0, max_tries - tries) - unsp[u] = {'tries': tries, 'tries_remaining': tries_remaining, + unsp[us] = {'tries': tries, 'tries_remaining': tries_remaining, 'external': True} return json.dumps(unsp, indent=4) @@ -431,7 +436,7 @@ def wallet_display(wallet_service, showprivkey, displayall=False, acctlist = [] # TODO - either optionally not show disabled utxos, or # mark them differently in display (labels; colors) - utxos = wallet_service.get_utxos_by_mixdepth(include_disabled=True, hexfmt=False) + utxos = wallet_service.get_utxos_by_mixdepth(include_disabled=True) for m in range(wallet_service.mixdepth + 1): branchlist = [] for address_type in [0, 1]: diff --git a/jmclient/jmclient/yieldgenerator.py b/jmclient/jmclient/yieldgenerator.py index b18eb90..f89af36 100644 --- a/jmclient/jmclient/yieldgenerator.py +++ b/jmclient/jmclient/yieldgenerator.py @@ -132,7 +132,8 @@ class YieldGeneratorBasic(YieldGenerator): change_addr = self.wallet_service.get_internal_addr(mixdepth) - utxos = self.wallet_service.select_utxos(mixdepth, total_amount, minconfs=1) + utxos = self.wallet_service.select_utxos(mixdepth, total_amount, + minconfs=1, includeaddr=True) my_total_in = sum([va['value'] for va in utxos.values()]) real_cjfee = calc_cj_fee(offer["ordertype"], offer["cjfee"], amount) change_value = my_total_in - amount - offer["txfee"] + real_cjfee @@ -141,7 +142,8 @@ class YieldGeneratorBasic(YieldGenerator): 'finding new utxos').format(change_value)) try: utxos = self.wallet_service.select_utxos(mixdepth, - total_amount + jm_single().DUST_THRESHOLD, minconfs=1) + total_amount + jm_single().DUST_THRESHOLD, + minconfs=1, includeaddr=True) except Exception: jlog.info('dont have the required UTXOs to make a ' 'output above the dust threshold, quitting') diff --git a/jmclient/test/commontest.py b/jmclient/test/commontest.py index 01ce37a..36ed7c4 100644 --- a/jmclient/test/commontest.py +++ b/jmclient/test/commontest.py @@ -6,11 +6,13 @@ import binascii import random from decimal import Decimal -from jmbase import get_log +from jmbase import (get_log, hextobin, utxostr_to_utxo, + utxo_to_utxostr, listchanger, dictchanger) + from jmclient import ( jm_single, open_test_wallet_maybe, estimate_tx_fee, BlockchainInterface, get_p2sh_vbyte, BIP32Wallet, - SegwitLegacyWallet, WalletService) + SegwitLegacyWallet, WalletService, BTC_P2SH_P2WPKH) from jmbase.support import chunks import jmbitcoin as btc @@ -45,9 +47,11 @@ class DummyBlockchainInterface(BlockchainInterface): pass def is_address_imported(self, addr): pass - + + def get_current_block_height(self): + return 10**6 + def pushtx(self, txhex): - print("pushing: " + str(txhex)) return True def insert_fake_query_results(self, fqr): @@ -56,7 +60,7 @@ class DummyBlockchainInterface(BlockchainInterface): def setQUSFail(self, state): self.qusfail = state - def query_utxo_set(self, txouts,includeconf=False): + def query_utxo_set(self, txouts, includeconf=False): if self.qusfail: #simulate failure to find the utxo return [None] @@ -72,12 +76,14 @@ class DummyBlockchainInterface(BlockchainInterface): known_outs = {"03243f4a659e278a1333f8308f6aaf32db4692ee7df0340202750fd6c09150f6:1": "03a2d1cbe977b1feaf8d0d5cc28c686859563d1520b28018be0c2661cf1ebe4857", "498faa8b22534f3b443c6b0ce202f31e12f21668b4f0c7a005146808f250d4c3:0": "02b4b749d54e96b04066b0803e372a43d6ffa16e75a001ae0ed4b235674ab286be", "3f3ea820d706e08ad8dc1d2c392c98facb1b067ae4c671043ae9461057bd2a3c:1": "023bcbafb4f68455e0d1d117c178b0e82a84e66414f0987453d78da034b299c3a9"} + known_outs = dictchanger(known_outs) #our wallet utxos, faked, for podle tests: utxos are doctored (leading 'f'), #and the lists are (amt, age) wallet_outs = {'f34b635ed8891f16c4ec5b8236ae86164783903e8e8bb47fa9ef2ca31f3c2d7a:0': [10000000, 2], 'f780d6e5e381bff01a3519997bb4fcba002493103a198fde334fd264f9835d75:1': [20000000, 6], 'fe574db96a4d43a99786b3ea653cda9e4388f377848f489332577e018380cff1:0': [50000000, 3], 'fd9711a2ef340750db21efb761f5f7d665d94b312332dc354e252c77e9c48349:0': [50000000, 6]} + wallet_outs = dictchanger(wallet_outs) if includeconf and set(txouts).issubset(set(wallet_outs)): #includeconf used as a trigger for a podle check; @@ -88,16 +94,16 @@ class DummyBlockchainInterface(BlockchainInterface): 'confirms': wallet_outs[to][1]}) return results if txouts[0] in known_outs: - addr = btc.pubkey_to_p2sh_p2wpkh_address( - known_outs[txouts[0]], get_p2sh_vbyte()) + scr = BTC_P2SH_P2WPKH.pubkey_to_script(known_outs[txouts[0]]) + addr = btc.CCoinAddress.from_scriptPubKey(scr) return [{'value': 200000000, 'address': addr, - 'script': btc.address_to_script(addr), + 'script': scr, 'confirms': 20}] for t in txouts: result_dict = {'value': 10000000000, 'address': "mrcNu71ztWjAQA6ww9kHiW3zBWSQidHXTQ", - 'script': '76a91479b000887626b294a914501a4cd226b58b23598388ac'} + 'script': hextobin('76a91479b000887626b294a914501a4cd226b58b23598388ac')} if includeconf: result_dict['confirms'] = 20 result.append(result_dict) @@ -110,18 +116,10 @@ class DummyBlockchainInterface(BlockchainInterface): def create_wallet_for_sync(wallet_structure, a, **kwargs): #We need a distinct seed for each run so as not to step over each other; #make it through a deterministic hash - seedh = btc.sha256("".join([str(x) for x in a]))[:32] + seedh = btc.b2x(btc.Hash("".join([str(x) for x in a]).encode("utf-8")))[:32] return make_wallets( 1, [wallet_structure], fixed_seeds=[seedh], **kwargs)[0]['wallet'] - -def binarize_tx(tx): - for o in tx['outs']: - o['script'] = binascii.unhexlify(o['script']) - for i in tx['ins']: - i['outpoint']['hash'] = binascii.unhexlify(i['outpoint']['hash']) - - def make_sign_and_push(ins_full, wallet_service, amount, @@ -130,7 +128,12 @@ def make_sign_and_push(ins_full, hashcode=btc.SIGHASH_ALL, estimate_fee = False): """Utility function for easily building transactions - from wallets + from wallets. + `ins_full` should be a list of dicts in format returned + by wallet.select_utxos: + {(txid, index): {"script":..,"value":..,"path":..}} + ... although the path is not used. + The "script" and "value" data is used to allow signing. """ assert isinstance(wallet_service, WalletService) total = sum(x['value'] for x in ins_full.values()) @@ -143,22 +146,21 @@ def make_sign_and_push(ins_full, 'address': output_addr}, {'value': total - amount - fee_est, 'address': change_addr}] - de_tx = btc.deserialize(btc.mktx(ins, outs)) + tx = btc.mktx(ins, outs) scripts = {} - for index, ins in enumerate(de_tx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - script = wallet_service.addr_to_script(ins_full[utxo]['address']) - scripts[index] = (script, ins_full[utxo]['value']) - binarize_tx(de_tx) - de_tx = wallet_service.sign_tx(de_tx, scripts, hashcode=hashcode) + for i, j in enumerate(ins): + scripts[i] = (ins_full[j]["script"], ins_full[j]["value"]) + + success, msg = wallet_service.sign_tx(tx, scripts, hashcode=hashcode) + if not success: + return False #pushtx returns False on any error - push_succeed = jm_single().bc_interface.pushtx(btc.serialize(de_tx)) + push_succeed = jm_single().bc_interface.pushtx(tx.serialize()) if push_succeed: - txid = btc.txhash(btc.serialize(de_tx)) # in normal operation this happens automatically # but in some tests there is no monitoring loop: - wallet_service.process_new_tx(de_tx, txid) - return txid + wallet_service.process_new_tx(tx) + return tx.GetTxid()[::-1] else: return False diff --git a/jmclient/test/test_client_protocol.py b/jmclient/test/test_client_protocol.py index 0ced712..7583a55 100644 --- a/jmclient/test/test_client_protocol.py +++ b/jmclient/test/test_client_protocol.py @@ -1,7 +1,8 @@ #! /usr/bin/env python '''test client-protocol interfacae.''' -from jmbase import get_log +from jmbase import get_log, bintohex, hextobin +from jmbase.commands import * from jmclient import load_test_config, Taker,\ JMClientProtocolFactory, jm_single, Maker, WalletService from jmclient.client_protocol import JMTakerClientProtocol @@ -14,12 +15,10 @@ from twisted.protocols.amp import UnknownRemoteError from twisted.protocols import amp from twisted.trial import unittest from twisted.test import proto_helpers -from jmbase.commands import * from taker_test_data import t_raw_signed_tx from commontest import default_max_cj_fee import json import jmbitcoin as bitcoin - import twisted twisted.internet.base.DelayedCall.debug = True @@ -97,7 +96,7 @@ class DummyMaker(Maker): def on_auth_received(self, nick, offer, commitment, cr, amount, kphex): # success, utxos, auth_pub, cj_addr, change_addr, btc_sig - return True, [], '', '', '', '' + return True, [], b"", '', '', '' def on_tx_received(self, nick, txhex, offerinfo): # success, sigs @@ -227,8 +226,8 @@ class JMTestServerProtocol(JMBaseProtocol): self.defaultCallbacks(d2) #To test, this must include a valid ecdsa sig fullmsg = "fullmsgforverify" - priv = "aa"*32 + "01" - pub = bitcoin.privkey_to_pubkey(priv) + priv = b"\xaa"*32 + b"\x01" + pub = bintohex(bitcoin.privkey_to_pubkey(priv)) sig = bitcoin.ecdsa_sign(fullmsg, priv) d3 = self.callRemote(JMRequestMsgSigVerify, msg="msgforverify", @@ -262,7 +261,7 @@ class JMTestServerProtocolFactory(protocol.ServerFactory): class DummyClientProtocolFactory(JMClientProtocolFactory): def buildProtocol(self, addr): - return JMTakerClientProtocol(self, self.client, nick_priv="aa"*32) + return JMTakerClientProtocol(self, self.client, nick_priv=b"\xaa"*32 + b"\x01") class TrialTestJMClientProto(unittest.TestCase): @@ -364,8 +363,8 @@ class TestMakerClientProtocol(unittest.TestCase): @inlineCallbacks def test_JMRequestMsgSigVerify(self): fullmsg = 'fullmsgforverify' - priv = 'aa'*32 + '01' - pub = bitcoin.privkey_to_pubkey(priv) + priv = b"\xaa"*32 + b"\x01" + pub = bintohex(bitcoin.privkey_to_pubkey(priv)) sig = bitcoin.ecdsa_sign(fullmsg, priv) yield self.init_client() yield self.callClient( diff --git a/jmclient/test/test_coinjoin.py b/jmclient/test/test_coinjoin.py index 6428a2b..810bf71 100644 --- a/jmclient/test/test_coinjoin.py +++ b/jmclient/test/test_coinjoin.py @@ -6,14 +6,15 @@ Test doing full coinjoins, bypassing IRC import os import sys import pytest +import copy from twisted.internet import reactor -from jmbase import get_log +from jmbase import get_log, hextobin, bintohex from jmclient import load_test_config, jm_single,\ YieldGeneratorBasic, Taker, LegacyWallet, SegwitLegacyWallet,\ NO_ROUNDING from jmclient.podle import set_commitment_file -from commontest import make_wallets, binarize_tx, default_max_cj_fee +from commontest import make_wallets, default_max_cj_fee from test_taker import dummy_filter_orderbook import jmbitcoin as btc @@ -85,14 +86,14 @@ def init_coinjoin(taker, makers, orderbook, cj_amount): ioauth_data = list(response[1:]) ioauth_data[0] = list(ioauth_data[0].keys()) # maker_pk which is set up by jmdaemon - ioauth_data.append(None) + ioauth_data.append("00") maker_data[mid] = ioauth_data # this is handled by jmdaemon active_orders[mid]['utxos'] = response[1] active_orders[mid]['cjaddr'] = ioauth_data[2] active_orders[mid]['changeaddr'] = ioauth_data[3] - active_orders[mid]['offer'] = m.offerlist[0] + active_orders[mid]['offer'] = copy.deepcopy(m.offerlist[0]) active_orders[mid]['amount'] = cj_amount return active_orders, maker_data @@ -194,13 +195,12 @@ def test_coinjoin_mixdepth_wrap_taker(monkeypatch, tmpdir, setup_cj): taker_final_result = do_tx_signing(taker, makers, active_orders, txdata) assert taker_final_result is not False - tx = btc.deserialize(txdata[2]) - binarize_tx(tx) + tx = btc.CMutableTransaction.deserialize(hextobin(txdata[2])) wallet_service = wallet_services[-1] # TODO change for new tx monitoring: - wallet_service.remove_old_utxos_(tx) - wallet_service.add_new_utxos_(tx, b'\x00' * 32) # fake txid + wallet_service.remove_old_utxos(tx) + wallet_service.add_new_utxos(tx) balances = wallet_service.get_balance_by_mixdepth() assert balances[0] == cj_amount @@ -250,14 +250,13 @@ def test_coinjoin_mixdepth_wrap_maker(monkeypatch, tmpdir, setup_cj): taker_final_result = do_tx_signing(taker, makers, active_orders, txdata) assert taker_final_result is not False - tx = btc.deserialize(txdata[2]) - binarize_tx(tx) + tx = btc.CMutableTransaction.deserialize(hextobin(txdata[2])) for i in range(MAKER_NUM): wallet_service = wallet_services[i] # TODO as above re: monitoring - wallet_service.remove_old_utxos_(tx) - wallet_service.add_new_utxos_(tx, b'\x00' * 32) # fake txid + wallet_service.remove_old_utxos(tx) + wallet_service.add_new_utxos(tx) balances = wallet_service.get_balance_by_mixdepth() assert balances[0] == cj_amount diff --git a/jmclient/test/test_commitment_utils.py b/jmclient/test/test_commitment_utils.py index 31000ab..070e7e8 100644 --- a/jmclient/test/test_commitment_utils.py +++ b/jmclient/test/test_commitment_utils.py @@ -4,11 +4,14 @@ import pytest from jmclient import (load_test_config, jm_single) from jmclient.commitment_utils import get_utxo_info, validate_utxo_data - +from jmbitcoin import select_chain_params def test_get_utxo_info(): load_test_config() + # this test tests mainnet keys, so temporarily switch network + select_chain_params("bitcoin") jm_single().config.set("BLOCKCHAIN", "network", "mainnet") + dbci = DummyBlockchainInterface() privkey = "L1RrrnXkcKut5DEMwtDthjwRcTTwED36thyL1DebVrKuwvohjMNi" #to verify use from_wif_privkey and privkey_to_address @@ -28,7 +31,7 @@ def test_get_utxo_info(): with pytest.raises(Exception) as e_info: u, priv = get_utxo_info(fakeutxo + privkey) #invalid index - fu2 = "ab"*32 + ":00004" + fu2 = "ab"*32 + ":-1" with pytest.raises(Exception) as e_info: u, priv = get_utxo_info(fu2 + "," + privkey) #invalid privkey @@ -53,3 +56,5 @@ def test_get_utxo_info(): retval = validate_utxo_data(utxodatas, False) assert not retval dbci.setQUSFail(False) + select_chain_params("bitcoin/regtest") + jm_single().config.set("BLOCKCHAIN", "network", "mainnet") diff --git a/jmclient/test/test_maker.py b/jmclient/test/test_maker.py index 1bc53e9..a7046f2 100644 --- a/jmclient/test/test_maker.py +++ b/jmclient/test/test_maker.py @@ -29,8 +29,7 @@ def construct_tx_offerlist(cjaddr, changeaddr, maker_utxos, maker_utxos_value, 'txfee': 0 } - utxos = { utxo['outpoint']['hash'] + ':' + str(utxo['outpoint']['index']): - {'utxo': utxo, 'value': maker_utxos_value} for utxo in maker_utxos } + utxos = { utxo: {'utxo': utxo, 'value': maker_utxos_value} for utxo in maker_utxos } offerlist = { 'utxos': utxos, @@ -46,59 +45,56 @@ def construct_tx_offerlist(cjaddr, changeaddr, maker_utxos, maker_utxos_value, def create_tx_inputs(count=1): inp = [] for i in range(count): - inp.append({'outpoint': {'hash': '0'*64, 'index': i}, - 'script': '', - 'sequence': 4294967295}) + inp.append((b"\x00"*32, i)) return inp -def create_tx_outputs(*scripts_amount): +def create_tx_outputs(*addrs_amount): outp = [] - for script, amount in scripts_amount: - outp.append({'script': script, 'value': amount}) + for addr, amount in addrs_amount: + outp.append({'address': addr, 'value': amount}) return outp def address_p2pkh_generator(): - return get_address_generator(b'\x76\xa9\x14', b'\x88\xac', get_p2pk_vbyte()) + return get_address_generator(b'\x76\xa9\x14', b'\x88\xac') def address_p2sh_generator(): - return get_address_generator(b'\xa9\x14', b'\x87', get_p2sh_vbyte()) + return get_address_generator(b'\xa9\x14', b'\x87', p2sh=True) -def get_address_generator(script_pre, script_post, vbyte): +def get_address_generator(script_pre, script_post, p2sh=False): counter = 0 while True: script = script_pre + struct.pack(b'=LQQ', 0, 0, counter) + script_post - addr = btc.script_to_address(script, vbyte) - yield addr, binascii.hexlify(script).decode('ascii') + if p2sh: + addr = btc.CCoinAddress.from_scriptPubKey( + btc.CScript(script).to_p2sh_scriptPubKey()) + else: + addr = btc.CCoinAddress.from_scriptPubKey(btc.CScript(script)) + yield str(addr), binascii.hexlify(script).decode('ascii') counter += 1 -def create_tx_and_offerlist(cj_addr, cj_change_addr, other_output_scripts, - cj_script=None, cj_change_script=None, offertype='swreloffer'): - assert len(other_output_scripts) % 2 == 0, "bug in test" +def create_tx_and_offerlist(cj_addr, cj_change_addr, other_output_addrs, + offertype='swreloffer'): + assert len(other_output_addrs) % 2 == 0, "bug in test" cj_value = 100000000 maker_total_value = cj_value*3 - if cj_script is None: - cj_script = btc.address_to_script(cj_addr) - if cj_change_script is None: - cj_change_script = btc.address_to_script(cj_change_addr) - inputs = create_tx_inputs(3) outputs = create_tx_outputs( - (cj_script, cj_value), - (cj_change_script, maker_total_value - cj_value), # cjfee=0, txfee=0 - *((script, cj_value + (i%2)*(50000000+i)) \ - for i, script in enumerate(other_output_scripts)) + (cj_addr, cj_value), + (cj_change_addr, maker_total_value - cj_value), # cjfee=0, txfee=0 + *((addr, cj_value + (i%2)*(50000000+i)) \ + for i, addr in enumerate(other_output_addrs)) ) maker_utxos = [inputs[0]] - tx = btc.deserialize(btc.mktx(inputs, outputs)) + tx = btc.mktx(inputs, outputs) offerlist = construct_tx_offerlist(cj_addr, cj_change_addr, maker_utxos, maker_total_value, cj_value, offertype) @@ -119,21 +115,20 @@ def test_verify_unsigned_tx_sw_valid(setup_env_nodeps): # test standard cj tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr, - [next(p2sh_gen)[1] for s in range(4)], cj_script, cj_change_script) + [next(p2sh_gen)[0] for s in range(4)]) assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "standard sw cj" # test cj with mixed outputs tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr, - list(chain((next(p2sh_gen)[1] for s in range(3)), - (next(p2pkh_gen)[1] for s in range(1)))), - cj_script, cj_change_script) + list(chain((next(p2sh_gen)[0] for s in range(3)), + (next(p2pkh_gen)[0] for s in range(1))))) assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "sw cj with p2pkh output" # test cj with only p2pkh outputs tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr, - [next(p2pkh_gen)[1] for s in range(4)], cj_script, cj_change_script) + [next(p2pkh_gen)[0] for s in range(4)]) assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "sw cj with only p2pkh outputs" @@ -152,21 +147,20 @@ def test_verify_unsigned_tx_nonsw_valid(setup_env_nodeps): # test standard cj tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr, - [next(p2pkh_gen)[1] for s in range(4)], cj_script, cj_change_script, 'reloffer') + [next(p2pkh_gen)[0] for s in range(4)], offertype='reloffer') assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "standard nonsw cj" # test cj with mixed outputs tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr, - list(chain((next(p2sh_gen)[1] for s in range(1)), - (next(p2pkh_gen)[1] for s in range(3)))), - cj_script, cj_change_script, 'reloffer') + list(chain((next(p2sh_gen)[0] for s in range(1)), + (next(p2pkh_gen)[0] for s in range(3)))), offertype='reloffer') assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "nonsw cj with p2sh output" # test cj with only p2sh outputs tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr, - [next(p2sh_gen)[1] for s in range(4)], cj_script, cj_change_script, 'reloffer') + [next(p2sh_gen)[0] for s in range(4)], offertype='reloffer') assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "nonsw cj with only p2sh outputs" @@ -175,4 +169,5 @@ def test_verify_unsigned_tx_nonsw_valid(setup_env_nodeps): def setup_env_nodeps(monkeypatch): monkeypatch.setattr(jmclient.configure, 'get_blockchain_interface_instance', lambda x: DummyBlockchainInterface()) + btc.select_chain_params("bitcoin/regtest") load_test_config() diff --git a/jmclient/test/test_payjoin.py b/jmclient/test/test_payjoin.py index 3670100..6d0ba14 100644 --- a/jmclient/test/test_payjoin.py +++ b/jmclient/test/test_payjoin.py @@ -75,14 +75,11 @@ def final_checks(wallet_services, amount, txfee, tsb, msb, source_mixdepth=0): return True @pytest.mark.parametrize('wallet_cls, wallet_structures, mean_amt', - [([LegacyWallet, LegacyWallet], - [[4, 0, 0, 0, 0]] * 2, 1.0), + [ # note we have removed LegacyWallet test cases. ([SegwitLegacyWallet, SegwitLegacyWallet], [[1, 3, 0, 0, 0]] * 2, 2.0), ([SegwitWallet, SegwitWallet], [[1, 0, 0, 0, 0]] * 2, 4.0), - ([LegacyWallet, SegwitWallet], - [[4, 0, 0, 0, 0]] * 2, 1.0), ([SegwitLegacyWallet, SegwitWallet], [[1, 3, 0, 0, 0]] * 2, 2.0), ([SegwitWallet, SegwitLegacyWallet], diff --git a/jmclient/test/test_podle.py b/jmclient/test/test_podle.py index d49fe80..a643b7e 100644 --- a/jmclient/test/test_podle.py +++ b/jmclient/test/test_podle.py @@ -28,8 +28,8 @@ def test_commitment_retries(setup_podle): """ allowed = jm_single().config.getint("POLICY", "taker_utxo_retries") #make some pretend commitments - dummy_priv_utxo_pairs = [(bitcoin.sha256(os.urandom(10)), - bitcoin.sha256(os.urandom(10))+":0") for _ in range(10)] + dummy_priv_utxo_pairs = [(bitcoin.Hash(os.urandom(10)), + bitcoin.b2x(bitcoin.Hash(os.urandom(10)))+":0") for _ in range(10)] #test a single commitment request of all 10 for x in dummy_priv_utxo_pairs: p = generate_podle([x], allowed) @@ -46,15 +46,15 @@ def generate_single_podle_sig(priv, i): This calls the underlying 'raw' code based on the class PoDLE, not the library 'generate_podle' which intelligently searches and updates commitments. """ - dummy_utxo = bitcoin.sha256(priv) + ":3" - podle = PoDLE(dummy_utxo, binascii.hexlify(priv).decode('ascii')) + dummy_utxo = bitcoin.b2x(bitcoin.Hash(priv)) + ":3" + podle = PoDLE(dummy_utxo, priv) r = podle.generate_podle(i) return (r['P'], r['P2'], r['sig'], r['e'], r['commit']) def test_rand_commitments(setup_podle): for i in range(20): - priv = os.urandom(32) + priv = os.urandom(32)+b"\x01" Pser, P2ser, s, e, commitment = generate_single_podle_sig(priv, 1 + i%5) assert verify_podle(Pser, P2ser, s, e, commitment) #tweak commitments to verify failure @@ -90,7 +90,7 @@ def test_external_commitments(setup_podle): tries = jm_single().config.getint("POLICY","taker_utxo_retries") for i in range(10): priv = os.urandom(32) - dummy_utxo = bitcoin.sha256(priv)+":2" + dummy_utxo = (bitcoin.Hash(priv), 2) ecs[dummy_utxo] = {} ecs[dummy_utxo]['reveal']={} for j in range(tries): @@ -104,16 +104,16 @@ def test_external_commitments(setup_podle): assert external[u]['P'] == ecs[u]['P'] for i in range(tries): for x in ['P2', 's', 'e']: - assert external[u]['reveal'][str(i)][x] == ecs[u]['reveal'][i][x] + assert external[u]['reveal'][i][x] == ecs[u]['reveal'][i][x] #add a dummy used commitment, then try again - update_commitments(commitment="ab"*32) + update_commitments(commitment=b"\xab"*32) ecs = {} known_commits = [] known_utxos = [] tries = 3 for i in range(1, 6): - u = binascii.hexlify(struct.pack(b'B', i)*32).decode('ascii') + u = (struct.pack(b'B', i)*32, i+3) known_utxos.append(u) priv = struct.pack(b'B', i)*32+b"\x01" ecs[u] = {} @@ -131,8 +131,9 @@ def test_external_commitments(setup_podle): #this should find the remaining one utxo and return from it assert generate_podle([], max_tries=tries, allow_external=known_utxos) #test commitment removal - to_remove = ecs[binascii.hexlify(struct.pack(b'B', 3)*32).decode('ascii')] - update_commitments(external_to_remove={binascii.hexlify(struct.pack(b'B', 3)*32).decode('ascii'):to_remove}) + tru = (struct.pack(b"B", 3)*32, 3+3) + to_remove = {tru: ecs[tru]} + update_commitments(external_to_remove=to_remove) #test that an incorrectly formatted file raises with open(get_commitment_file(), "rb") as f: validjson = json.loads(f.read().decode('utf-8')) @@ -152,14 +153,14 @@ def test_podle_constructor(setup_podle): """Tests rules about construction of PoDLE object are conformed to. """ - priv = "aa"*32 + priv = b"\xaa"*32 #pub and priv together not allowed with pytest.raises(PoDLEError) as e_info: p = PoDLE(priv=priv, P="dummypub") #no pub or priv is allowed, i forget if this is useful for something p = PoDLE() #create from priv - p = PoDLE(priv=priv+"01", u="dummyutxo") + p = PoDLE(priv=priv+b"\x01", u=(struct.pack(b"B", 7)*32, 4)) pdict = p.generate_podle(2) assert all([k in pdict for k in ['used', 'utxo', 'P', 'P2', 'commit', 'sig', 'e']]) #using the valid data, serialize/deserialize test @@ -181,7 +182,7 @@ def test_podle_constructor(setup_podle): with pytest.raises(PoDLEError) as e_info: p.generate_podle(0) #Test construction from pubkey - pub = bitcoin.privkey_to_pubkey(priv+"01") + pub = bitcoin.privkey_to_pubkey(priv+b"\x01") p = PoDLE(P=pub) with pytest.raises(PoDLEError) as e_info: p.get_commitment() diff --git a/jmbitcoin/test/test_keys.py b/jmclient/test/test_privkeys.py similarity index 75% rename from jmbitcoin/test/test_keys.py rename to jmclient/test/test_privkeys.py index 06c48fd..2fb6884 100644 --- a/jmbitcoin/test/test_keys.py +++ b/jmclient/test/test_privkeys.py @@ -2,13 +2,16 @@ '''Public and private key validity and formatting tests.''' import jmbitcoin as btc +from jmclient import (BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, + jm_single, load_test_config) import binascii +import struct import json import pytest import os testdir = os.path.dirname(os.path.realpath(__file__)) -def test_read_raw_privkeys(): +def test_read_raw_privkeys(setup_keys): badkeys = [b'', b'\x07'*31,b'\x07'*34, b'\x07'*33] for b in badkeys: with pytest.raises(Exception) as e_info: @@ -18,7 +21,7 @@ def test_read_raw_privkeys(): c, k = btc.read_privkey(g[0]) assert c == g[1] -def test_wif_privkeys_invalid(): +def test_wif_privkeys_invalid(setup_keys): #first try to create wif privkey from key of wrong length bad_privs = [b'\x01\x02'*17] #some silly private key but > 33 bytes @@ -27,7 +30,7 @@ def test_wif_privkeys_invalid(): for priv in bad_privs: with pytest.raises(Exception) as e_info: - fake_wif = btc.wif_compressed_privkey(binascii.hexlify(priv).decode('ascii')) + fake_wif = BTCEngine.privkey_to_wif(priv) #Create a wif with wrong length bad_wif1 = btc.bin_to_b58check(b'\x01\x02'*34, b'\x80') @@ -35,7 +38,7 @@ def test_wif_privkeys_invalid(): bad_wif2 = btc.bin_to_b58check(b'\x07'*33, b'\x80') for bw in [bad_wif1, bad_wif2]: with pytest.raises(Exception) as e_info: - fake_priv = btc.from_wif_privkey(bw) + fake_priv, keytype = BTCEngine.wif_to_privkey(bw) #Some invalid b58 from bitcoin repo; #none of these are valid as any kind of key or address @@ -49,15 +52,14 @@ def test_wif_privkeys_invalid(): print('testing this key: ' + bad_key) #should throw exception with pytest.raises(Exception) as e_info: - from_wif_key = btc.from_wif_privkey(bad_key, - btc.get_version_byte(bad_key)) + from_wif_key, keytype = BTCEngine.wif_to_privkey(bad_key) #in case the b58 check encoding is valid, we should #also check if the leading version byte is in the #expected set, and throw an error if not. if chr(btc.get_version_byte(bad_key)) not in b'\x80\xef': raise Exception("Invalid version byte") -def test_wif_privkeys_valid(): +def test_wif_privkeys_valid(setup_keys): with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f: json_data = f.read() valid_keys_list = json.loads(json_data) @@ -65,16 +67,20 @@ def test_wif_privkeys_valid(): key, hex_key, prop_dict = a if prop_dict["isPrivkey"]: netval = "testnet" if prop_dict["isTestnet"] else "mainnet" + jm_single().config.set("BLOCKCHAIN", "network", netval) print('testing this key: ' + key) assert btc.get_version_byte( key) in b'\x80\xef', "not valid network byte" comp = prop_dict["isCompressed"] - from_wif_key = btc.from_wif_privkey( - key, - compressed=comp, - vbyte=btc.from_int_to_byte(btc.from_byte_to_int(btc.get_version_byte(key))-128)) - expected_key = hex_key - if comp: expected_key += '01' + if not comp: + # we only handle compressed keys + continue + from_wif_key, keytype = BTCEngine.wif_to_privkey(key) + expected_key = binascii.unhexlify(hex_key) + b"\x01" assert from_wif_key == expected_key, "Incorrect key decoding: " + \ str(from_wif_key) + ", should be: " + str(expected_key) + jm_single().config.set("BLOCKCHAIN", "network", "testnet") +@pytest.fixture(scope='module') +def setup_keys(): + load_test_config() \ No newline at end of file diff --git a/jmclient/test/test_taker.py b/jmclient/test/test_taker.py index bc00d78..3cb0bd3 100644 --- a/jmclient/test/test_taker.py +++ b/jmclient/test/test_taker.py @@ -10,13 +10,21 @@ import pytest import json import struct from base64 import b64encode +from jmbase import (utxostr_to_utxo, utxo_to_utxostr, hextobin, + dictchanger, listchanger) from jmclient import load_test_config, jm_single, set_commitment_file,\ get_commitment_file, SegwitLegacyWallet, Taker, VolatileStorage,\ - get_network, WalletService, NO_ROUNDING + get_network, WalletService, NO_ROUNDING, BTC_P2PKH from taker_test_data import t_utxos_by_mixdepth, t_orderbook,\ t_maker_response, t_chosen_orders, t_dummy_ext from commontest import default_max_cj_fee +def convert_utxos(utxodict): + return_dict = {} + for uk, val in utxodict.items(): + return_dict[utxostr_to_utxo(uk)[1]] = val + return return_dict + class DummyWallet(SegwitLegacyWallet): def __init__(self): storage = VolatileStorage() @@ -36,32 +44,30 @@ class DummyWallet(SegwitLegacyWallet): script = self._ENGINE.address_to_script(data['address']) self._script_map[script] = path - def get_utxos_by_mixdepth(self, verbose=True, includeheight=False): - return t_utxos_by_mixdepth - - def get_utxos_by_mixdepth_(self, verbose=True, include_disabled=False, - includeheight=False): - utxos = self.get_utxos_by_mixdepth(verbose) - - utxos_conv = {} - for md, utxo_data in utxos.items(): - md_utxo = utxos_conv.setdefault(md, {}) - for i, (utxo_hex, data) in enumerate(utxo_data.items()): - utxo, index = utxo_hex.split(':') - data_conv = { - 'script': self._ENGINE.address_to_script(data['address']), - 'path': (b'dummy', md, i), - 'value': data['value'] - } - md_utxo[(binascii.unhexlify(utxo), int(index))] = data_conv - - return utxos_conv + def get_utxos_by_mixdepth(self, include_disabled=False, verbose=True, + includeheight=False): + # utxostr conversion routines because taker_test_data uses hex: + retval = {} + for mixdepth, v in t_utxos_by_mixdepth.items(): + retval[mixdepth] = {} + for i, (utxo, val) in enumerate(v.items()): + retval[mixdepth][utxostr_to_utxo(utxo)[1]] = val + val["script"] = self._ENGINE.address_to_script(val['address']) + val["path"] = (b'dummy', mixdepth, i) + return retval def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None, - maxheight=None): + maxheight=None, includeaddr=False): if amount > self.get_balance_by_mixdepth()[mixdepth]: raise Exception("Not enough funds") - return t_utxos_by_mixdepth[mixdepth] + # comment as for get_utxos_by_mixdepth: + retval = {} + for k, v in t_utxos_by_mixdepth[mixdepth].items(): + success, u = utxostr_to_utxo(k) + assert success + retval[u] = v + retval[u]["script"] = self.addr_to_script(retval[u]["address"]) + return retval def get_internal_addr(self, mixing_depth, bci=None): if self.inject_addr_get_failure: @@ -70,7 +76,7 @@ class DummyWallet(SegwitLegacyWallet): def sign_tx(self, tx, addrs): print("Pretending to sign on addresses: " + str(addrs)) - return tx + return True, None def sign(self, tx, i, priv, amount): """Sign a transaction; the amount field @@ -97,10 +103,10 @@ class DummyWallet(SegwitLegacyWallet): musGZczug3BAbqobmYherywCwL9REgNaNm """ for p in privs: - addrs[p] = bitcoin.privkey_to_address(p, False, magicbyte=0x6f) + addrs[p] = BTC_P2PKH.privkey_to_address(p) for p, a in iteritems(addrs): if a == addr: - return binascii.hexlify(p).decode('ascii') + return p raise ValueError("No such keypair") def _is_my_bip32_path(self, path): @@ -170,7 +176,7 @@ def test_make_commitment(setup_taker, failquery, external): amount = 110000000 taker = get_taker([(mixdepth, amount, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", NO_ROUNDING)]) taker.cjamount = amount - taker.input_utxos = t_utxos_by_mixdepth[0] + taker.input_utxos = convert_utxos(t_utxos_by_mixdepth[0]) if failquery: jm_single().bc_interface.setQUSFail(True) taker.make_commitment() @@ -194,12 +200,13 @@ def test_auth_pub_not_found(setup_taker): res = taker.initialize(orderbook) taker.orderbook = copy.deepcopy(t_chosen_orders) #total_cjfee unaffected, all same maker_response = copy.deepcopy(t_maker_response) - utxos = ["03243f4a659e278a1333f8308f6aaf32db4692ee7df0340202750fd6c09150f6:1", - "498faa8b22534f3b443c6b0ce202f31e12f21668b4f0c7a005146808f250d4c3:0", - "3f3ea820d706e08ad8dc1d2c392c98facb1b067ae4c671043ae9461057bd2a3c:1"] + utxos = [utxostr_to_utxo(x)[1] for x in [ + "03243f4a659e278a1333f8308f6aaf32db4692ee7df0340202750fd6c09150f6:1", + "498faa8b22534f3b443c6b0ce202f31e12f21668b4f0c7a005146808f250d4c3:0", + "3f3ea820d706e08ad8dc1d2c392c98facb1b067ae4c671043ae9461057bd2a3c:1"]] fake_query_results = [{'value': 200000000, 'address': "mrKTGvFfYUEqk52qPKUroumZJcpjHLQ6pn", - 'script': '76a914767c956efe6092a775fea39a06d1cac9aae956d788ac', + 'script': hextobin('76a914767c956efe6092a775fea39a06d1cac9aae956d788ac'), 'utxo': utxos[i], 'confirms': 20} for i in range(3)] jm_single().bc_interface.insert_fake_query_results(fake_query_results) @@ -223,8 +230,8 @@ def test_auth_pub_not_found(setup_taker): ([(0, 199850001, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False, 2, False, None, None), #trigger sub dust change for taker #edge case triggers that do fail - ([(0, 199850000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False, - 2, False, None, None), #trigger negative change + ([(0, 199851000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False, + 2, False, None, None), #trigger negative change ([(0, 199599800, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False, 2, False, None, None), #trigger sub dust change for maker ([(0, 20000000, 3, "INTERNAL", 0, NO_ROUNDING)], True, False, @@ -232,7 +239,7 @@ def test_auth_pub_not_found(setup_taker): ([(0, 20000000, 3, "INTERNAL", 0, NO_ROUNDING)], False, False, 7, False, None, None), #test not enough cp ([(0, 80000000, 3, "INTERNAL", 0, NO_ROUNDING)], False, False, - 2, False, None, "30000"), #test failed commit + 2, False, None, "30000"), #test failed commit ([(0, 20000000, 3, "INTERNAL", 0, NO_ROUNDING)], False, False, 2, True, None, None), #test unauthed response ([(0, 5000000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, True, @@ -282,14 +289,15 @@ def test_taker_init(setup_taker, schedule, highfee, toomuchcoins, minmakers, if notauthed: #Doctor one of the maker response data fields maker_response["J659UPUSLLjHJpaB"][1] = "xx" #the auth pub - if schedule[0][1] == 199850000: + if schedule[0][1] == 199851000: #triggers negative change - #makers offer 3000 txfee; we estimate ~ 147*10 + 2*34 + 10=1548 bytes - #times 30k = 46440, so we pay 43440, plus maker fees = 3*0.0002*200000000 - #roughly, gives required selected = amt + 163k, hence the above = - #2btc - 150k sats = 199850000 (tweaked because of aggressive coin selection) + #((109 + 4*64)*ins + 34 * outs + 8)/4. plug in 9 ins and 8 outs gives + #tx size estimate = 1101 bytes. Times 30 ~= 33030. + #makers offer 3000 txfee, so we pay 30030, plus maker fees = 3*0.0002*200000000 + #roughly, gives required selected = amt + 120k+30k, hence the above = + #2btc - 140k sats = 199851000 (tweaked because of aggressive coin selection) #simulate the effect of a maker giving us a lot more utxos - taker.utxos["dummy_for_negative_change"] = ["a", "b", "c", "d", "e"] + taker.utxos["dummy_for_negative_change"] = [(struct.pack(b"B", a) *32, a+1) for a in range(7,12)] with pytest.raises(ValueError) as e_info: res = taker.receive_utxos(maker_response) return clean_up() @@ -361,7 +369,7 @@ def test_taker_init(setup_taker, schedule, highfee, toomuchcoins, minmakers, [ (7), ]) -def test_unconfirm_confirm(schedule_len): +def test_unconfirm_confirm(setup_taker, schedule_len): """These functions are: do-nothing by default (unconfirm, for Taker), and merely update schedule index for confirm (useful for schedules/tumbles). This tests that the on_finished callback correctly reports the fromtx @@ -369,20 +377,26 @@ def test_unconfirm_confirm(schedule_len): The exception to the above is that the txd passed in must match self.latest_tx, so we use a dummy value here for that. """ + class DummyTx(object): + pass test_unconfirm_confirm.txflag = True def finished_for_confirms(res, fromtx=False, waittime=0, txdetails=None): assert res #confirmed should always send true test_unconfirm_confirm.txflag = fromtx taker = get_taker(schedule_len=schedule_len, on_finished=finished_for_confirms) - taker.latest_tx = {"outs": "blah"} - taker.unconfirm_callback({"ins": "foo", "outs": "blah"}, "b") + taker.latest_tx = DummyTx() + taker.latest_tx.vout = "blah" + fake_txd = DummyTx() + fake_txd.vin = "foo" + fake_txd.vout = "blah" + taker.unconfirm_callback(fake_txd, "b") for i in range(schedule_len-1): taker.schedule_index += 1 - fromtx = taker.confirm_callback({"ins": "foo", "outs": "blah"}, "b", 1) + fromtx = taker.confirm_callback(fake_txd, "b", 1) assert test_unconfirm_confirm.txflag taker.schedule_index += 1 - fromtx = taker.confirm_callback({"ins": "foo", "outs": "blah"}, "b", 1) + fromtx = taker.confirm_callback(fake_txd, "b", 1) assert not test_unconfirm_confirm.txflag @pytest.mark.parametrize( @@ -396,16 +410,13 @@ def test_on_sig(setup_taker, dummyaddr, schedule): #then, create a signature with various inputs, pass in in b64 to on_sig. #in order for it to verify, the DummyBlockchainInterface will have to #return the right values in query_utxo_set - + utxos = [(struct.pack(b"B", x) * 32, 1) for x in range(5)] #create 2 privkey + utxos that are to be ours privs = [x*32 + b"\x01" for x in [struct.pack(b'B', y) for y in range(1,6)]] - utxos = [str(x)*64+":1" for x in range(5)] - fake_query_results = [{'value': 200000000, - 'utxo': utxos[x], - 'address': bitcoin.privkey_to_address(privs[x], False, magicbyte=0x6f), - 'script': bitcoin.mk_pubkey_script( - bitcoin.privkey_to_address(privs[x], False, magicbyte=0x6f)), - 'confirms': 20} for x in range(5)] + scripts = [BTC_P2PKH.privkey_to_script(privs[x]) for x in range(5)] + addrs = [BTC_P2PKH.privkey_to_address(privs[x]) for x in range(5)] + fake_query_results = [{'value': 200000000, 'utxo': utxos[x], 'address': addrs[x], + 'script': scripts[x], 'confirms': 20} for x in range(5)] dbci = DummyBlockchainInterface() dbci.insert_fake_query_results(fake_query_results) @@ -414,47 +425,52 @@ def test_on_sig(setup_taker, dummyaddr, schedule): outs = [{'value': 100000000, 'address': dummyaddr}, {'value': 899990000, 'address': dummyaddr}] tx = bitcoin.mktx(utxos, outs) - - de_tx = bitcoin.deserialize(tx) + # since tx will be updated as it is signed, unlike in real life + # (where maker signing operation doesn't happen here), we'll create + # a second copy without the signatures: + tx2 = bitcoin.mktx(utxos, outs) + #prepare the Taker with the right intermediate data taker = get_taker(schedule=schedule) taker.nonrespondants=["cp1", "cp2", "cp3"] - taker.latest_tx = de_tx + taker.latest_tx = tx #my inputs are the first 2 utxos taker.input_utxos = {utxos[0]: - {'address': bitcoin.privkey_to_address(privs[0], False, magicbyte=0x6f), - 'script': bitcoin.mk_pubkey_script( - bitcoin.privkey_to_address(privs[0], False, magicbyte=0x6f)), + {'address': addrs[0], + 'script': scripts[0], 'value': 200000000}, utxos[1]: - {'address': bitcoin.privkey_to_address(privs[1], False, magicbyte=0x6f), - 'script': bitcoin.mk_pubkey_script( - bitcoin.privkey_to_address(privs[1], False, magicbyte=0x6f)), + {'address': addrs[1], + 'script': scripts[1], 'value': 200000000}} taker.utxos = {None: utxos[:2], "cp1": [utxos[2]], "cp2": [utxos[3]], "cp3":[utxos[4]]} for i in range(2): # placeholders required for my inputs - taker.latest_tx['ins'][i]['script'] = 'deadbeef' + taker.latest_tx.vin[i].scriptSig = bitcoin.CScript(hextobin('deadbeef')) + tx2.vin[i].scriptSig = bitcoin.CScript(hextobin('deadbeef')) #to prepare for my signing, need to mark cjaddr: taker.my_cj_addr = dummyaddr #make signatures for the last 3 fake utxos, considered as "not ours": - tx3 = bitcoin.sign(tx, 2, privs[2]) - sig3 = b64encode(binascii.unhexlify(bitcoin.deserialize(tx3)['ins'][2]['script'])) + sig, msg = bitcoin.sign(tx2, 2, privs[2]) + assert sig, "Failed to sign: " + msg + sig3 = b64encode(tx2.vin[2].scriptSig) taker.on_sig("cp1", sig3) #try sending the same sig again; should be ignored taker.on_sig("cp1", sig3) - tx4 = bitcoin.sign(tx, 3, privs[3]) - sig4 = b64encode(binascii.unhexlify(bitcoin.deserialize(tx4)['ins'][3]['script'])) + sig, msg = bitcoin.sign(tx2, 3, privs[3]) + assert sig, "Failed to sign: " + msg + sig4 = b64encode(tx2.vin[3].scriptSig) #try sending junk instead of cp2's correct sig - taker.on_sig("cp2", str("junk")) + assert not taker.on_sig("cp2", str("junk")), "incorrectly accepted junk signature" taker.on_sig("cp2", sig4) - tx5 = bitcoin.sign(tx, 4, privs[4]) + sig, msg = bitcoin.sign(tx2, 4, privs[4]) + assert sig, "Failed to sign: " + msg #Before completing with the final signature, which will trigger our own #signing, try with an injected failure of query utxo set, which should #prevent this signature being accepted. dbci.setQUSFail(True) - sig5 = b64encode(binascii.unhexlify(bitcoin.deserialize(tx5)['ins'][4]['script'])) - taker.on_sig("cp3", sig5) + sig5 = b64encode(tx2.vin[4].scriptSig) + assert not taker.on_sig("cp3", sig5), "incorrectly accepted sig5" #allow it to succeed, and try again dbci.setQUSFail(False) #this should succeed and trigger the we-sign code @@ -465,7 +481,7 @@ def test_on_sig(setup_taker, dummyaddr, schedule): [ ([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw")]), ]) -def test_auth_counterparty(schedule): +def test_auth_counterparty(setup_taker, schedule): taker = get_taker(schedule=schedule) first_maker_response = t_maker_response["J659UPUSLLjHJpaB"] utxo, auth_pub, cjaddr, changeaddr, sig, maker_pub = first_maker_response diff --git a/jmclient/test/test_tx_creation.py b/jmclient/test/test_tx_creation.py index d251b43..7c1e75c 100644 --- a/jmclient/test/test_tx_creation.py +++ b/jmclient/test/test_tx_creation.py @@ -1,6 +1,9 @@ #! /usr/bin/env python '''Test of unusual transaction types creation and push to -network to check validity.''' + network to check validity. + Note as of Feb 2020: earlier versions included multisig + p2(w)sh tests, these have been removed since Joinmarket + does not use this feature.''' import time import binascii @@ -24,62 +27,12 @@ vpubs = ["03e9a06e539d6bf5cf1ca5c41b59121fa3df07a338322405a312c67b6349a707e9", "028a2f126e3999ff66d01dcb101ab526d3aa1bf5cbdc4bde14950a4cead95f6fcb", "02bea84d70e74f7603746b62d79bf035e16d982b56e6a1ee07dfd3b9130e8a2ad9"] - -@pytest.mark.parametrize( - "nw, wallet_structures, mean_amt, sdev_amt, amount, pubs, k", [ - (1, [[2, 1, 4, 0, 0]], 4, 1.4, 600000000, vpubs[1:4], 2), - (1, [[3, 3, 0, 0, 3]], 4, 1.4, 100000000, vpubs[:4], 3), - ]) -def test_create_p2sh_output_tx(setup_tx_creation, nw, wallet_structures, - mean_amt, sdev_amt, amount, pubs, k): - wallets = make_wallets(nw, wallet_structures, mean_amt, sdev_amt) - for w in wallets.values(): - w['wallet'].sync_wallet(fast=True) - for k, w in enumerate(wallets.values()): - wallet_service = w['wallet'] - ins_full = wallet_service.select_utxos(0, amount) - script = bitcoin.mk_multisig_script(pubs, k) - output_addr = bitcoin.p2sh_scriptaddr(bitcoin.safe_from_hex(script), - magicbyte=196) - txid = make_sign_and_push(ins_full, - wallet_service, - amount, - output_addr=output_addr) - assert txid - -def test_script_to_address(setup_tx_creation): - sample_script = "a914307f099a3bfedec9a09682238db491bade1b467f87" - assert bitcoin.script_to_address( - sample_script, vbyte=5) == "367SYUMqo1Fi4tQsycnmCtB6Ces1Z7EZLH" - assert bitcoin.script_to_address( - sample_script, vbyte=196) == "2MwfecDHsQTm4Gg3RekQdpqAMR15BJrjfRF" - -def test_mktx(setup_tx_creation): - """Testing exceptional conditions; not guaranteed - to create valid tx objects""" - #outpoint structure must be {"outpoint":{"hash":hash, "index": num}} - ins = [{'outpoint': {"hash":x*64, "index":0}, - "script": "", "sequence": 4294967295} for x in ["a", "b", "c"]] - pub = vpubs[0] - addr = bitcoin.pubkey_to_address(pub, magicbyte=get_p2pk_vbyte()) - script = bitcoin.address_to_script(addr) - outs = [script + ":1000", addr+":2000",{"script":script, "value":3000}] - tx = bitcoin.mktx(ins, outs) - print(tx) - #rewrite with invalid output - outs.append({"foo": "bar"}) - with pytest.raises(Exception) as e_info: - tx = bitcoin.mktx(ins, outs) - -def test_bintxhash(setup_tx_creation): - tx = "abcdef1234" - x = bitcoin.bin_txhash(tx) - assert binascii.hexlify(x).decode('ascii') == "121480fc2cccd5103434a9c88b037e08ef6c4f9f95dfb85b56f7043a344613fe" - def test_all_same_priv(setup_tx_creation): #recipient - priv = "aa"*32 + "01" - addr = bitcoin.privkey_to_address(priv, magicbyte=get_p2pk_vbyte()) + priv = b"\xaa"*32 + b"\x01" + pub = bitcoin.privkey_to_pubkey(priv) + addr = str(bitcoin.CCoinAddress.from_scriptPubKey( + bitcoin.CScript([bitcoin.OP_0, bitcoin.Hash160(pub)]))) wallet_service = make_wallets(1, [[1,0,0,0,0]], 1)[0]['wallet'] #make another utxo on the same address addrinwallet = wallet_service.get_addr(0,0,0) @@ -89,52 +42,41 @@ def test_all_same_priv(setup_tx_creation): outs = [{"address": addr, "value": 1000000}] ins = list(insfull.keys()) tx = bitcoin.mktx(ins, outs) - tx = bitcoin.signall(tx, wallet_service.get_key_from_addr(addrinwallet)) - -@pytest.mark.parametrize( - "signall", - [ - (True,), - (False,), - ]) -def test_verify_tx_input(setup_tx_creation, signall): - priv = "aa"*32 + "01" - addr = bitcoin.privkey_to_address(priv, magicbyte=get_p2pk_vbyte()) + scripts = {} + for i, j in enumerate(ins): + scripts[i] = (insfull[j]["script"], insfull[j]["value"]) + success, msg = wallet_service.sign_tx(tx, scripts) + assert success, msg + +def test_verify_tx_input(setup_tx_creation): + priv = b"\xaa"*32 + b"\x01" + pub = bitcoin.privkey_to_pubkey(priv) + script = bitcoin.pubkey_to_p2sh_p2wpkh_script(pub) + addr = str(bitcoin.CCoinAddress.from_scriptPubKey(script)) wallet_service = make_wallets(1, [[2,0,0,0,0]], 1)[0]['wallet'] wallet_service.sync_wallet(fast=True) insfull = wallet_service.select_utxos(0, 110000000) - print(insfull) outs = [{"address": addr, "value": 1000000}] ins = list(insfull.keys()) tx = bitcoin.mktx(ins, outs) - desertx = bitcoin.deserialize(tx) - print(desertx) - if signall: - privdict = {} - for index, ins in enumerate(desertx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - ad = insfull[utxo]['address'] - priv = wallet_service.get_key_from_addr(ad) - privdict[utxo] = priv - tx = bitcoin.signall(tx, privdict) - else: - for index, ins in enumerate(desertx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - ad = insfull[utxo]['address'] - priv = wallet_service.get_key_from_addr(ad) - tx = bitcoin.sign(tx, index, priv) - desertx2 = bitcoin.deserialize(tx) - print(desertx2) - sig, pub = bitcoin.deserialize_script(desertx2['ins'][0]['script']) - print(sig, pub) - pubscript = bitcoin.address_to_script(bitcoin.pubkey_to_address( - pub, magicbyte=get_p2pk_vbyte())) - sig = binascii.unhexlify(sig) - pub = binascii.unhexlify(pub) - sig_good = bitcoin.verify_tx_input(tx, 0, pubscript, - sig, pub) - assert sig_good - + scripts = {0: (insfull[ins[0]]["script"], bitcoin.coins_to_satoshi(1))} + success, msg = wallet_service.sign_tx(tx, scripts) + assert success, msg + # testing Joinmarket's ability to verify transaction inputs + # of others: pretend we don't have a wallet owning the transaction, + # and instead verify an input using the (sig, pub, scriptCode) data + # that is sent by counterparties: + cScrWit = tx.wit.vtxinwit[0].scriptWitness + sig = cScrWit.stack[0] + pub = cScrWit.stack[1] + scriptSig = tx.vin[0].scriptSig + tx2 = bitcoin.mktx(ins, outs) + res = bitcoin.verify_tx_input(tx2, 0, scriptSig, + bitcoin.pubkey_to_p2sh_p2wpkh_script(pub), + amount = bitcoin.coins_to_satoshi(1), + witness = bitcoin.CScript([sig, pub])) + assert res + def test_absurd_fees(setup_tx_creation): """Test triggering of ValueError exception if the transaction fees calculated from the blockchain @@ -157,7 +99,6 @@ def test_create_sighash_txs(setup_tx_creation): wallet_service.sync_wallet(fast=True) amount = 350000000 ins_full = wallet_service.select_utxos(0, amount) - print("using hashcode: " + str(sighash)) txid = make_sign_and_push(ins_full, wallet_service, amount, hashcode=sighash) assert txid @@ -165,118 +106,39 @@ def test_create_sighash_txs(setup_tx_creation): with pytest.raises(Exception) as e_info: fake_utxos = wallet_service.select_utxos(4, 1000000000) - -def test_spend_p2sh_utxos(setup_tx_creation): - #make a multisig address from 3 privs - privs = [struct.pack(b'B', x) * 32 + b'\x01' for x in range(1, 4)] - pubs = [bitcoin.privkey_to_pubkey(binascii.hexlify(priv).decode('ascii')) for priv in privs] - script = bitcoin.mk_multisig_script(pubs, 2) - msig_addr = bitcoin.p2sh_scriptaddr(script, magicbyte=196) - #pay into it - wallet_service = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet'] - wallet_service.sync_wallet(fast=True) - amount = 350000000 - ins_full = wallet_service.select_utxos(0, amount) - txid = make_sign_and_push(ins_full, wallet_service, amount, output_addr=msig_addr) - assert txid - #wait for mining - time.sleep(1) - #spend out; the input can be constructed from the txid of previous - msig_in = txid + ":0" - ins = [msig_in] - #random output address and change addr - output_addr = wallet_service.get_internal_addr(1) - amount2 = amount - 50000 - outs = [{'value': amount2, 'address': output_addr}] - tx = bitcoin.mktx(ins, outs) - sigs = [] - for priv in privs[:2]: - sigs.append(bitcoin.get_p2sh_signature(tx, 0, script, binascii.hexlify(priv).decode('ascii'))) - tx = bitcoin.apply_multisignatures(tx, 0, script, sigs) - txid = jm_single().bc_interface.pushtx(tx) - assert txid - def test_spend_p2wpkh(setup_tx_creation): #make 3 p2wpkh outputs from 3 privs privs = [struct.pack(b'B', x) * 32 + b'\x01' for x in range(1, 4)] - pubs = [bitcoin.privkey_to_pubkey( - binascii.hexlify(priv).decode('ascii')) for priv in privs] + pubs = [bitcoin.privkey_to_pubkey(priv) for priv in privs] scriptPubKeys = [bitcoin.pubkey_to_p2wpkh_script(pub) for pub in pubs] - addresses = [bitcoin.pubkey_to_p2wpkh_address(pub) for pub in pubs] + addresses = [str(bitcoin.CCoinAddress.from_scriptPubKey( + spk)) for spk in scriptPubKeys] #pay into it wallet_service = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet'] wallet_service.sync_wallet(fast=True) amount = 35000000 p2wpkh_ins = [] - for addr in addresses: + for i, addr in enumerate(addresses): ins_full = wallet_service.select_utxos(0, amount) txid = make_sign_and_push(ins_full, wallet_service, amount, output_addr=addr) assert txid - p2wpkh_ins.append(txid + ":0") + p2wpkh_ins.append((txid, 0)) + txhex = jm_single().bc_interface.get_transaction(txid) #wait for mining - time.sleep(1) + jm_single().bc_interface.tick_forward_chain(1) #random output address output_addr = wallet_service.get_internal_addr(1) amount2 = amount*3 - 50000 outs = [{'value': amount2, 'address': output_addr}] tx = bitcoin.mktx(p2wpkh_ins, outs) - sigs = [] - for i, priv in enumerate(privs): - # sign each of 3 inputs - tx = bitcoin.p2wpkh_sign(tx, i, binascii.hexlify(priv), - amount, native=True) - # check that verify_tx_input correctly validates; - # to do this, we need to extract the signature and get the scriptCode - # of this pubkey - scriptCode = bitcoin.pubkey_to_p2pkh_script(pubs[i]) - witness = bitcoin.deserialize(tx)['ins'][i]['txinwitness'] - assert len(witness) == 2 - assert witness[1] == pubs[i] - sig = witness[0] - assert bitcoin.verify_tx_input(tx, i, scriptPubKeys[i], sig, - pubs[i], scriptCode=scriptCode, amount=amount) - txid = jm_single().bc_interface.pushtx(tx) - assert txid - -def test_spend_p2wsh(setup_tx_creation): - #make 2 x 2 of 2multisig outputs; will need 4 privs - privs = [struct.pack(b'B', x) * 32 + b'\x01' for x in range(1, 5)] - privs = [binascii.hexlify(priv).decode('ascii') for priv in privs] - pubs = [bitcoin.privkey_to_pubkey(priv) for priv in privs] - redeemScripts = [bitcoin.mk_multisig_script(pubs[i:i+2], 2) for i in [0, 2]] - scriptPubKeys = [bitcoin.pubkeys_to_p2wsh_multisig_script(pubs[i:i+2]) for i in [0, 2]] - addresses = [bitcoin.pubkeys_to_p2wsh_multisig_address(pubs[i:i+2]) for i in [0, 2]] - #pay into it - wallet_service = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet'] - wallet_service.sync_wallet(fast=True) - amount = 35000000 - p2wsh_ins = [] - for addr in addresses: - ins_full = wallet_service.select_utxos(0, amount) - txid = make_sign_and_push(ins_full, wallet_service, amount, output_addr=addr) - assert txid - p2wsh_ins.append(txid + ":0") - #wait for mining - time.sleep(1) - #random output address and change addr - output_addr = wallet_service.get_internal_addr(1) - amount2 = amount*2 - 50000 - outs = [{'value': amount2, 'address': output_addr}] - tx = bitcoin.mktx(p2wsh_ins, outs) - sigs = [] - for i in range(2): - sigs = [] - for priv in privs[i*2:i*2+2]: - # sign input j with each of 2 keys - sig = bitcoin.get_p2sh_signature(tx, i, redeemScripts[i], priv, amount=amount) - sigs.append(sig) - # check that verify_tx_input correctly validates; - assert bitcoin.verify_tx_input(tx, i, scriptPubKeys[i], sig, - bitcoin.privkey_to_pubkey(priv), - scriptCode=redeemScripts[i], amount=amount) - tx = bitcoin.apply_p2wsh_multisignatures(tx, i, redeemScripts[i], sigs) - txid = jm_single().bc_interface.pushtx(tx) + for i, priv in enumerate(privs): + # sign each of 3 inputs; note that bitcoin.sign + # automatically validates each signature it creates. + sig, msg = bitcoin.sign(tx, i, priv, amount=amount, native=True) + if not sig: + assert False, msg + txid = jm_single().bc_interface.pushtx(tx.serialize()) assert txid def test_spend_freeze_script(setup_tx_creation): @@ -320,7 +182,6 @@ def test_spend_freeze_script(setup_tx_creation): push_success = jm_single().bc_interface.pushtx(tx) assert push_success == required_success - @pytest.fixture(scope="module") def setup_tx_creation(): load_test_config() diff --git a/jmclient/test/test_valid_addresses.py b/jmclient/test/test_valid_addresses.py index a3d8238..fcacc64 100644 --- a/jmclient/test/test_valid_addresses.py +++ b/jmclient/test/test_valid_addresses.py @@ -1,10 +1,18 @@ from jmclient.configure import validate_address, load_test_config from jmclient import jm_single +import jmbitcoin as btc import json import pytest import os testdir = os.path.dirname(os.path.realpath(__file__)) +def address_valid_somewhere(addr): + for x in ["bitcoin", "bitcoin/testnet", "bitcoin/regtest"]: + btc.select_chain_params(x) + if validate_address(addr)[0]: + return True + return False + def test_non_addresses(setup_addresses): #could flesh this out with other examples res, msg = validate_address(2) @@ -17,9 +25,8 @@ def test_b58_invalid_addresses(setup_addresses): invalid_key_list = json.loads(json_data) for k in invalid_key_list: bad_key = k[0] - res, message = validate_address(bad_key) - assert res == False, "Incorrectly validated address: " + bad_key + " with message: " + message - + res = address_valid_somewhere(bad_key) + assert res == False, "Incorrectly validated address: " + bad_key def test_b58_valid_addresses(): with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f: @@ -30,30 +37,36 @@ def test_b58_valid_addresses(): if not prop_dict["isPrivkey"]: if prop_dict["isTestnet"]: jm_single().config.set("BLOCKCHAIN", "network", "testnet") + btc.select_chain_params("bitcoin/testnet") else: jm_single().config.set("BLOCKCHAIN", "network", "mainnet") + btc.select_chain_params("bitcoin") #if using pytest -s ; sanity check to see what's actually being tested - print('testing this address: ', addr) res, message = validate_address(addr) assert res == True, "Incorrectly failed to validate address: " + addr + " with message: " + message jm_single().config.set("BLOCKCHAIN", "network", "testnet") + btc.select_chain_params("bitcoin/regtest") def test_valid_bech32_addresses(): valids = ["BC1QW508D6QEJXTDG4Y5R3ZARVARY0C5XW7KV8F3T4", "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sl5k7", - "bc1pw508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7k7grplx", - "BC1SW50QA3JX3S", - "bc1zw508d6qejxtdg4y5r3zarvaryvg6kdaj", + # TODO these are valid bech32 addresses but rejected by bitcointx + # because they are not witness version 0; add others. + #"bc1pw508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7k7grplx", + #"BC1SW50QA3JX3S", + #"bc1zw508d6qejxtdg4y5r3zarvaryvg6kdaj", "tb1qqqqqp399et2xygdj5xreqhjjvcmzhxw4aywxecjdzew6hylgvsesrxh6hy"] for va in valids: - print("Testing this address: ", va) if va.lower()[:2] == "bc": jm_single().config.set("BLOCKCHAIN", "network", "mainnet") + btc.select_chain_params("bitcoin") else: jm_single().config.set("BLOCKCHAIN", "network", "testnet") + btc.select_chain_params("bitcoin/testnet") res, message = validate_address(va) assert res == True, "Incorrect failed to validate address: " + va + " with message: " + message jm_single().config.set("BLOCKCHAIN", "network", "testnet") + btc.select_chain_params("bitcoin/regtest") def test_invalid_bech32_addresses(): invalids = [ @@ -68,8 +81,7 @@ def test_invalid_bech32_addresses(): "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3pjxtptv", "bc1gmk9yu"] for iva in invalids: - print("Testing this address: ", iva) - res, message = validate_address(iva) + res = address_valid_somewhere(iva) assert res == False, "Incorrectly validated address: " + iva @pytest.fixture(scope="module") diff --git a/jmclient/test/test_wallet.py b/jmclient/test/test_wallet.py index 0926947..e0e0e4a 100644 --- a/jmclient/test/test_wallet.py +++ b/jmclient/test/test_wallet.py @@ -6,12 +6,14 @@ from binascii import hexlify, unhexlify import pytest import jmbitcoin as btc -from commontest import binarize_tx, ensure_bip65_activated -from jmbase import get_log +from commontest import ensure_bip65_activated +from jmbase import (get_log, utxostr_to_utxo, utxo_to_utxostr, + hextobin, bintohex) from jmclient import load_test_config, jm_single, \ SegwitLegacyWallet,BIP32Wallet, BIP49Wallet, LegacyWallet,\ VolatileStorage, get_network, cryptoengine, WalletError,\ SegwitWallet, WalletService, SegwitLegacyWalletFidelityBonds,\ + BTC_P2PKH, BTC_P2SH_P2WPKH,\ FidelityBondMixin, FidelityBondWatchonlyWallet, wallet_gettimelockaddress from test_blockchaininterface import sync_test_wallet @@ -20,10 +22,7 @@ log = get_log() def signed_tx_is_segwit(tx): - for inp in tx['ins']: - if 'txinwitness' not in inp: - return False - return True + return tx.has_witness() def assert_segwit(tx): @@ -47,10 +46,11 @@ def get_populated_wallet(amount=10**8, num=3): def fund_wallet_addr(wallet, addr, value_btc=1): - txin_id = jm_single().bc_interface.grab_coins(addr, value_btc) + # special case, grab_coins returns hex from rpc: + txin_id = hextobin(jm_single().bc_interface.grab_coins(addr, value_btc)) txinfo = jm_single().bc_interface.get_transaction(txin_id) - txin = btc.deserialize(unhexlify(txinfo['hex'])) - utxo_in = wallet.add_new_utxos_(txin, unhexlify(txin_id), 1) + txin = btc.CMutableTransaction.deserialize(btc.x(txinfo["hex"])) + utxo_in = wallet.add_new_utxos(txin, 1) assert len(utxo_in) == 1 return list(utxo_in.keys())[0] @@ -367,13 +367,15 @@ def test_signing_imported(setup_wallet, wif, keytype, type_check): path = wallet.import_private_key(MIXDEPTH, wif, keytype) utxo = fund_wallet_addr(wallet, wallet.get_address_from_path(path)) # The dummy output is constructed as an unspendable p2sh: - tx = btc.deserialize(btc.mktx(['{}:{}'.format( - hexlify(utxo[0]).decode('ascii'), utxo[1])], - [btc.p2sh_scriptaddr(b"\x00",magicbyte=196) + ':' + str(10**8 - 9000)])) + tx = btc.mktx([utxo], + [{"address": str(btc.CCoinAddress.from_scriptPubKey( + btc.CScript(b"\x00").to_p2sh_scriptPubKey())), + "value": 10**8 - 9000}]) script = wallet.get_script_from_path(path) - tx = wallet.sign_tx(tx, {0: (script, 10**8)}) + success, msg = wallet.sign_tx(tx, {0: (script, 10**8)}) + assert success, msg type_check(tx) - txout = jm_single().bc_interface.pushtx(btc.serialize(tx)) + txout = jm_single().bc_interface.pushtx(tx.serialize()) assert txout @@ -385,17 +387,19 @@ def test_signing_imported(setup_wallet, wif, keytype, type_check): def test_signing_simple(setup_wallet, wallet_cls, type_check): jm_single().config.set('BLOCKCHAIN', 'network', 'testnet') storage = VolatileStorage() - wallet_cls.initialize(storage, get_network()) + wallet_cls.initialize(storage, get_network(), entropy=b"\xaa"*16) wallet = wallet_cls(storage) utxo = fund_wallet_addr(wallet, wallet.get_internal_addr(0)) # The dummy output is constructed as an unspendable p2sh: - tx = btc.deserialize(btc.mktx(['{}:{}'.format( - hexlify(utxo[0]).decode('ascii'), utxo[1])], - [btc.p2sh_scriptaddr(b"\x00",magicbyte=196) + ':' + str(10**8 - 9000)])) + tx = btc.mktx([utxo], + [{"address": str(btc.CCoinAddress.from_scriptPubKey( + btc.CScript(b"\x00").to_p2sh_scriptPubKey())), + "value": 10**8 - 9000}]) script = wallet.get_script(0, 1, 0) - tx = wallet.sign_tx(tx, {0: (script, 10**8)}) + success, msg = wallet.sign_tx(tx, {0: (script, 10**8)}) + assert success, msg type_check(tx) - txout = jm_single().bc_interface.pushtx(btc.serialize(tx)) + txout = jm_single().bc_interface.pushtx(tx.serialize()) assert txout def test_timelocked_output_signing(setup_wallet): @@ -450,18 +454,18 @@ def test_add_utxos(setup_wallet): for md in range(1, wallet.max_mixdepth + 1): assert balances[md] == 0 - utxos = wallet.get_utxos_by_mixdepth_() + utxos = wallet.get_utxos_by_mixdepth() assert len(utxos[0]) == num_tx for md in range(1, wallet.max_mixdepth + 1): assert not utxos[md] with pytest.raises(Exception): # no funds in mixdepth - wallet.select_utxos_(1, amount) + wallet.select_utxos(1, amount) with pytest.raises(Exception): # not enough funds - wallet.select_utxos_(0, amount * (num_tx + 1)) + wallet.select_utxos(0, amount * (num_tx + 1)) wallet.reset_utxos() assert wallet.get_balance_by_mixdepth()[0] == 0 @@ -472,12 +476,12 @@ def test_select_utxos(setup_wallet): amount = 10**8 wallet = get_populated_wallet(amount) - utxos = wallet.select_utxos_(0, amount // 2) + utxos = wallet.select_utxos(0, amount // 2) assert len(utxos) == 1 utxos = list(utxos.keys()) - more_utxos = wallet.select_utxos_(0, int(amount * 1.5), utxo_filter=utxos) + more_utxos = wallet.select_utxos(0, int(amount * 1.5), utxo_filter=utxos) assert len(more_utxos) == 2 assert utxos[0] not in more_utxos @@ -488,14 +492,11 @@ def test_add_new_utxos(setup_wallet): scripts = [wallet.get_new_script(x, True) for x in range(3)] tx_scripts = list(scripts) - tx_scripts.append(b'\x22'*17) - - tx = btc.deserialize(btc.mktx( - ['0'*64 + ':2'], [{'script': hexlify(s).decode('ascii'), 'value': 10**8} - for s in tx_scripts])) - binarize_tx(tx) - txid = b'\x01' * 32 - added = wallet.add_new_utxos_(tx, txid, 1) + tx = btc.mktx( + [(b"\x00"*32, 2)], + [{"address": wallet.script_to_addr(s), + "value": 10**8} for s in tx_scripts]) + added = wallet.add_new_utxos(tx, 1) assert len(added) == len(scripts) added_scripts = {x['script'] for x in added.values()} @@ -517,21 +518,20 @@ def test_remove_old_utxos(setup_wallet): for i in range(3): txin = jm_single().bc_interface.grab_coins( wallet.get_internal_addr(1), 1) - wallet.add_utxo(unhexlify(txin), 0, wallet.get_script(1, 1, i), 10**8, 1) + wallet.add_utxo(btc.x(txin), 0, wallet.get_script(1, 1, i), 10**8, 1) - inputs = wallet.select_utxos_(0, 10**8) - inputs.update(wallet.select_utxos_(1, 2 * 10**8)) + inputs = wallet.select_utxos(0, 10**8) + inputs.update(wallet.select_utxos(1, 2 * 10**8)) assert len(inputs) == 3 tx_inputs = list(inputs.keys()) tx_inputs.append((b'\x12'*32, 6)) - tx = btc.deserialize(btc.mktx( - ['{}:{}'.format(hexlify(txid).decode('ascii'), i) for txid, i in tx_inputs], - ['0' * 36 + ':' + str(3 * 10**8 - 1000)])) - binarize_tx(tx) + tx = btc.mktx(tx_inputs, + [{"address": "2N9gfkUsFW7Kkb1Eurue7NzUxUt7aNJiS1U", + "value": 3 * 10**8 - 1000}]) - removed = wallet.remove_old_utxos_(tx) + removed = wallet.remove_old_utxos(tx) assert len(removed) == len(inputs) for txid in removed: @@ -760,6 +760,14 @@ def test_wallet_mixdepth_decrease(setup_wallet): max_mixdepth = wallet.max_mixdepth assert max_mixdepth >= 1, "bad default value for mixdepth for this test" utxo = fund_wallet_addr(wallet, wallet.get_internal_addr(max_mixdepth), 1) + bci = jm_single().bc_interface + unspent_list = bci.rpc('listunspent', [0]) + # filter on label, but note (a) in certain circumstances (in- + # wallet transfer) it is possible for the utxo to be labeled + # with the external label, and (b) the wallet will know if it + # belongs or not anyway (is_known_addr): + our_unspent_list = [x for x in unspent_list if ( + bci.is_address_labeled(x, wallet.get_wallet_name()))] assert wallet.get_balance_by_mixdepth()[max_mixdepth] == 10**8 wallet.close() storage_data = wallet._storage.file_data @@ -777,7 +785,7 @@ def test_wallet_mixdepth_decrease(setup_wallet): # wallet.select_utxos will still return utxos from higher mixdepths # because we explicitly ask for a specific mixdepth - assert utxo in new_wallet.select_utxos_(max_mixdepth, 10**7) + assert utxo in new_wallet.select_utxos(max_mixdepth, 10**7) def test_watchonly_wallet(setup_wallet): jm_single().config.set('BLOCKCHAIN', 'network', 'testnet') @@ -819,6 +827,7 @@ def test_watchonly_wallet(setup_wallet): @pytest.fixture(scope='module') def setup_wallet(): load_test_config() + btc.select_chain_params("bitcoin/regtest") #see note in cryptoengine.py: cryptoengine.BTC_P2WPKH.VBYTE = 100 jm_single().bc_interface.tick_forward_chain_interval = 2 diff --git a/jmclient/test/test_wallets.py b/jmclient/test/test_wallets.py index 66d6058..7344ead 100644 --- a/jmclient/test/test_wallets.py +++ b/jmclient/test/test_wallets.py @@ -8,7 +8,8 @@ from commontest import create_wallet_for_sync, make_sign_and_push import json import pytest -from jmbase import get_log +from jmbase import get_log, hextobin +import jmbitcoin as btc from jmclient import ( load_test_config, jm_single, estimate_tx_fee, BitcoinCoreInterface, Mnemonic) @@ -43,9 +44,10 @@ def test_query_utxo_set(setup_wallets): txid = do_tx(wallet_service, 90000000) txid2 = do_tx(wallet_service, 20000000) print("Got txs: ", txid, txid2) - res1 = jm_single().bc_interface.query_utxo_set(txid + ":0", includeunconf=True) + res1 = jm_single().bc_interface.query_utxo_set( + (txid, 0), includeunconf=True) res2 = jm_single().bc_interface.query_utxo_set( - [txid + ":0", txid2 + ":1"], + [(txid, 0), (txid2, 1)], includeconf=True, includeunconf=True) assert len(res1) == 1 assert len(res2) == 2 @@ -53,7 +55,7 @@ def test_query_utxo_set(setup_wallets): assert not 'confirms' in res1[0] assert 'confirms' in res2[0] assert 'confirms' in res2[1] - res3 = jm_single().bc_interface.query_utxo_set("ee" * 32 + ":25") + res3 = jm_single().bc_interface.query_utxo_set((b"\xee" * 32, 25)) assert res3 == [None] @@ -69,11 +71,11 @@ def test_wrong_network_bci(setup_wallets): def test_pushtx_errors(setup_wallets): """Ensure pushtx fails return False """ - badtxhex = "aaaa" - assert not jm_single().bc_interface.pushtx(badtxhex) + badtx = b"\xaa\xaa" + assert not jm_single().bc_interface.pushtx(badtx) #Break the authenticated jsonrpc and try again jm_single().bc_interface.jsonRpc.port = 18333 - assert not jm_single().bc_interface.pushtx(t_raw_signed_tx) + assert not jm_single().bc_interface.pushtx(hextobin(t_raw_signed_tx)) #rebuild a valid jsonrpc inside the bci load_test_config() diff --git a/jmclient/test/test_walletservice.py b/jmclient/test/test_walletservice.py index a652a40..f46238b 100644 --- a/jmclient/test/test_walletservice.py +++ b/jmclient/test/test_walletservice.py @@ -39,7 +39,6 @@ def test_address_reuse_freezing(setup_walletservice): """ context = {'cb_called': 0} def reuse_callback(utxostr): - print("Address reuse freezing callback on utxo: ", utxostr) context['cb_called'] += 1 # we must fund after initial sync (for imports), hence # "populated" with no coins diff --git a/jmclient/test/test_yieldgenerator.py b/jmclient/test/test_yieldgenerator.py index 2fd2539..332c50b 100644 --- a/jmclient/test/test_yieldgenerator.py +++ b/jmclient/test/test_yieldgenerator.py @@ -1,6 +1,6 @@ import unittest - +from jmbitcoin import CMutableTxOut, CMutableTransaction from jmclient import load_test_config, jm_single,\ SegwitLegacyWallet, VolatileStorage, YieldGeneratorBasic, \ get_network, WalletService @@ -28,14 +28,12 @@ class CustomUtxoWallet(SegwitLegacyWallet): self.add_utxo_at_mixdepth(m, b) def add_utxo_at_mixdepth(self, mixdepth, balance): - tx = {'outs': [{'script': self.get_internal_script(mixdepth), - 'value': balance}]} - # We need to generate a fake "txid" that has to be unique for all the - # UTXOs that are added to the wallet. For that, we simply use the - # script, and make it fit the required length (32 bytes). - txid = tx['outs'][0]['script'] + b'x' * 32 - txid = txid[:32] - self.add_new_utxos_(tx, txid, 1) + txout = CMutableTxOut(balance, self.get_internal_script(mixdepth)) + tx = CMutableTransaction() + tx.vout = [txout] + # (note: earlier requirement that txid be generated uniquely is now + # automatic; tx.GetTxid() functions correctly within the wallet). + self.add_new_utxos(tx, 1) def assert_utxos_from_mixdepth(self, utxos, expected): """Asserts that the list of UTXOs (as returned from UTXO selection diff --git a/jmdaemon/test/test_message_channel.py b/jmdaemon/test/test_message_channel.py index 949e36e..9ca3891 100644 --- a/jmdaemon/test/test_message_channel.py +++ b/jmdaemon/test/test_message_channel.py @@ -15,6 +15,7 @@ import base64 import struct import traceback import threading +import binascii import jmbitcoin as bitcoin from dummy_mc import DummyMessageChannel @@ -22,10 +23,11 @@ from dummy_mc import DummyMessageChannel jlog = get_log() def make_valid_nick(i=0): - nick_priv = hashlib.sha256(struct.pack(b'B', i)*16).hexdigest() + '01' - nick_pubkey = bitcoin.privtopub(nick_priv) - nick_pkh_raw = hashlib.sha256(nick_pubkey.encode('ascii')).digest()[:NICK_HASH_LENGTH] - nick_pkh = bitcoin.b58encode(nick_pkh_raw) + nick_priv = hashlib.sha256(struct.pack(b'B', i)*16).digest() + b"\x01" + nick_pubkey = bitcoin.privkey_to_pubkey(nick_priv) + nick_pkh_raw = hashlib.sha256(binascii.hexlify( + nick_pubkey)).digest()[:NICK_HASH_LENGTH] + nick_pkh = bitcoin.base58.encode(nick_pkh_raw) #right pad to maximum possible; b58 is not fixed length. #Use 'O' as one of the 4 not included chars in base58. nick_pkh += 'O' * (NICK_MAX_ENCODED - len(nick_pkh)) diff --git a/scripts/add-utxo.py b/scripts/add-utxo.py index e564df6..ea6de44 100755 --- a/scripts/add-utxo.py +++ b/scripts/add-utxo.py @@ -17,7 +17,7 @@ import jmbitcoin as btc from jmclient import load_program_config, jm_single, get_p2pk_vbyte,\ open_wallet, WalletService, add_external_commitments, update_commitments,\ PoDLE, get_podle_commitments, get_utxo_info, validate_utxo_data, quit,\ - get_wallet_path, add_base_options + get_wallet_path, add_base_options, BTCEngine, BTC_P2SH_P2WPKH from jmbase.support import EXIT_SUCCESS, EXIT_FAILURE, EXIT_ARGERROR, jmprint @@ -32,9 +32,10 @@ def add_ext_commitments(utxo_datas): This calls the underlying 'raw' code based on the class PoDLE, not the library 'generate_podle' which intelligently searches and updates commitments. """ - #Convert priv to hex - hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte()) - podle = PoDLE(u, hexpriv) + #Convert priv from wif; require P2SH-P2WPKH keys + rawpriv, keytype = BTCEngine.wif_to_privkey(priv) + assert keytype == BTC_P2SH_P2WPKH + podle = PoDLE(u, rawpriv) r = podle.generate_podle(i) return (r['P'], r['P2'], r['sig'], r['e'], r['commit']) diff --git a/scripts/convert_old_wallet.py b/scripts/convert_old_wallet.py index 4a23cc9..1896547 100755 --- a/scripts/convert_old_wallet.py +++ b/scripts/convert_old_wallet.py @@ -7,10 +7,9 @@ from binascii import hexlify, unhexlify from collections import defaultdict from pyaes import AESModeOfOperationCBC, Decrypter from jmbase import JM_APP_NAME -from jmclient import Storage, load_program_config +from jmclient import Storage, load_program_config, BTCEngine from jmclient.wallet_utils import get_password, get_wallet_cls,\ cli_get_wallet_passphrase_check, get_wallet_path -from jmbitcoin import wif_compressed_privkey class ConvertException(Exception): @@ -99,7 +98,7 @@ def new_wallet_from_data(data, file_name): for md in data['imported']: for privkey in data['imported'][md]: privkey += b'\x01' - wif = wif_compressed_privkey(hexlify(privkey).decode('ascii')) + wif = BTCEngine.privkey_to_wif(privkey) wallet.import_private_key(md, wif) wallet.save() diff --git a/scripts/joinmarket-qt.py b/scripts/joinmarket-qt.py index d10bdc8..12eee3a 100755 --- a/scripts/joinmarket-qt.py +++ b/scripts/joinmarket-qt.py @@ -75,7 +75,7 @@ from jmclient import load_program_config, get_network, update_persist_config,\ wallet_generate_recover_bip39, wallet_display, get_utxos_enabled_disabled,\ NO_ROUNDING, get_max_cj_fee_values, get_default_max_absolute_fee, \ get_default_max_relative_fee, RetryableStorageError, add_base_options, \ - FidelityBondMixin + BTCEngine, BTC_P2SH_P2WPKH, FidelityBondMixin from qtsupport import ScheduleWizard, TumbleRestartWizard, config_tips,\ config_types, QtHandler, XStream, Buttons, OkButton, CancelButton,\ PasswordDialog, MyTreeWidget, JMQtMessageBox, BLUE_FG,\ @@ -1530,14 +1530,18 @@ class JMMainWindow(QMainWindow): done = False def privkeys_thread(): + # To explain this (given setting was already done in + # load_program_config), see: + # https://github.com/Simplexum/python-bitcointx/blob/9f1fa67a5445f8c187ef31015a4008bc5a048eea/bitcointx/__init__.py#L242-L243 + # note, we ignore the return value as we only want to apply + # the chainparams setting logic: + get_blockchain_interface_instance(jm_single().config) for addr in addresses: time.sleep(0.1) if done: break priv = self.wallet_service.get_key_from_addr(addr) - private_keys[addr] = btc.wif_compressed_privkey( - priv, - vbyte=get_p2pk_vbyte()) + private_keys[addr] = BTCEngine.privkey_to_wif(priv) self.computing_privkeys_signal.emit() self.show_privkeys_signal.emit() @@ -1569,10 +1573,13 @@ class JMMainWindow(QMainWindow): privkeys_fn + '.json'), "wb") as f: for addr, pk in private_keys.items(): #sanity check - if not addr == btc.pubkey_to_p2sh_p2wpkh_address( - btc.privkey_to_pubkey( - btc.from_wif_privkey(pk, vbyte=get_p2pk_vbyte()) - ), get_p2sh_vbyte()): + rawpriv, keytype = BTCEngine.wif_to_privkey(pk) + if not keytype == BTC_P2SH_P2WPKH: + JMQtMessageBox(None, "Failed to create privkey export, " + "should be keytype p2sh-p2wpkh but is not.", + mbtype='crit') + return + if not addr == self.wallet_service._ENGINE.privkey_to_address(rawpriv): JMQtMessageBox(None, "Failed to create privkey export -" +\ " critical error in key parsing.", mbtype='crit') @@ -2009,7 +2016,7 @@ if isinstance(jm_single().bc_interface, RegtestBitcoinCoreInterface): jm_single().bc_interface.simulating = True jm_single().maker_timeout_sec = 15 #trigger start with a fake tx - jm_single().bc_interface.pushtx("00"*20) + jm_single().bc_interface.pushtx(b"\x00"*20) #prepare for logging for dname in ['logs', 'wallets', 'cmtdata']: diff --git a/scripts/sendtomany.py b/scripts/sendtomany.py index 23941db..33118a0 100755 --- a/scripts/sendtomany.py +++ b/scripts/sendtomany.py @@ -8,10 +8,10 @@ for other reasons). from pprint import pformat from optparse import OptionParser import jmbitcoin as btc -from jmbase import get_log, jmprint +from jmbase import get_log, jmprint, bintohex, utxostr_to_utxo from jmclient import load_program_config, estimate_tx_fee, jm_single,\ get_p2pk_vbyte, validate_address, get_utxo_info, add_base_options,\ - validate_utxo_data, quit + validate_utxo_data, quit, BTCEngine, BTC_P2SH_P2WPKH, BTC_P2PKH log = get_log() @@ -39,10 +39,12 @@ def sign(utxo, priv, destaddrs, segwit=True): log.info("Using fee: " + str(fee)) for i, addr in enumerate(destaddrs): outs.append({'address': addr, 'value': share}) - unsigned_tx = btc.mktx(ins, outs) + tx = btc.mktx(ins, outs) amtforsign = amt if segwit else None - return btc.sign(unsigned_tx, 0, btc.from_wif_privkey( - priv, vbyte=get_p2pk_vbyte()), amount=amtforsign) + rawpriv, _ = BTCEngine.wif_to_privkey(priv) + success, msg = btc.sign(tx, 0, rawpriv, amount=amtforsign) + assert success, msg + return tx def main(): parser = OptionParser( @@ -106,17 +108,20 @@ def main(): for d in destaddrs: if not validate_address(d): quit(parser, "Address was not valid; wrong network?: " + d) - txsigned = sign(u, priv, destaddrs, segwit = not options.nonsegwit) + success, utxo = utxostr_to_utxo(u) + if not success: + quit(parser, "Failed to load utxo from string: " + utxo) + txsigned = sign(utxo, priv, destaddrs, segwit = not options.nonsegwit) if not txsigned: log.info("Transaction signing operation failed, see debug messages for details.") return - log.debug("Got signed transaction:\n" + txsigned) + log.info("Got signed transaction:\n" + bintohex(txsigned.serialize())) log.debug("Deserialized:") - log.debug(pformat(btc.deserialize(txsigned))) + log.debug(pformat(str(txsigned))) if input('Would you like to push to the network? (y/n):')[0] != 'y': log.info("You chose not to broadcast the transaction, quitting.") return - jm_single().bc_interface.pushtx(txsigned) + jm_single().bc_interface.pushtx(txsigned.serialize()) if __name__ == "__main__": main() diff --git a/test/test_segwit.py b/test/test_segwit.py index a222e5f..b5cc7a4 100644 --- a/test/test_segwit.py +++ b/test/test_segwit.py @@ -7,7 +7,7 @@ from common import make_wallets from pprint import pformat import jmbitcoin as btc import pytest -from jmbase import get_log +from jmbase import get_log, hextobin, bintohex from jmclient import load_test_config, jm_single, LegacyWallet log = get_log() @@ -19,9 +19,9 @@ def test_segwit_valid_txs(setup_segwit): for j in valid_txs: if len(j) < 2: continue - deserialized_tx = btc.deserialize(str(j[1])) + deserialized_tx = btc.CMutableTransaction.deserialize(hextobin(j[1])) print(pformat(deserialized_tx)) - assert btc.serialize(deserialized_tx) == str(j[1]) + assert deserialized_tx.serialize() == hextobin(j[1]) #TODO use bcinterface to decoderawtransaction #and compare the json values @@ -63,8 +63,8 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, sw_wallet_service = make_wallets(1, [[len(segwit_ins), 0, 0, 0, 0]], segwit_amt)[0]['wallet'] sw_wallet_service.sync_wallet(fast=True) - nsw_utxos = nsw_wallet_service.get_utxos_by_mixdepth(hexfmt=False)[MIXDEPTH] - sw_utxos = sw_wallet_service.get_utxos_by_mixdepth(hexfmt=False)[MIXDEPTH] + nsw_utxos = nsw_wallet_service.get_utxos_by_mixdepth()[MIXDEPTH] + sw_utxos = sw_wallet_service.get_utxos_by_mixdepth()[MIXDEPTH] assert len(o_ins) <= len(nsw_utxos), "sync failed" assert len(segwit_ins) <= len(sw_utxos), "sync failed" @@ -92,8 +92,8 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, # FIXME: encoding mess, mktx should accept binary input formats tx_ins = [] - for i, (txid, data) in sorted(all_ins.items(), key=lambda x: x[0]): - tx_ins.append('{}:{}'.format(binascii.hexlify(txid[0]).decode('ascii'), txid[1])) + for i, (txin, data) in sorted(all_ins.items(), key=lambda x: x[0]): + tx_ins.append(txin) # create outputs FEE = 50000 @@ -104,11 +104,11 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, change_amt = total_amt_in_sat - amount - FEE tx_outs = [ - {'script': binascii.hexlify(cj_script).decode('ascii'), + {'address': nsw_wallet_service.script_to_addr(cj_script), 'value': amount}, - {'script': binascii.hexlify(change_script).decode('ascii'), + {'address': nsw_wallet_service.script_to_addr(change_script), 'value': change_amt}] - tx = btc.deserialize(btc.mktx(tx_ins, tx_outs)) + tx = btc.mktx(tx_ins, tx_outs) # import new addresses to bitcoind jm_single().bc_interface.import_addresses( @@ -120,18 +120,20 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, for nsw_in_index in o_ins: inp = nsw_ins[nsw_in_index][1] scripts[nsw_in_index] = (inp['script'], inp['value']) - tx = nsw_wallet_service.sign_tx(tx, scripts) + success, msg = nsw_wallet_service.sign_tx(tx, scripts) + assert success, msg scripts = {} for sw_in_index in segwit_ins: inp = sw_ins[sw_in_index][1] scripts[sw_in_index] = (inp['script'], inp['value']) - tx = sw_wallet_service.sign_tx(tx, scripts) + success, msg = sw_wallet_service.sign_tx(tx, scripts) + assert success, msg print(tx) # push and verify - txid = jm_single().bc_interface.pushtx(btc.serialize(tx)) + txid = jm_single().bc_interface.pushtx(tx.serialize()) assert txid balances = jm_single().bc_interface.get_received_by_addr(