From 070c5bf9b9ddc19d34e8e824c0e7903b73b06a06 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Mon, 17 Feb 2020 14:40:31 +0000 Subject: [PATCH] python-bitcointx backend for jmbitcoin. Replaces core transaction, address, serialization and sign functionality for Bitcoin with python-bitcointx backend. Removes bech32 and btscript modules from jmbitcoin. Removes all string, hex, binary conversion routines. A generic hex/binary conversion now is added to jmbase. Removes all transaction serialization and deserialization routines. Removes the now irrelevant test modules. Remaining functions in jmbitcoin remove any parsing of hex format, requiring callers to use binary only. One additional test added, testing the remaining function in secp256k1_transaction.py: the signing of transactions. Deserialized form is now bitcointx.CMutableTransaction. For jmbase, in addition to the above, generic conversions for utxos to and from strings is added, and a dynamic conversion for AMP messages to binary-only. Within the code, utxos are now only in (binarytxid, int) form, except where converted for communcation. Tthe largest part of the changes are the modifications to jmbitcoin calls in jmclient; as well as different encapsulation with CMutableTransaction, there is also a removal of some but not all hex parsing; it remains for rpc calls to Core and for AMP message parsing. Backwards compatibility must be ensured so some joinmarket protocol messages still use hex, and it is also preserved in persistence of PoDLE data. As part of this, some significant simplification of certain legacy functions within the wallet has been done. jmdaemon is entirely unaltered (save for one test which simulates jmclient code). --- jmbase/jmbase/__init__.py | 7 +- jmbase/jmbase/support.py | 132 ++- jmbitcoin/jmbitcoin/__init__.py | 10 +- jmbitcoin/jmbitcoin/bech32.py | 122 -- jmbitcoin/jmbitcoin/btscript.py | 142 --- .../jmbitcoin/secp256k1_deterministic.py | 36 +- jmbitcoin/jmbitcoin/secp256k1_main.py | 519 +-------- jmbitcoin/jmbitcoin/secp256k1_transaction.py | 1012 +++-------------- jmbitcoin/setup.py | 2 +- jmbitcoin/test/test_addresses.py | 56 - jmbitcoin/test/test_bech32.py | 131 --- jmbitcoin/test/test_bip32.py | 4 +- jmbitcoin/test/test_btc_formatting.py | 57 - jmbitcoin/test/test_ecc_signing.py | 30 +- jmbitcoin/test/test_main.py | 61 - jmbitcoin/test/test_tx_serialize.py | 234 ---- jmbitcoin/test/test_tx_signing.py | 91 ++ jmclient/jmclient/__init__.py | 3 +- jmclient/jmclient/blockchaininterface.py | 39 +- jmclient/jmclient/client_protocol.py | 63 +- jmclient/jmclient/commitment_utils.py | 33 +- jmclient/jmclient/configure.py | 50 +- jmclient/jmclient/cryptoengine.py | 71 +- jmclient/jmclient/maker.py | 185 ++- jmclient/jmclient/output.py | 2 +- jmclient/jmclient/podle.py | 241 ++-- jmclient/jmclient/taker.py | 302 +++-- jmclient/jmclient/taker_utils.py | 29 +- jmclient/jmclient/wallet.py | 138 +-- jmclient/jmclient/wallet_service.py | 82 +- jmclient/jmclient/wallet_utils.py | 15 +- jmclient/jmclient/yieldgenerator.py | 6 +- jmclient/test/commontest.py | 62 +- jmclient/test/test_client_protocol.py | 17 +- jmclient/test/test_coinjoin.py | 23 +- jmclient/test/test_commitment_utils.py | 9 +- jmclient/test/test_maker.py | 67 +- jmclient/test/test_payjoin.py | 5 +- jmclient/test/test_podle.py | 29 +- .../test/test_privkeys.py | 32 +- jmclient/test/test_taker.py | 160 +-- jmclient/test/test_tx_creation.py | 241 +--- jmclient/test/test_valid_addresses.py | 32 +- jmclient/test/test_wallet.py | 93 +- jmclient/test/test_wallets.py | 16 +- jmclient/test/test_walletservice.py | 1 - jmclient/test/test_yieldgenerator.py | 16 +- jmdaemon/test/test_message_channel.py | 10 +- scripts/add-utxo.py | 9 +- scripts/convert_old_wallet.py | 5 +- scripts/joinmarket-qt.py | 25 +- scripts/sendtomany.py | 23 +- test/test_segwit.py | 28 +- 53 files changed, 1555 insertions(+), 3253 deletions(-) delete mode 100644 jmbitcoin/jmbitcoin/bech32.py delete mode 100644 jmbitcoin/jmbitcoin/btscript.py delete mode 100644 jmbitcoin/test/test_addresses.py delete mode 100644 jmbitcoin/test/test_bech32.py delete mode 100644 jmbitcoin/test/test_btc_formatting.py delete mode 100644 jmbitcoin/test/test_main.py delete mode 100644 jmbitcoin/test/test_tx_serialize.py create mode 100644 jmbitcoin/test/test_tx_signing.py rename jmbitcoin/test/test_keys.py => jmclient/test/test_privkeys.py (75%) diff --git a/jmbase/jmbase/__init__.py b/jmbase/jmbase/__init__.py index ad3931e..cdbbedb 100644 --- a/jmbase/jmbase/__init__.py +++ b/jmbase/jmbase/__init__.py @@ -2,7 +2,10 @@ from .support import (get_log, chunks, debug_silence, jmprint, joinmarket_alert, core_alert, get_password, set_logging_level, set_logging_color, - lookup_appdata_folder, - JM_WALLET_NAME_PREFIX, JM_APP_NAME) + lookup_appdata_folder, bintohex, bintolehex, + hextobin, lehextobin, utxostr_to_utxo, + utxo_to_utxostr, EXIT_ARGERROR, EXIT_FAILURE, + EXIT_SUCCESS, hexbin, dictchanger, listchanger, + cv, JM_WALLET_NAME_PREFIX, JM_APP_NAME) from .commands import * diff --git a/jmbase/jmbase/support.py b/jmbase/jmbase/support.py index 7175bb5..6afad0a 100644 --- a/jmbase/jmbase/support.py +++ b/jmbase/jmbase/support.py @@ -1,8 +1,9 @@ import logging, sys +import binascii from getpass import getpass from os import path, environ - +from functools import wraps # JoinMarket version JM_CORE_VERSION = '0.7.0dev' @@ -80,6 +81,71 @@ handler = JoinMarketStreamHandler() handler.setFormatter(logFormatter) log.addHandler(handler) +# hex/binary conversion routines used by dependent packages +def hextobin(h): + """Convert a hex string to bytes""" + return binascii.unhexlify(h.encode('utf8')) + + +def bintohex(b): + """Convert bytes to a hex string""" + return binascii.hexlify(b).decode('utf8') + + +def lehextobin(h): + """Convert a little-endian hex string to bytes + + Lets you write uint256's and uint160's the way the Satoshi codebase shows + them. + """ + return binascii.unhexlify(h.encode('utf8'))[::-1] + + +def bintolehex(b): + """Convert bytes to a little-endian hex string + + Lets you show uint256's and uint160's the way the Satoshi codebase shows + them. + """ + return binascii.hexlify(b[::-1]).decode('utf8') + +def utxostr_to_utxo(x): + if not isinstance(x, str): + return (False, "not a string") + y = x.split(":") + if len(y) != 2: + return (False, + "string is not two items separated by :") + try: + n = int(y[1]) + except: + return (False, "utxo index was not an integer.") + if n < 0: + return (False, "utxo index must not be negative.") + if len(y[0]) != 64: + return (False, "txid is not 64 hex characters.") + try: + txid = binascii.unhexlify(y[0]) + except: + return (False, "txid is not hex.") + return (True, (txid, n)) + +def utxo_to_utxostr(u): + if not isinstance(u, tuple): + return (False, "utxo is not a tuple.") + if not len(u) == 2: + return (False, "utxo should have two elements.") + if not isinstance(u[0], bytes): + return (False, "txid should be bytes.") + if not isinstance(u[1], int): + return (False, "index should be int.") + if u[1] < 0: + return (False, "index must be a positive integer.") + if not len(u[0]) == 32: + return (False, "txid must be 32 bytes.") + txid = binascii.hexlify(u[0]).decode("ascii") + return (True, txid + ":" + str(u[1])) + def jmprint(msg, level="info"): """ Provides the ability to print messages with consistent formatting, outside the logging system @@ -150,3 +216,67 @@ def lookup_appdata_folder(appname): def print_jm_version(option, opt_str, value, parser): print("JoinMarket " + JM_CORE_VERSION) sys.exit(EXIT_SUCCESS) + +# helper functions for conversions of format between over-the-wire JM +# and internal. See details in hexbin() docstring. + +def cv(x): + success, utxo = utxostr_to_utxo(x) + if success: + return utxo + else: + try: + b = hextobin(x) + return b + except: + return x + +def listchanger(l): + rlist = [] + for x in l: + if isinstance(x, list): + rlist.append(listchanger(x)) + elif isinstance(x, dict): + rlist.append(dictchanger(x)) + else: + rlist.append(cv(x)) + return rlist + +def dictchanger(d): + rdict = {} + for k, v in d.items(): + if isinstance(v, dict): + rdict[cv(k)] = dictchanger(v) + elif isinstance(v, list): + rdict[cv(k)] = listchanger(v) + else: + rdict[cv(k)] = cv(v) + return rdict + +def hexbin(func): + """ Decorator for functions of taker and maker receiving over + the wire AMP arguments that may be in hex or hextxid:n format + and converting all to binary. + Functions to which this decorator applies should have all arguments + be one of: + - hex string (keys), converted here to binary + - lists of keys or txid:n strings (converted here to binary, or + (txidbytes, n)) + - lists of lists or dicts, to which these rules apply recursively. + - any other string (unchanged) + - dicts with keys as per above; values are altered recursively according + to the rules above. + """ + @wraps(func) + def func_wrapper(inst, *args, **kwargs): + newargs = [] + for arg in args: + if isinstance(arg, (list, tuple)): + newargs.append(listchanger(arg)) + elif isinstance(arg, dict): + newargs.append(dictchanger(arg)) + else: + newargs.append(cv(arg)) + return func(inst, *newargs, **kwargs) + + return func_wrapper \ No newline at end of file diff --git a/jmbitcoin/jmbitcoin/__init__.py b/jmbitcoin/jmbitcoin/__init__.py index 784673a..00fc519 100644 --- a/jmbitcoin/jmbitcoin/__init__.py +++ b/jmbitcoin/jmbitcoin/__init__.py @@ -2,8 +2,14 @@ import coincurve as secp256k1 from jmbitcoin.secp256k1_main import * from jmbitcoin.secp256k1_transaction import * from jmbitcoin.secp256k1_deterministic import * -from jmbitcoin.btscript import * -from jmbitcoin.bech32 import * from jmbitcoin.amount import * from jmbitcoin.bip21 import * +from bitcointx import select_chain_params +from bitcointx.core import (x, b2x, b2lx, lx, COutPoint, CTxOut, CTxIn, + CTxInWitness, CTxWitness, CMutableTransaction, + Hash160, coins_to_satoshi, satoshi_to_coins) +from bitcointx.core.script import (CScript, OP_0, SignatureHash, SIGHASH_ALL, + SIGVERSION_WITNESS_V0, CScriptWitness) +from bitcointx.wallet import (CBitcoinSecret, P2WPKHBitcoinAddress, CCoinAddress, + P2SHCoinAddress) diff --git a/jmbitcoin/jmbitcoin/bech32.py b/jmbitcoin/jmbitcoin/bech32.py deleted file mode 100644 index 8d3f9bf..0000000 --- a/jmbitcoin/jmbitcoin/bech32.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright (c) 2017 Pieter Wuille -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -"""Reference implementation for Bech32 and segwit addresses.""" - -CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" - - -def bech32_polymod(values): - """Internal function that computes the Bech32 checksum.""" - generator = [0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3] - chk = 1 - for value in values: - top = chk >> 25 - chk = (chk & 0x1ffffff) << 5 ^ value - for i in range(5): - chk ^= generator[i] if ((top >> i) & 1) else 0 - return chk - - -def bech32_hrp_expand(hrp): - """Expand the HRP into values for checksum computation.""" - return [ord(x) >> 5 for x in hrp] + [0] + [ord(x) & 31 for x in hrp] - - -def bech32_verify_checksum(hrp, data): - """Verify a checksum given HRP and converted data characters.""" - return bech32_polymod(bech32_hrp_expand(hrp) + data) == 1 - - -def bech32_create_checksum(hrp, data): - """Compute the checksum values given HRP and data.""" - values = bech32_hrp_expand(hrp) + data - polymod = bech32_polymod(values + [0, 0, 0, 0, 0, 0]) ^ 1 - return [(polymod >> 5 * (5 - i)) & 31 for i in range(6)] - - -def bech32_encode(hrp, data): - """Compute a Bech32 string given HRP and data values.""" - combined = data + bech32_create_checksum(hrp, data) - return hrp + '1' + ''.join([CHARSET[d] for d in combined]) - - -def bech32_decode(bech): - """Validate a Bech32 string, and determine HRP and data.""" - if ((any(ord(x) < 33 or ord(x) > 126 for x in bech)) or - (bech.lower() != bech and bech.upper() != bech)): - return (None, None) - bech = bech.lower() - pos = bech.rfind('1') - if pos < 1 or pos + 7 > len(bech) or len(bech) > 90: - return (None, None) - if not all(x in CHARSET for x in bech[pos+1:]): - return (None, None) - hrp = bech[:pos] - data = [CHARSET.find(x) for x in bech[pos+1:]] - if not bech32_verify_checksum(hrp, data): - return (None, None) - return (hrp, data[:-6]) - - -def convertbits(data, frombits, tobits, pad=True): - """General power-of-2 base conversion.""" - acc = 0 - bits = 0 - ret = [] - maxv = (1 << tobits) - 1 - max_acc = (1 << (frombits + tobits - 1)) - 1 - for value in data: - if value < 0 or (value >> frombits): - return None - acc = ((acc << frombits) | value) & max_acc - bits += frombits - while bits >= tobits: - bits -= tobits - ret.append((acc >> bits) & maxv) - if pad: - if bits: - ret.append((acc << (tobits - bits)) & maxv) - elif bits >= frombits or ((acc << (tobits - bits)) & maxv): - return None - return ret - - -def bech32addr_decode(hrp, addr): - """Decode a segwit address.""" - hrpgot, data = bech32_decode(addr) - if hrpgot != hrp: - return (None, None) - decoded = convertbits(data[1:], 5, 8, False) - if decoded is None or len(decoded) < 2 or len(decoded) > 40: - return (None, None) - if data[0] > 16: - return (None, None) - if data[0] == 0 and len(decoded) != 20 and len(decoded) != 32: - return (None, None) - return (data[0], decoded) - - -def bech32addr_encode(hrp, witver, witprog): - """Encode a segwit address.""" - ret = bech32_encode(hrp, [witver] + convertbits(witprog, 8, 5)) - if bech32addr_decode(hrp, ret) == (None, None): - return None - return ret diff --git a/jmbitcoin/jmbitcoin/btscript.py b/jmbitcoin/jmbitcoin/btscript.py deleted file mode 100644 index c658c4e..0000000 --- a/jmbitcoin/jmbitcoin/btscript.py +++ /dev/null @@ -1,142 +0,0 @@ -#OP codes; disabled commented. - -# push value -OP_0 = 0x00 -OP_FALSE = OP_0 -OP_PUSHDATA1 = 0x4c -OP_PUSHDATA2 = 0x4d -OP_PUSHDATA4 = 0x4e -OP_1NEGATE = 0x4f -OP_RESERVED = 0x50 -OP_1 = 0x51 -OP_TRUE = OP_1 -OP_2 = 0x52 -OP_3 = 0x53 -OP_4 = 0x54 -OP_5 = 0x55 -OP_6 = 0x56 -OP_7 = 0x57 -OP_8 = 0x58 -OP_9 = 0x59 -OP_10 = 0x5a -OP_11 = 0x5b -OP_12 = 0x5c -OP_13 = 0x5d -OP_14 = 0x5e -OP_15 = 0x5f -OP_16 = 0x60 - -# control -OP_NOP = 0x61 -OP_VER = 0x62 -OP_IF = 0x63 -OP_NOTIF = 0x64 -#OP_VERIF = 0x65 -#OP_VERNOTIF = 0x66 -OP_ELSE = 0x67 -OP_ENDIF = 0x68 -OP_VERIFY = 0x69 -OP_RETURN = 0x6a - -# stack ops -OP_TOALTSTACK = 0x6b -OP_FROMALTSTACK = 0x6c -OP_2DROP = 0x6d -OP_2DUP = 0x6e -OP_3DUP = 0x6f -OP_2OVER = 0x70 -OP_2ROT = 0x71 -OP_2SWAP = 0x72 -OP_IFDUP = 0x73 -OP_DEPTH = 0x74 -OP_DROP = 0x75 -OP_DUP = 0x76 -OP_NIP = 0x77 -OP_OVER = 0x78 -OP_PICK = 0x79 -OP_ROLL = 0x7a -OP_ROT = 0x7b -OP_SWAP = 0x7c -OP_TUCK = 0x7d - -# splice ops -#OP_CAT = 0x7e -#OP_SUBSTR = 0x7f -#OP_LEFT = 0x80 -#OP_RIGHT = 0x81 -OP_SIZE = 0x82 - -# bit logic -#OP_INVERT = 0x83 -#OP_AND = 0x84 -#OP_OR = 0x85 -#OP_XOR = 0x86 -OP_EQUAL = 0x87 -OP_EQUALVERIFY = 0x88 -OP_RESERVED1 = 0x89 -OP_RESERVED2 = 0x8a - -# numeric -OP_1ADD = 0x8b -OP_1SUB = 0x8c -#OP_2MUL = 0x8d -#OP_2DIV = 0x8e -OP_NEGATE = 0x8f -OP_ABS = 0x90 -OP_NOT = 0x91 -OP_0NOTEQUAL = 0x92 - -OP_ADD = 0x93 -OP_SUB = 0x94 -#OP_MUL = 0x95 -#OP_DIV = 0x96 -#OP_MOD = 0x97 -#OP_LSHIFT = 0x98 -#OP_RSHIFT = 0x99 - -OP_BOOLAND = 0x9a -OP_BOOLOR = 0x9b -OP_NUMEQUAL = 0x9c -OP_NUMEQUALVERIFY = 0x9d -OP_NUMNOTEQUAL = 0x9e -OP_LESSTHAN = 0x9f -OP_GREATERTHAN = 0xa0 -OP_LESSTHANOREQUAL = 0xa1 -OP_GREATERTHANOREQUAL = 0xa2 -OP_MIN = 0xa3 -OP_MAX = 0xa4 - -OP_WITHIN = 0xa5 - -# crypto -OP_RIPEMD160 = 0xa6 -OP_SHA1 = 0xa7 -OP_SHA256 = 0xa8 -OP_HASH160 = 0xa9 -OP_HASH256 = 0xaa -OP_CODESEPARATOR = 0xab -OP_CHECKSIG = 0xac -OP_CHECKSIGVERIFY = 0xad -OP_CHECKMULTISIG = 0xae -OP_CHECKMULTISIGVERIFY = 0xaf - -# expansion -OP_NOP1 = 0xb0 -OP_NOP2 = 0xb1 -OP_CHECKLOCKTIMEVERIFY = OP_NOP2 -OP_NOP3 = 0xb2 -OP_NOP4 = 0xb3 -OP_NOP5 = 0xb4 -OP_NOP6 = 0xb5 -OP_NOP7 = 0xb6 -OP_NOP8 = 0xb7 -OP_NOP9 = 0xb8 -OP_NOP10 = 0xb9 - -# template matching params -OP_SMALLINTEGER = 0xfa -OP_PUBKEYS = 0xfb -OP_PUBKEYHASH = 0xfd -OP_PUBKEY = 0xfe - -OP_INVALIDOPCODE = 0xff diff --git a/jmbitcoin/jmbitcoin/secp256k1_deterministic.py b/jmbitcoin/jmbitcoin/secp256k1_deterministic.py index 1baad53..eb9023d 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_deterministic.py +++ b/jmbitcoin/jmbitcoin/secp256k1_deterministic.py @@ -2,6 +2,8 @@ from jmbitcoin.secp256k1_main import * import hmac import hashlib import struct +from bitcointx.core import Hash160, Hash +from bitcointx import base58 # Below code ASSUMES binary inputs and compressed pubkeys MAINNET_PRIVATE = b'\x04\x88\xAD\xE4' @@ -11,6 +13,8 @@ TESTNET_PUBLIC = b'\x04\x35\x87\xCF' PRIVATE = [MAINNET_PRIVATE, TESTNET_PRIVATE] PUBLIC = [MAINNET_PUBLIC, TESTNET_PUBLIC] +privtopub = privkey_to_pubkey + # BIP32 child key derivation def raw_bip32_ckd(rawtuple, i): @@ -19,25 +23,25 @@ def raw_bip32_ckd(rawtuple, i): if vbytes in PRIVATE: priv = key - pub = privtopub(key, False) + pub = privtopub(key) else: pub = key if i >= 2**31: if vbytes in PUBLIC: raise Exception("Can't do private derivation on public key!") - I = hmac.new(chaincode, b'\x00' + priv[:32] + encode(i, 256, 4), + I = hmac.new(chaincode, b'\x00' + priv[:32] + struct.pack(b'>L', i), hashlib.sha512).digest() else: - I = hmac.new(chaincode, pub + encode(i, 256, 4), + I = hmac.new(chaincode, pub + struct.pack(b'>L', i), hashlib.sha512).digest() if vbytes in PRIVATE: - newkey = add_privkeys(I[:32] + b'\x01', priv, False) - fingerprint = bin_hash160(privtopub(key, False))[:4] + newkey = add_privkeys(I[:32] + b'\x01', priv) + fingerprint = Hash160(privtopub(key))[:4] if vbytes in PUBLIC: - newkey = add_pubkeys([privtopub(I[:32] + b'\x01', False), key], False) - fingerprint = bin_hash160(key)[:4] + newkey = add_pubkeys([privtopub(I[:32] + b'\x01'), key]) + fingerprint = Hash160(key)[:4] return (vbytes, depth + 1, fingerprint, i, I[32:], newkey) @@ -47,18 +51,17 @@ def bip32_serialize(rawtuple): i = struct.pack(b'>L', i) chaincode = chaincode keydata = b'\x00' + key[:-1] if vbytes in PRIVATE else key - bindata = vbytes + from_int_to_byte( - depth % 256) + fingerprint + i + chaincode + keydata - return b58encode(bindata + bin_dbl_sha256(bindata)[:4]) + bindata = vbytes + struct.pack(b'B',depth % 256) + fingerprint + i + chaincode + keydata + return base58.encode(bindata + Hash(bindata)[:4]) def bip32_deserialize(data): - dbin = b58decode(data) - if bin_dbl_sha256(dbin[:-4])[:4] != dbin[-4:]: + dbin = base58.decode(data) + if Hash(dbin[:-4])[:4] != dbin[-4:]: raise Exception("Invalid checksum") vbytes = dbin[0:4] depth = dbin[4] fingerprint = dbin[5:9] - i = decode(dbin[9:13], 256) + i = struct.unpack(b'>L',dbin[9:13])[0] chaincode = dbin[13:45] key = dbin[46:78] + b'\x01' if vbytes in PRIVATE else dbin[45:78] return (vbytes, depth, fingerprint, i, chaincode, key) @@ -68,7 +71,7 @@ def raw_bip32_privtopub(rawtuple): if vbytes in PUBLIC: return rawtuple newvbytes = MAINNET_PUBLIC if vbytes == MAINNET_PRIVATE else TESTNET_PUBLIC - return (newvbytes, depth, fingerprint, i, chaincode, privtopub(key, False)) + return (newvbytes, depth, fingerprint, i, chaincode, privtopub(key)) def bip32_privtopub(data): return bip32_serialize(raw_bip32_privtopub(bip32_deserialize(data))) @@ -77,13 +80,12 @@ def bip32_ckd(data, i): return bip32_serialize(raw_bip32_ckd(bip32_deserialize(data), i)) def bip32_master_key(seed, vbytes=MAINNET_PRIVATE): - I = hmac.new( - from_string_to_bytes("Bitcoin seed"), seed, hashlib.sha512).digest() + I = hmac.new("Bitcoin seed".encode("utf-8"), seed, hashlib.sha512).digest() return bip32_serialize((vbytes, 0, b'\x00' * 4, 0, I[32:], I[:32] + b'\x01' )) def bip32_extract_key(data): - return safe_hexlify(bip32_deserialize(data)[-1]) + return bip32_deserialize(data)[-1] def bip32_descend(*args): if len(args) == 2: diff --git a/jmbitcoin/jmbitcoin/secp256k1_main.py b/jmbitcoin/jmbitcoin/secp256k1_main.py index e91fc59..db0e7cb 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_main.py +++ b/jmbitcoin/jmbitcoin/secp256k1_main.py @@ -7,213 +7,16 @@ import base64 import struct import coincurve as secp256k1 +from bitcointx import base58 +from bitcointx.core import Hash +from bitcointx.signmessage import BitcoinMessage + #Required only for PoDLE calculation: N = 115792089237316195423570985008687907852837564279074904382605163141518161494337 BTC_P2PK_VBYTE = {"mainnet": b'\x00', "testnet": b'\x6f', "regtest": 100} BTC_P2SH_VBYTE = {"mainnet": b'\x05', "testnet": b'\xc4'} -#Standard prefix for Bitcoin message signing. -BITCOIN_MESSAGE_MAGIC = b'\x18' + b'Bitcoin Signed Message:\n' - -string_types = (str) -string_or_bytes_types = (str, bytes) -int_types = (int, float) - -# Base switching -code_strings = { - 2: '01', - 10: '0123456789', - 16: '0123456789abcdef', - 32: 'abcdefghijklmnopqrstuvwxyz234567', - 58: '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz', - 256: ''.join([chr(x) for x in range(256)]) -} - -def lpad(msg, symbol, length): - if len(msg) >= length: - return msg - return symbol * (length - len(msg)) + msg - -def get_code_string(base): - if base in code_strings: - return code_strings[base] - else: - raise ValueError("Invalid base!") - -def bin_to_b58check(inp, magicbyte=b'\x00'): - if not isinstance(magicbyte, int): - magicbyte = struct.unpack(b'B', magicbyte)[0] - assert(0 <= magicbyte <= 0xff) - if magicbyte == 0: - inp_fmtd = struct.pack(b'B', magicbyte) + inp - while magicbyte > 0: - inp_fmtd = struct.pack(b'B', magicbyte % 256) + inp - magicbyte //= 256 - checksum = bin_dbl_sha256(inp_fmtd)[:4] - return b58encode(inp_fmtd + checksum) - -def safe_from_hex(s): - return binascii.unhexlify(s) - -def from_int_to_byte(a): - return struct.pack(b'B', a) - -def from_byte_to_int(a): - return struct.unpack(b'B', a)[0] - -def from_string_to_bytes(a): - return a if isinstance(a, bytes) else bytes(a, 'utf-8') - -def safe_hexlify(a): - return binascii.hexlify(a).decode('ascii') - -class SerializationError(Exception): - """Base class for serialization errors""" - - -class SerializationTruncationError(SerializationError): - """Serialized data was truncated - Thrown by deserialize() and stream_deserialize() - """ - -def ser_read(f, n): - """Read from a stream safely - Raises SerializationError and SerializationTruncationError appropriately. - Use this instead of f.read() in your classes stream_(de)serialization() - functions. - """ - MAX_SIZE = 0x02000000 - if n > MAX_SIZE: - raise SerializationError('Asked to read 0x%x bytes; MAX_SIZE exceeded' % n) - r = f.read(n) - if len(r) < n: - raise SerializationTruncationError('Asked to read %i bytes, but only got %i' % (n, len(r))) - return r - -B58_DIGITS = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' - -class Base58Error(Exception): - pass - -class InvalidBase58Error(Base58Error): - """Raised on generic invalid base58 data, such as bad characters. - Checksum failures raise Base58ChecksumError specifically. - """ - pass - -def b58encode(b): - """Encode bytes to a base58-encoded string""" - - # Convert big-endian bytes to integer - n = int('0x0' + binascii.hexlify(b).decode('ascii'), 16) - - # Divide that integer into bas58 - res = [] - while n > 0: - n, r = divmod(n, 58) - res.append(B58_DIGITS[r]) - res = ''.join(res[::-1]) - - # Encode leading zeros as base58 zeros - czero = b'\x00' - if sys.version_info >= (3,0): - # In Python3 indexing a bytes returns numbers, not characters. - czero = 0 - pad = 0 - for c in b: - if c == czero: - pad += 1 - else: - break - return B58_DIGITS[0] * pad + res - -def b58decode(s): - """Decode a base58-encoding string, returning bytes""" - if not s: - return b'' - - # Convert the string to an integer - n = 0 - for c in s: - n *= 58 - if c not in B58_DIGITS: - raise InvalidBase58Error('Character %r is not a valid base58 character' % c) - digit = B58_DIGITS.index(c) - n += digit - - # Convert the integer to bytes - h = '%x' % n - if len(h) % 2: - h = '0' + h - res = bytes(binascii.unhexlify(h.encode('utf8'))) - - # Add padding back. - pad = 0 - for c in s[:-1]: - if c == B58_DIGITS[0]: pad += 1 - else: break - return b'\x00' * pad + res - -def uint256encode(s): - """Convert bytes to uint256""" - r = 0 - t = struct.unpack(b"> (i * 32) & 0xffffffff) - return r - -def encode(val, base, minlen=0): - base, minlen = int(base), int(minlen) - code_string = get_code_string(base) - result_bytes = bytes() - while val > 0: - curcode = code_string[val % base] - result_bytes = bytes([ord(curcode)]) + result_bytes - val //= base - - pad_size = minlen - len(result_bytes) - - padding_element = b'\x00' if base == 256 else b'1' \ - if base == 58 else b'0' - if (pad_size > 0): - result_bytes = padding_element*pad_size + result_bytes - - result_string = ''.join([chr(y) for y in result_bytes]) - result = result_bytes if base == 256 else result_string - - return result - -def decode(string, base): - if base == 256 and isinstance(string, str): - string = bytes(bytearray.fromhex(string)) - base = int(base) - code_string = get_code_string(base) - result = 0 - if base == 256: - def extract(d, cs): - if isinstance(d, int): - return d - else: - return struct.unpack(b'B', d)[0] - else: - def extract(d, cs): - return cs.find(d if isinstance(d, str) else chr(d)) - - if base == 16: - string = string.lower() - while len(string) > 0: - result *= base - result += extract(string[0], code_string) - string = string[1:] - return result - """PoDLE related primitives """ def getG(compressed=True): @@ -237,165 +40,6 @@ def podle_PrivateKey(priv): """ return secp256k1.PrivateKey(priv) - -def privkey_to_address(priv, from_hex=True, magicbyte=0): - return pubkey_to_address(privkey_to_pubkey(priv, from_hex), magicbyte) - -privtoaddr = privkey_to_address - -# Hashes -def bin_hash160(string): - intermed = hashlib.sha256(string).digest() - return hashlib.new('ripemd160', intermed).digest() - -def hash160(string): - if not isinstance(string, bytes): - string = string.encode('utf-8') - return safe_hexlify(bin_hash160(string)) - -def bin_sha256(string): - binary_data = string if isinstance(string, bytes) else bytes(string, - 'utf-8') - return hashlib.sha256(binary_data).digest() - -def sha256(string): - return safe_hexlify(bin_sha256(string)) - -def bin_dbl_sha256(bytes_to_hash): - if not isinstance(bytes_to_hash, bytes): - bytes_to_hash = bytes_to_hash.encode('utf-8') - return hashlib.sha256(hashlib.sha256(bytes_to_hash).digest()).digest() - -def dbl_sha256(string): - return hashlib.sha256(hashlib.sha256(string).digest()).hexdigest() - -def hash_to_int(x): - if len(x) in [40, 64]: - return decode(x, 16) - return decode(x, 256) - -def num_to_var_int(x): - if not isinstance(x, int): - if len(x) == 0: - return b'\x00' - x = struct.unpack(b'B', x)[0] - if x < 253: return from_int_to_byte(x) - elif x < 65536: return from_int_to_byte(253) + struct.pack(b'= (3,0): - newpriv = secp256k1.PrivateKey(secret=native_bytes(priv)) - else: - newpriv = secp256k1.PrivateKey(secret=bytes_to_native_str(priv)) + newpriv = secp256k1.PrivateKey(secret=native_bytes(priv)) return newpriv.public_key.format(compressed) -def privkey_to_pubkey(priv, usehex=True): - '''To avoid changing the interface from the legacy system, - allow an *optional* hex argument here (called differently from - maker/taker code to how it's called in bip32 code), then - pass to the standard hexbin decorator under the hood. - ''' - return privkey_to_pubkey_inner(priv, usehex) +# b58check wrapper functions around bitcointx.base58 functions: +# (avoids complexity of key management structure) + +def bin_to_b58check(inp, magicbyte=b'\x00'): + """ The magic byte (prefix byte) should be passed either + as a single byte or an integer. What is returned is a string + in base58 encoding, with the prefix and the checksum. + """ + if not isinstance(magicbyte, int): + magicbyte = struct.unpack(b'B', magicbyte)[0] + assert(0 <= magicbyte <= 0xff) + if magicbyte == 0: + inp_fmtd = struct.pack(b'B', magicbyte) + inp + while magicbyte > 0: + inp_fmtd = struct.pack(b'B', magicbyte % 256) + inp + magicbyte //= 256 + checksum = Hash(inp_fmtd)[:4] + return base58.encode(inp_fmtd + checksum) + +def b58check_to_bin(s): + data = base58.decode(s) + assert Hash(data[:-4])[:4] == data[-4:] + return struct.pack(b"B", data[0]), data[1:-4] -privtopub = privkey_to_pubkey +def get_version_byte(s): + return b58check_to_bin(s)[0] -@hexbin -def is_valid_pubkey(pubkey, usehex, require_compressed=False): +def ecdsa_sign(msg, priv, formsg=False): + hashed_msg = BitcoinMessage(msg).GetHash() + sig = ecdsa_raw_sign(hashed_msg, priv, rawmsg=True, formsg=formsg) + return base64.b64encode(sig).decode('ascii') + +def ecdsa_verify(msg, sig, pub): + hashed_msg = BitcoinMessage(msg).GetHash() + sig = base64.b64decode(sig) + return ecdsa_raw_verify(hashed_msg, pub, sig, rawmsg=True) + +def is_valid_pubkey(pubkey, require_compressed=False): """ Returns True if the serialized pubkey is a valid secp256k1 pubkey serialization or False if not; returns False for an uncompressed encoding if require_compressed is True. @@ -458,8 +124,8 @@ def is_valid_pubkey(pubkey, usehex, require_compressed=False): return False return True -@hexbin -def multiply(s, pub, usehex, rawpub=True, return_serialized=True): + +def multiply(s, pub, return_serialized=True): '''Input binary compressed pubkey P(33 bytes) and scalar s(32 bytes), return s*P. The return value is a binary compressed public key, @@ -470,24 +136,19 @@ def multiply(s, pub, usehex, rawpub=True, return_serialized=True): ''' newpub = secp256k1.PublicKey(pub) #see note to "tweak_mul" function in podle.py - if sys.version_info >= (3,0): - res = newpub.multiply(native_bytes(s)) - else: - res = newpub.multiply(bytes_to_native_str(s)) + res = newpub.multiply(native_bytes(s)) if not return_serialized: return res return res.format() -@hexbin -def add_pubkeys(pubkeys, usehex): +def add_pubkeys(pubkeys): '''Input a list of binary compressed pubkeys and return their sum as a binary compressed pubkey.''' pubkey_list = [secp256k1.PublicKey(x) for x in pubkeys] r = secp256k1.PublicKey.combine_keys(pubkey_list) return r.format() -@hexbin -def add_privkeys(priv1, priv2, usehex): +def add_privkeys(priv1, priv2): '''Add privkey 1 to privkey 2. Input keys must be in binary either compressed or not. Returned key will have the same compression state. @@ -504,58 +165,34 @@ def add_privkeys(priv1, priv2, usehex): res += b'\x01' return res -@hexbin + def ecdsa_raw_sign(msg, priv, - usehex, - rawpriv=True, rawmsg=False, - usenonce=None, formsg=False): '''Take the binary message msg and sign it with the private key priv. - By default priv is just a 32 byte string, if rawpriv is false - it is assumed to be hex encoded (note only works if usehex=False). If rawmsg is True, no sha256 hash is applied to msg before signing. In this case, msg must be a precalculated hash (256 bit). If rawmsg is False, the secp256k1 lib will hash the message as part of the ECDSA-SHA256 signing algo. - If usenonce is not None, its value is passed to the secp256k1 library - sign() function as the ndata value, which is then used in conjunction - with a custom nonce generating function, such that the nonce used in the ECDSA - sign algorithm is exactly that value (ndata there, usenonce here). 32 bytes. Return value: the calculated signature.''' if rawmsg and len(msg) != 32: raise Exception("Invalid hash input to ECDSA raw sign.") - if rawpriv: - compressed, p = read_privkey(priv) - newpriv = secp256k1.PrivateKey(p) - else: - newpriv = secp256k1.PrivateKey.from_hex(priv) + + compressed, p = read_privkey(priv) + newpriv = secp256k1.PrivateKey(p) if formsg: sig = newpriv.sign_recoverable(msg) return sig - #Donations, thus custom nonce, currently disabled, hence not covered. - elif usenonce: #pragma: no cover - raise NotImplementedError - #if len(usenonce) != 32: - # raise ValueError("Invalid nonce passed to ecdsa_sign: " + str( - # usenonce)) - #nf = ffi.addressof(_noncefunc.lib, "nonce_function_rand") - #ndata = ffi.new("char [32]", usenonce) - #usenonce = (nf, ndata) - #sig = newpriv.ecdsa_sign(msg, raw=rawmsg, custom_nonce=usenonce) else: - #partial fix for secp256k1-transient not including customnonce; - #partial because donations will crash on windows in the "if". if rawmsg: sig = newpriv.sign(msg, hasher=None) else: sig = newpriv.sign(msg) return sig -@hexbin -def ecdsa_raw_verify(msg, pub, sig, usehex, rawmsg=False): +def ecdsa_raw_verify(msg, pub, sig, rawmsg=False): '''Take the binary message msg and binary signature sig, and verify it against the pubkey pub. If rawmsg is True, no sha256 hash is applied to msg before verifying. @@ -575,56 +212,6 @@ def ecdsa_raw_verify(msg, pub, sig, usehex, rawmsg=False): retval = newpub.verify(sig, msg, hasher=None) else: retval = newpub.verify(sig, msg) - except: + except Exception as e: return False return retval - -def estimate_tx_size(ins, outs, txtype='p2pkh'): - '''Estimate transaction size. - The txtype field as detailed below is used to distinguish - the type, but there is at least one source of meaningful roughness: - we assume the output types are the same as the input (to be fair, - outputs only contribute a little to the overall total). This combined - with a few bytes variation in signature sizes means we will expect, - say, 10% inaccuracy here. - - Assuming p2pkh: - out: 8+1+3+2+20=34, in: 1+32+4+1+1+~73+1+1+33=147, - ver:4,seq:4, +2 (len in,out) - total ~= 34*len_out + 147*len_in + 10 (sig sizes vary slightly) - Assuming p2sh M of N multisig: - "ins" must contain M, N so ins= (numins, M, N) (crude assuming all same) - 74*M + 34*N + 45 per input, so total ins ~ len_ins * (45+74M+34N) - so total ~ 34*len_out + (45+74M+34N)*len_in + 10 - Assuming p2sh-p2wpkh: - witness are roughly 3+~73+33 for each input - (txid, vin, 4+20 for witness program encoded as scriptsig, 4 for sequence) - non-witness input fields are roughly 32+4+4+20+4=64, so total becomes - n_in * 64 + 4(ver) + 4(locktime) + n_out*34 - Assuming p2wpkh native: - witness as previous case - non-witness loses the 24 witnessprogram, replaced with 1 zero, - in the scriptSig, so becomes: - n_in * 41 + 4(ver) + 4(locktime) +2 (len in, out) + n_out*34 - ''' - if txtype == 'p2pkh': - return 10 + ins * 147 + 34 * outs - elif txtype == 'p2sh-p2wpkh': - #return the estimate for the witness and non-witness - #portions of the transaction, assuming that all the inputs - #are of segwit type p2sh-p2wpkh - # Note as of Jan19: this misses 2 bytes (trivial) for len in, out - # and also overestimates output size by 2 bytes. - witness_estimate = ins*109 - non_witness_estimate = 4 + 4 + outs*34 + ins*64 - return (witness_estimate, non_witness_estimate) - elif txtype == 'p2wpkh': - witness_estimate = ins*109 - non_witness_estimate = 4 + 4 + 2 + outs*31 + ins*41 - return (witness_estimate, non_witness_estimate) - elif txtype == 'p2shMofN': - ins, M, N = ins - return 10 + (45 + 74*M + 34*N) * ins + 34 * outs - else: - raise NotImplementedError("Transaction size estimation not" + - "yet implemented for type: " + txtype) diff --git a/jmbitcoin/jmbitcoin/secp256k1_transaction.py b/jmbitcoin/jmbitcoin/secp256k1_transaction.py index b8982da..aca9903 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_transaction.py +++ b/jmbitcoin/jmbitcoin/secp256k1_transaction.py @@ -9,548 +9,95 @@ import struct # note, only used for non-cryptographic randomness: import random from jmbitcoin.secp256k1_main import * -from jmbitcoin.bech32 import * -import jmbitcoin as btc -P2PKH_PRE, P2PKH_POST = b'\x76\xa9\x14', b'\x88\xac' -P2SH_P2WPKH_PRE, P2SH_P2WPKH_POST = b'\xa9\x14', b'\x87' -P2WPKH_PRE = b'\x00\x14' -P2WSH_PRE = b'\x00\x20' - -# Transaction serialization and deserialization - -def deserialize(txinp): - if isinstance(txinp, basestring) and not isinstance(txinp, bytes): - tx = BytesIO(binascii.unhexlify(txinp)) - hexout = True - else: - tx = BytesIO(txinp) - hexout = False - - def hex_string(scriptbytes, hexout): - if hexout: - return binascii.hexlify(scriptbytes).decode('ascii') - else: - return scriptbytes - - def read_as_int(bytez): - if bytez == 2: - return struct.unpack(b' len(newtx["outs"]): - raise Exception( - "Transactions with sighash single should have len in <= len out") - newtx["outs"] = newtx["outs"][:i+1] - for out in newtx["outs"][:i]: - out['value'] = 2**64 - 1 - out['script'] = "" - for j, inp in enumerate(newtx["ins"]): - if j != i: - inp["sequence"] = 0 - if hashcode & SIGHASH_ANYONECANPAY: - newtx["ins"] = [newtx["ins"][i]] - else: - pass - return newtx - -def segwit_txid(tx, hashcode=None): - #An easy way to construct the old-style hash (which is the real txid, - #the one without witness or marker/flag, is to remove all txinwitness - #entries from the deserialized form of the full tx, then reserialize, - #because serialize uses that as a flag to decide which serialization - #style to apply. - dtx = deserialize(tx) - for vin in dtx["ins"]: - if "txinwitness" in vin: - del vin["txinwitness"] - reserialized_tx = serialize(dtx) - return txhash(reserialized_tx, hashcode) - -def txhash(tx, hashcode=None, check_sw=True): - """ Creates the appropriate sha256 hash as required - either for signing or calculating txids. - The hashcode argument is used to distinguish the case - where we are hashing for signing (sighashing); by default - it is None, and this indicates we are calculating a txid. - If check_sw is True it checks the serialized format for - segwit flag bytes, and produces the correct form for txid (not wtxid). - """ - if not isinstance(tx, basestring): - tx = serialize(tx) - if isinstance(tx, basestring) and not isinstance(tx, bytes): - tx = binascii.unhexlify(tx) - if check_sw and from_byte_to_int(tx[4:5]) == 0: - if not from_byte_to_int(tx[5:6]) == 1: - #This invalid, but a raise is a DOS vector in some contexts. - return None - return segwit_txid(tx, hashcode) - if hashcode: - if not isinstance(hashcode, int): - hashcode = struct.unpack(b'B', hashcode)[0] - return dbl_sha256(from_string_to_bytes(tx) + struct.pack(b'= 0 - if not isinstance(tx, bytes): - tx = binascii.unhexlify(tx) - if not isinstance(script, bytes): - script = binascii.unhexlify(script) - if scriptCode is not None and not isinstance(scriptCode, bytes): - scriptCode = binascii.unhexlify(scriptCode) - if isinstance(sig, bytes): - sig = binascii.hexlify(sig).decode('ascii') - if isinstance(pub, bytes): - pub = binascii.hexlify(pub).decode('ascii') - - hashcode = binascii.unhexlify(sig[-2:]) - - if amount: - modtx = segwit_signature_form(deserialize(tx), i, - scriptCode, amount, hashcode, decoder_func=lambda x: x) - else: - modtx = signature_form(tx, i, script, hashcode) - return ecdsa_tx_verify(modtx, sig, pub, hashcode) - - -def sign(tx, i, priv, hashcode=SIGHASH_ALL, usenonce=None, amount=None, - native=False): - """ - Given a serialized transaction tx, an input index i, and a privkey - in bytes or hex, returns a serialized transaction, in hex always, - into which the signature and/or witness has been inserted. The field - `amount` flags whether segwit signing is to be done, and the field - `native` flags that native segwit p2wpkh signing is to be done. Note - that signing multisig is to be done with the alternative functions - get_p2sh_signature or get_p2wsh_signature (and non N of N multisig - scripthash signing is not currently supported). - """ - if isinstance(tx, basestring) and not isinstance(tx, bytes): - tx = binascii.unhexlify(tx) - if len(priv) <= 33: - priv = safe_hexlify(priv) - if amount: - serobj = p2wpkh_sign(tx, i, priv, amount, hashcode=hashcode, - usenonce=usenonce, native=native) - else: - pub = privkey_to_pubkey(priv, True) - address = pubkey_to_address(pub) - signing_tx = signature_form(tx, i, mk_pubkey_script(address), hashcode) - sig = ecdsa_tx_sign(signing_tx, priv, hashcode, usenonce=usenonce) - txobj = deserialize(tx) - txobj["ins"][i]["script"] = serialize_script([sig, pub]) - serobj = serialize(txobj) - if isinstance(serobj, basestring) and isinstance(serobj, bytes): - return binascii.hexlify(serobj).decode('ascii') - else: - return serobj - -def p2wpkh_sign(tx, i, priv, amount, hashcode=SIGHASH_ALL, native=False, - usenonce=None): - """Given a serialized transaction, index, private key in hex, - amount in satoshis and optionally hashcode, return the serialized - transaction containing a signature and witness for this input; it's - assumed that the input is of type pay-to-witness-pubkey-hash. - If native is False, it's treated as p2sh nested. + Given a transaction tx of type CMutableTransaction, an input index i, + and a raw privkey in bytes, updates the CMutableTransaction to contain + the newly appended signature. + Only three scriptPubKey types supported: p2pkh, p2wpkh, p2sh-p2wpkh. + Note that signing multisig must be done outside this function, using + the wrapped library. + Returns: (signature, "signing succeeded") + or: (None, errormsg) in case of failure """ - pub = privkey_to_pubkey(priv) - # Convert the input tx and script to hex so that the deserialize() - # call creates hex-encoded fields - script = binascii.hexlify(pubkey_to_p2wpkh_script(pub)).decode('ascii') - scriptCode = binascii.hexlify(pubkey_to_p2pkh_script(pub)).decode('ascii') - if isinstance(tx, bytes): - tx = binascii.hexlify(tx).decode('ascii') - signing_tx = segwit_signature_form(deserialize(tx), i, scriptCode, amount, - hashcode=hashcode) - sig = ecdsa_tx_sign(signing_tx, priv, hashcode, usenonce=usenonce) - txobj = deserialize(tx) - if not native: - txobj["ins"][i]["script"] = "16" + script - else: - txobj["ins"][i]["script"] = "" - txobj["ins"][i]["txinwitness"] = [sig, pub] - return serialize(txobj) - -def signall(tx, priv): - # if priv is a dictionary, assume format is - # { 'txinhash:txinidx' : privkey } - if isinstance(priv, dict): - for e, i in enumerate(deserialize(tx)["ins"]): - k = priv["%s:%d" % (i["outpoint"]["hash"], i["outpoint"]["index"])] - tx = sign(tx, e, k) - else: - for i in range(len(deserialize(tx)["ins"])): - tx = sign(tx, i, priv) - return tx + # script verification flags + flags = set() -def get_p2sh_signature(tx, i, redeem_script, pk, amount=None, hashcode=SIGHASH_ALL): - """ - Tx is assumed to be serialized. redeem_script is for example the - output of mk_multisig_script. - pk is the private key, and must be passed in hex. - If amount is not None, the output of get_p2wsh_signature is returned. - What is returned is a single signature. - """ - if isinstance(tx, str): - tx = binascii.unhexlify(tx) - if isinstance(redeem_script, str): - redeem_script = binascii.unhexlify(redeem_script) - if amount: - return get_p2wsh_signature(tx, i, redeem_script, pk, amount, hashcode) - modtx = signature_form(tx, i, redeem_script, hashcode) - return ecdsa_tx_sign(modtx, pk, hashcode) + def return_err(e): + return None, "Error in signing: " + repr(e) -def get_p2wsh_signature(tx, i, redeem_script, pk, amount, hashcode=SIGHASH_ALL): - """ See note to multisign for the value to pass in as `script`. - Tx is assumed to be serialized. - """ - modtx = segwit_signature_form(deserialize(tx), i, redeem_script, amount, - hashcode, decoder_func=lambda x: x) - return ecdsa_tx_sign(modtx, pk, hashcode) + assert isinstance(tx, CMutableTransaction) + # using direct local access to libsecp256k1 binding, because + # python-bitcoinlib uses OpenSSL key management: + pub = privkey_to_pubkey(priv) -def apply_p2wsh_multisignatures(tx, i, script, sigs): - """Sigs must be passed in as a list, and must be a - complete list for this multisig, in the same order - as the list of pubkeys when creating the scriptPubKey. - """ - if isinstance(script, str): - script = binascii.unhexlify(script) - sigs = [binascii.unhexlify(x) if x[:2] == '30' else x for x in sigs] - if isinstance(tx, str): - return safe_hexlify(apply_p2wsh_multisignatures( - binascii.unhexlify(tx), i, script, sigs)) - txobj = deserialize(tx) - txobj["ins"][i]["script"] = "" - txobj["ins"][i]["txinwitness"] = [None] + sigs + [script] - return serialize(txobj) + if not amount: + # p2pkh only supported here: + input_scriptPubKey = pubkey_to_p2pkh_script(pub) + sighash = SignatureHash(input_scriptPubKey, tx, i, hashcode) + try: + sig = ecdsa_raw_sign(sighash, priv, rawmsg=True) + bytes([hashcode]) + except Exception as e: + return return_err(e) + tx.vin[i].scriptSig = CScript([sig, pub]) + # Verify the signature worked. + try: + VerifyScript(tx.vin[i].scriptSig, + input_scriptPubKey, tx, i, flags=flags) + except Exception as e: + return return_err(e) + return sig, "signing succeeded" -def apply_multisignatures(*args): - # tx,i,script,sigs OR tx,i,script,sig1,sig2...,sig[n] - tx, i, script = args[0], int(args[1]), args[2] - sigs = args[3] if isinstance(args[3], list) else list(args[3:]) + else: + # segwit case; we currently support p2wpkh native or under p2sh. + + # see line 1256 of bitcointx.core.scripteval.py: + flags.add(SCRIPT_VERIFY_P2SH) + + input_scriptPubKey = pubkey_to_p2wpkh_script(pub) + # only created for convenience access to scriptCode: + input_address = P2WPKHBitcoinAddress.from_scriptPubKey(input_scriptPubKey) + # function name is misleading here; redeemScript only applies to p2sh. + scriptCode = input_address.to_redeemScript() + + sighash = SignatureHash(scriptCode, tx, i, hashcode, amount=amount, + sigversion=SIGVERSION_WITNESS_V0) + try: + sig = ecdsa_raw_sign(sighash, priv, rawmsg=True) + bytes([hashcode]) + except Exception as e: + return return_err(e) + if native: + flags.add(SCRIPT_VERIFY_WITNESS) + else: + tx.vin[i].scriptSig = CScript([input_scriptPubKey]) - if isinstance(script, str): - script = binascii.unhexlify(script) - sigs = [binascii.unhexlify(x) if x[:2] == '30' else x for x in sigs] - if isinstance(tx, str): - return safe_hexlify(apply_multisignatures( - binascii.unhexlify(tx), i, script, sigs)) + witness = [sig, pub] + ctxwitness = CTxInWitness(CScriptWitness(witness)) + tx.wit.vtxinwit[i] = ctxwitness + # Verify the signature worked. + try: + VerifyScript(tx.vin[i].scriptSig, input_scriptPubKey, tx, i, + flags=flags, amount=amount, witness=tx.wit.vtxinwit[i].scriptWitness) + except ValidationError as e: + return return_err(e) - txobj = deserialize(tx) - txobj["ins"][i]["script"] = serialize_script([None] + sigs + [script]) - return serialize(txobj) + return sig, "signing succeeded" def apply_freeze_signature(tx, i, redeem_script, sig): if isinstance(redeem_script, str): @@ -863,14 +213,16 @@ def apply_freeze_signature(tx, i, redeem_script, sig): return serialize(txobj) def mktx(ins, outs, version=1, locktime=0): - """ Given a list of input dicts with key "output" - which are txid:n strings in hex, and a list of outputs - which are dicts with keys "address", "value", outputs - a hex serialized tranasction encoding this data. + """ Given a list of input tuples (txid(bytes), n(int)), + and a list of outputs which are dicts with + keys "address" (value should be *str* not CCoinAddress), + "value" (value should be integer satoshis), outputs a + CMutableTransaction object. Tx version and locktime are optionally set, for non-default locktimes, inputs are given nSequence as per below comment. """ - txobj = {"locktime": locktime, "version": version, "ins": [], "outs": []} + vin = [] + vout = [] # This does NOT trigger rbf and mimics Core's standard behaviour as of # Jan 2019. # Tx creators wishing to use rbf will need to set it explicitly outside @@ -880,48 +232,36 @@ def mktx(ins, outs, version=1, locktime=0): else: sequence = 0xffffffff for i in ins: - if isinstance(i, dict) and "outpoint" in i: - txobj["ins"].append(i) - else: - if isinstance(i, dict) and "output" in i: - i = i["output"] - txobj["ins"].append({ - "outpoint": {"hash": i[:64], - "index": int(i[65:])}, - "script": "", - "sequence": sequence - }) + outpoint = CMutableOutPoint((i[0][::-1]), i[1]) + inp = CMutableTxIn(prevout=outpoint, nSequence=sequence) + vin.append(inp) for o in outs: - if isinstance(o, str): - addr = o[:o.find(':')] - val = int(o[o.find(':') + 1:]) - o = {} - if re.match('^[0-9a-fA-F]*$', addr): - o["script"] = addr - else: - o["address"] = addr - o["value"] = val - - outobj = {} - if "address" in o: - outobj["script"] = address_to_script(o["address"]) - elif "script" in o: - outobj["script"] = o["script"] - else: - raise Exception("Could not find 'address' or 'script' in output.") - outobj["value"] = o["value"] - txobj["outs"].append(outobj) - return serialize(txobj) - -def make_shuffled_tx(ins, outs, deser=True, version=1, locktime=0): - """ Simple utility to ensure transaction + # note the to_scriptPubKey method is only available for standard + # address types + out = CMutableTxOut(o["value"], + CCoinAddress(o["address"]).to_scriptPubKey()) + vout.append(out) + return CMutableTransaction(vin, vout, nLockTime=locktime) + +def make_shuffled_tx(ins, outs, version=1, locktime=0): + """ Simple wrapper to ensure transaction inputs and outputs are randomly ordered. Can possibly be replaced by BIP69 in future """ random.shuffle(ins) random.shuffle(outs) - tx = mktx(ins, outs, version=version, locktime=locktime) - if deser: - return deserialize(tx) - else: - return tx + return mktx(ins, outs, version=version, locktime=locktime) + +def verify_tx_input(tx, i, scriptSig, scriptPubKey, amount=None, + witness=None, native=False): + flags = set() + if witness: + flags.add(SCRIPT_VERIFY_P2SH) + if native: + flags.add(SCRIPT_VERIFY_WITNESS) + try: + VerifyScript(scriptSig, scriptPubKey, tx, i, + flags=flags, amount=amount, witness=witness) + except ValidationError as e: + return False + return True diff --git a/jmbitcoin/setup.py b/jmbitcoin/setup.py index 5409382..018025f 100644 --- a/jmbitcoin/setup.py +++ b/jmbitcoin/setup.py @@ -9,6 +9,6 @@ setup(name='joinmarketbitcoin', author_email='', license='GPL', packages=['jmbitcoin'], - install_requires=['future', 'coincurve', 'urldecode'], + install_requires=['future', 'coincurve', 'python-bitcointx', 'urldecode'], python_requires='>=3.5', zip_safe=False) diff --git a/jmbitcoin/test/test_addresses.py b/jmbitcoin/test/test_addresses.py deleted file mode 100644 index d7723b1..0000000 --- a/jmbitcoin/test/test_addresses.py +++ /dev/null @@ -1,56 +0,0 @@ -import jmbitcoin as btc -import json -import pytest -import os -testdir = os.path.dirname(os.path.realpath(__file__)) - -def validate_address(addr, nettype): - """A mock of jmclient.validate_address - """ - BTC_P2PK_VBYTE = {"mainnet": b'\x00', "testnet": b'\x6f'} - BTC_P2SH_VBYTE = {"mainnet": b'\x05', "testnet": b'\xc4'} - try: - ver = btc.get_version_byte(addr) - except AssertionError as e: - return False, 'Checksum wrong. Typo in address?' - except Exception as e: - return False, "Invalid bitcoin address" - if ver not in [BTC_P2PK_VBYTE[nettype], BTC_P2SH_VBYTE[nettype]]: - return False, 'Wrong address version. Testnet/mainnet confused?' - if len(btc.b58check_to_bin(addr)) != 20: - return False, "Address has correct checksum but wrong length." - return True, 'address validated' - -@pytest.mark.parametrize( - "net", - [ - # 1 - ("mainnet"), - # 2 - ("testnet") - ]) -def test_b58_invalid_addresses(net): - #none of these are valid as any kind of key or address - with open(os.path.join(testdir,"base58_keys_invalid.json"), "r") as f: - json_data = f.read() - invalid_key_list = json.loads(json_data) - for k in invalid_key_list: - bad_key = k[0] - res, message = validate_address(bad_key, nettype=net) - assert res == False, "Incorrectly validated address: " + bad_key + " with message: " + message - -def test_b58_valid_addresses(): - with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f: - json_data = f.read() - valid_keys_list = json.loads(json_data) - for a in valid_keys_list: - addr, pubkey, prop_dict = a - if not prop_dict["isPrivkey"]: - if prop_dict["isTestnet"]: - net = "testnet" - else: - net = "mainnet" - #if using pytest -s ; sanity check to see what's actually being tested - res, message = validate_address(addr, net) - assert res == True, "Incorrectly failed to validate address: " + addr + " with message: " + message - diff --git a/jmbitcoin/test/test_bech32.py b/jmbitcoin/test/test_bech32.py deleted file mode 100644 index ec5de0d..0000000 --- a/jmbitcoin/test/test_bech32.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2017 Pieter Wuille -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - - -"""Reference tests for segwit adresses""" - -import binascii -import unittest -import jmbitcoin as btc - -VALID_CHECKSUM = [ - "A12UEL5L", - "an83characterlonghumanreadablepartthatcontainsthenumber1andtheexcludedcharactersbio1tt5tgs", - "abcdef1qpzry9x8gf2tvdw0s3jn54khce6mua7lmqqqxw", - "11qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqc8247j", - "split1checkupstagehandshakeupstreamerranterredcaperred2y9e3w", -] - -INVALID_CHECKSUM = [ - " 1nwldj5", - "\x7F" + "1axkwrx", - "an84characterslonghumanreadablepartthatcontainsthenumber1andtheexcludedcharactersbio1569pvx", - "pzry9x0s0muk", - "1pzry9x0s0muk", - "x1b4n0q5v", - "li1dgmt3", - "de1lg7wt\xff", -] - -VALID_ADDRESS = [ - ["BC1QW508D6QEJXTDG4Y5R3ZARVARY0C5XW7KV8F3T4", "0014751e76e8199196d454941c45d1b3a323f1433bd6"], - ["tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sl5k7", - "00201863143c14c5166804bd19203356da136c985678cd4d27a1b8c6329604903262"], - ["bc1pw508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7k7grplx", - "5128751e76e8199196d454941c45d1b3a323f1433bd6751e76e8199196d454941c45d1b3a323f1433bd6"], - ["BC1SW50QA3JX3S", "6002751e"], - ["bc1zw508d6qejxtdg4y5r3zarvaryvg6kdaj", "5210751e76e8199196d454941c45d1b3a323"], - ["tb1qqqqqp399et2xygdj5xreqhjjvcmzhxw4aywxecjdzew6hylgvsesrxh6hy", - "0020000000c4a5cad46221b2a187905e5266362b99d5e91c6ce24d165dab93e86433"], -] - -INVALID_ADDRESS = [ - "tc1qw508d6qejxtdg4y5r3zarvary0c5xw7kg3g4ty", - "bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t5", - "BC13W508D6QEJXTDG4Y5R3ZARVARY0C5XW7KN40WF2", - "bc1rw5uspcuh", - "bc10w508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7kw5rljs90", - "BC1QR508D6QEJXTDG4Y5R3ZARVARYV98GJ9P", - "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sL5k7", - "bc1zw508d6qejxtdg4y5r3zarvaryvqyzf3du", - "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3pjxtptv", - "bc1gmk9yu", - -] - -INVALID_ADDRESS_ENC = [ - ("BC", 0, 20), - ("bc", 0, 21), - ("bc", 17, 32), - ("bc", 1, 1), - ("bc", 16, 41), -] - -class TestSegwitAddress(unittest.TestCase): - """Unit test class for segwit addressess.""" - - def test_valid_checksum(self): - """Test checksum creation and validation.""" - for test in VALID_CHECKSUM: - hrp, _ = btc.bech32_decode(test) - self.assertIsNotNone(hrp) - pos = test.rfind('1') - test = test[:pos+1] + chr(ord(test[pos + 1]) ^ 1) + test[pos+2:] - hrp, _ = btc.bech32_decode(test) - self.assertIsNone(hrp) - - def test_invalid_checksum(self): - """Test validation of invalid checksums.""" - for test in INVALID_CHECKSUM: - hrp, _ = btc.bech32_decode(test) - self.assertIsNone(hrp) - - def test_valid_address(self): - """Test whether valid addresses decode to the correct output.""" - for (address, hexscript) in VALID_ADDRESS: - hrp = "bc" - witver, witprog = btc.bech32addr_decode(hrp, address) - if witver is None: - hrp = "tb" - witver, witprog = btc.bech32addr_decode(hrp, address) - self.assertIsNotNone(witver) - scriptpubkey = btc.segwit_scriptpubkey(witver, witprog) - self.assertEqual(scriptpubkey, binascii.unhexlify(hexscript)) - addr = btc.bech32addr_encode(hrp, witver, witprog) - self.assertEqual(address.lower(), addr) - - def test_invalid_address(self): - """Test whether invalid addresses fail to decode.""" - for test in INVALID_ADDRESS: - witver, _ = btc.bech32addr_decode("bc", test) - self.assertIsNone(witver) - witver, _ = btc.bech32addr_decode("tb", test) - self.assertIsNone(witver) - - def test_invalid_address_enc(self): - """Test whether address encoding fails on invalid input.""" - for hrp, version, length in INVALID_ADDRESS_ENC: - code = btc.bech32addr_encode(hrp, version, [0] * length) - self.assertIsNone(code) - -if __name__ == "__main__": - unittest.main() diff --git a/jmbitcoin/test/test_bip32.py b/jmbitcoin/test/test_bip32.py index d0eb5b1..f8b6c53 100644 --- a/jmbitcoin/test/test_bip32.py +++ b/jmbitcoin/test/test_bip32.py @@ -103,9 +103,9 @@ def test_ckd_pubkeys(): def test_bip32_descend(): master = btc.bip32_master_key(b'\x07'*32) end_key = btc.bip32_descend(master, [2, 3, 10000]) - assert end_key=="6856ef965940a1a7b1311dc041050ac0013e326c7ff4e2c677a7694b4f0405c901" + assert end_key==binascii.unhexlify("6856ef965940a1a7b1311dc041050ac0013e326c7ff4e2c677a7694b4f0405c901") end_key = btc.bip32_descend(master, 2, 5, 4, 5) - assert end_key=="d2d816b6485103c0d7ff95482788f0e8e73fa11817079e006d47979d8196c4b101" + assert end_key==binascii.unhexlify("d2d816b6485103c0d7ff95482788f0e8e73fa11817079e006d47979d8196c4b101") diff --git a/jmbitcoin/test/test_btc_formatting.py b/jmbitcoin/test/test_btc_formatting.py deleted file mode 100644 index 66475d5..0000000 --- a/jmbitcoin/test/test_btc_formatting.py +++ /dev/null @@ -1,57 +0,0 @@ -#! /usr/bin/env python -'''Test bitcoin module data handling''' - -import jmbitcoin as btc -import pytest -import binascii - -#used in p2sh addresses -def test_hash160(): - assert '0e3397b4abc7a382b3ea2365883c3c7ca5f07600' == \ - btc.hash160('The quick brown fox jumps over the lazy dog') - -def test_bad_code_string(): - for i in [1,9,257,-3,"256"]: - with pytest.raises(ValueError) as e_info: - btc.get_code_string(i) - -#Tests of compactsize encoding, see: -#https://bitcoin.org/en/developer-reference#compactsize-unsigned-integers -#note that little endian is used. -@pytest.mark.parametrize( - "num, compactsize", - [ - (252, "fc"), - (253, "fdfd00"), - (254, "fdfe00"), - (515, "fd0302"), - (65535, "fdffff"), - (65536, "fe00000100"), - (65537, "fe01000100"), - (4294967295, "feffffffff"), - (4294967296, "ff0000000001000000"), - - ]) -def test_compact_size(num, compactsize): - assert btc.num_to_var_int(num) == binascii.unhexlify(compactsize) - -@pytest.mark.parametrize("frm, to", [ - (("16001405481b7f1d90c5a167a15b00e8af76eb6984ea59"), - ["001405481b7f1d90c5a167a15b00e8af76eb6984ea59"]), - (("483045022100ad0dda327945e581a5effd83d75d76a9f07c3128f4dc6d25a54" - "e5ad5dd629bd00220487a992959bd540dbc335c655e6485ebfb394129eb48038f" - "0a2d319782f7cb690121039319452b6abafb5fcf06096196d0c141b8bd18a3de7" - "e9b9352da800d671ccd84"), - ["3045022100ad0dda327945e581a5effd83d75d76a9f07c3128f4dc6d25a54e5ad5dd629bd00220487a992959bd540dbc335c655e6485ebfb394129eb48038f0a2d319782f7cb6901", - "039319452b6abafb5fcf06096196d0c141b8bd18a3de7e9b9352da800d671ccd84"]), - (("51"), [1]), - (("00"), [None]), - (("510000"), [1, None, None]), - (("636505aaaaaaaaaa53"), [99, 101, "aaaaaaaaaa", 3]), - (("51" + "4d0101" + "aa"*257), [1, "aa"*257]), - (("4e" + "03000100" + "aa"*65539), ["aa"*65539]), -]) -def test_deserialize_script(frm, to): - #print(len(btc.deserialize_script(frm)[0])) - assert btc.deserialize_script(frm) == to - assert btc.serialize_script(to) == frm diff --git a/jmbitcoin/test/test_ecc_signing.py b/jmbitcoin/test/test_ecc_signing.py index 570a292..e3e9a73 100644 --- a/jmbitcoin/test/test_ecc_signing.py +++ b/jmbitcoin/test/test_ecc_signing.py @@ -12,25 +12,23 @@ vectors = None def test_valid_sigs(setup_ecc): for v in vectors['vectors']: - msg = v['msg'] - sig = v['sig'] - priv = v['privkey'] - assert sig == btc.ecdsa_raw_sign(msg, priv, True, rawmsg=True)+'01' - #check that the signature verifies against the key(pair) - pubkey = btc.privtopub(priv) - assert btc.ecdsa_raw_verify(msg, pubkey, sig[:-2], True, rawmsg=True) - #check that it fails to verify against corrupted signatures + msg, sig, priv = (binascii.unhexlify( + v[a]) for a in ["msg", "sig", "privkey"]) + assert sig == btc.ecdsa_raw_sign(msg, priv, rawmsg=True)+ b'\x01' + # check that the signature verifies against the key(pair) + pubkey = btc.privkey_to_pubkey(priv) + assert btc.ecdsa_raw_verify(msg, pubkey, sig[:-1], rawmsg=True) + # check that it fails to verify against corrupted signatures for i in [0,1,2,4,7,25,55]: - #corrupt one byte - binsig = binascii.unhexlify(sig) - checksig = binascii.hexlify(binsig[:i] + btc.from_string_to_bytes(chr( - (ord(binsig[i:i+1])+1) %256)) + binsig[i+1:-1]).decode('ascii') + # corrupt one byte + checksig = sig[:i] + chr( + (ord(sig[i:i+1])+1) %256).encode() + sig[i+1:-1] - #this kind of corruption will sometimes lead to an assert - #failure (if the DER format is corrupted) and sometimes lead - #to a signature verification failure. + # this kind of corruption will sometimes lead to an assert + # failure (if the DER format is corrupted) and sometimes lead + # to a signature verification failure. try: - res = btc.ecdsa_raw_verify(msg, pubkey, checksig, True, rawmsg=True) + res = btc.ecdsa_raw_verify(msg, pubkey, checksig, rawmsg=True) except: continue assert res==False diff --git a/jmbitcoin/test/test_main.py b/jmbitcoin/test/test_main.py deleted file mode 100644 index 5f2bf10..0000000 --- a/jmbitcoin/test/test_main.py +++ /dev/null @@ -1,61 +0,0 @@ -#! /usr/bin/env python -'''Testing mostly exceptional cases in secp256k1_main. - Some of these may represent code that should be removed, TODO.''' - -import jmbitcoin as btc -import binascii -import pytest -import os -testdir = os.path.dirname(os.path.realpath(__file__)) - -def test_hex2b58check(): - assert btc.hex_to_b58check("aa"*32) == "12JAT9y2EcnV6DPUGikLJYjWwk5UmUEFXRiQVmTbfSLbL3njFzp" - -def test_bindblsha(): - assert btc.bin_dbl_sha256("abc") == binascii.unhexlify( - "4f8b42c22dd3729b519ba6f68d2da7cc5b2d606d05daed5ad5128cc03e6c6358") - -def test_lpad(): - assert btc.lpad("aaaa", "b", 5) == "baaaa" - assert btc.lpad("aaaa", "b", 4) == "aaaa" - assert btc.lpad("aaaa", "b", 3) == "aaaa" - -def test_safe_from_hex(): - assert btc.safe_from_hex('ff0100') == b'\xff\x01\x00' - -def test_hash2int(): - assert btc.hash_to_int("aa"*32) == \ - 77194726158210796949047323339125271902179989777093709359638389338608753093290 - -@btc.hexbin -def dummyforwrap(a, b, c, d="foo", e="bar"): - newa = a+b"\x01" - x, y = b - newb = [x+b"\x02", y+b"\x03"] - if d == "foo": - return newb[1] - else: - return newb[0] - -def test_hexbin(): - assert dummyforwrap("aa", ["bb", "cc"], True) == "cc03" - assert dummyforwrap("aa", ["bb", "cc"], True, d="baz") == "bb02" - assert dummyforwrap(b"\xaa", [b"\xbb", b"\xcc"], False) == b"\xcc\x03" - -def test_add_privkeys(): - with pytest.raises(Exception) as e_info: - btc.add_privkeys("aa"*32, "bb"*32+"01", True) - -def test_ecdsa_raw_sign(): - msg = "aa"*31 - with pytest.raises(Exception) as e_info: - btc.ecdsa_raw_sign(msg, None, None, rawmsg=True) - assert e_info.match("Invalid hash input") - #build non-raw priv object as input - privraw = "aa"*32 - msghash = b"\xbb"*32 - sig = binascii.hexlify(btc.ecdsa_raw_sign(msghash, privraw, False, rawpriv=False, rawmsg=True)).decode('ascii') - assert sig == "3045022100b81960b4969b423199dea555f562a66b7f49dea5836a0168361f1a5f8a3c8298022003eea7d7ee4462e3e9d6d59220f950564caeb77f7b1cdb42af3c83b013ff3b2f" - - - \ No newline at end of file diff --git a/jmbitcoin/test/test_tx_serialize.py b/jmbitcoin/test/test_tx_serialize.py deleted file mode 100644 index 225ee5d..0000000 --- a/jmbitcoin/test/test_tx_serialize.py +++ /dev/null @@ -1,234 +0,0 @@ -import jmbitcoin as btc -import pytest -import json -import os -testdir = os.path.dirname(os.path.realpath(__file__)) - -#TODO: fold these examples into the tx_valid.json file -@pytest.mark.parametrize( - "tx_type, tx_id, tx_hex", - [("simple-tx", - "f31916a1d398a4ec18d56a311c942bb6db934cee6aa8ac30af0b30aad9efb841", - "0100000001c74265f31fc5e24895fdc83f7157cc40045235f3a71ae326a219de9de873" + - "0d8b010000006a473044022076055917470b7ec4f4bb008096266cf816ebb089ad983e" + - "6a0f63340ba0e6a6cb022059ec938b996a75db10504e46830e13d399f28191b9832bd5" + - "f61df097b9e0d47801210291941334a00959af4aa5757abf81d2a7d1aca8adb3431c67" + - "e89419271ba71cb4feffffff023cdeeb03000000001976a914a2426748f14eba44b3f6" + - "abba3e8bce216ea233f388acf4ebf303000000001976a914bfa366464a464005ba0df8" + - "6024a6c3ed859f03ac88ac33280600"), - ("3partycoinjoin", - "d91278e125673f5b9201456b0c36efac3b2b6700fdd04fc2227352a63f941170", - "010000000cedf1756d1d679de268a533c2d009f1d06ba8918908369c5e3b818678603a" + - "4bcb030000006b483045022100a775815dcfaf706ee9b6e6015f162d20d227eb44cf88" + - "93fb3ac95f910058fbd50220500f1f3de75b3bdf7f6a0949528f836c694a5bd1effedb" + - "b02156bcb292a78e5d012102a726879b9d663467ab71726ff5469c8b0a4213a93643f6" + - "ed739fdbb1e81ca5b9ffffffff3e0ebec9b42f30657ef187bd263c57849541014700cc" + - "5480268dc63ce6997552070000006b4830450221008ec354dcb5d661a60651e988356a" + - "215ccaaa688be1e78861b3b89f35e0c1bb7b02203d2117791b74ad2aa594e4dc806d03" + - "ec2a1241c4955b013dc365b6eacc4ee6c9012102632c8b13d9b94f2226c28295b52686" + - "4bbece7ea283a90ca001e498e2a08dbebcffffffff841e4ed900a6592687d7684ec0dd" + - "0893d7a3b86df07bb2626a1f51ae74ee0ba6010000006a473044022017f96effaf9812" + - "0745323859e1b52699baf0993675d028882e42310deac6921102200daaab3008e9bce1" + - "2a3e136a7e0472a12da31121237ac382a6a64e4ab5ed2e5201210365a1491d9c10f866" + - "0a02f47a54e5c0285f186a9f5b4cb75405548cef3427c4eeffffffff7e131188b66001" + - "c0f5a4b45bd3503d2ce17f40e43e3803a7f30c20a8214abc95080000006a4730440220" + - "6c21a02e4bfcce3f9a5775ce7b9033bebb354b15167dec1a1b3f0c7255633234022071" + - "994111c455b1ddd4cd093d0d4f2ecbbf1283a0a1f982962127768e3c4e251301210394" + - "1f5e1834e8b1c1503f2d20b3f3e06db2b80de47f7f31b4cb39f75525753822ffffffff" + - "955544430628ea06493caa541d59374d7708dbef0e59a6653ac025f341c888c8040000" + - "006a473044022022c2afb23a7d401cdfff735ba390fac8dd2cbde2dcce4c3a8f988dfd" + - "a08b956c02204ba4ca6dc72993da6bf3d020530f01d7fc931ef28c78c2fa9b70657e50" + - "8873c801210240dc515cf540c43575615f18670d3f511f6f47b719650be60183ad9f61" + - "f805cdffffffffedf1756d1d679de268a533c2d009f1d06ba8918908369c5e3b818678" + - "603a4bcb010000006b483045022100ee3b322aab4c77debc6ebf48967f7677c03cd5fb" + - "df775885691f1162cca1f04f0220031a8c4017618562f4a2f4f0ece98bf13063c9e03b" + - "720b2eb3ead0ef024bee1f012103ce920bc0f19adf76e0f8b36eb62873d3cba991ee3e" + - "6504e0dbf114a0d5865827ffffffffa8b4085018a3e8848531eecf3164ec2b89ed5ed2" + - "172e22a19e3343f29faa3c03050000006a47304402202ff247fb0ec1eb9fa24903957d" + - "8973eb9a3a8e2dc4d6e6dbbb366c7563b49e680220301a0df3f1dbff1f939f14b3bf78" + - "687d8a2da26dafcb310ade85efd9cd37bd050121038cfc01875ccaacd4863515259e52" + - "b96188e83ca30358bce4111c4021bfc66343fffffffff5e7b4d41b53a1fce37b349d02" + - "33aed80be2b5da0ff45c59014ed5e4ff09b0a5030000006a473044022015035665d467" + - "19031a1e09fffb92484365a38f8c51171046bedc0eea1385cb7402200a296a8315f9a6" + - "3fe9a7020d4fa3be20a2c0587f98ae0d8d0560985d9cfb0748012103d9537b8ccb7206" + - "383edff317fcb1e4188036a18a84a6727104dd9b4d00638037ffffffffedf1756d1d67" + - "9de268a533c2d009f1d06ba8918908369c5e3b818678603a4bcb000000006a47304402" + - "200742b54bb0d8e62dbb87eaa00823152ff46106824d791bcb538e47b9f52e9df90220" + - "705de19cdc96c9dcb9f8dfff070e780c129524a56f369dc3633f69b1472c41d2012102" + - "90428148e18a34190de8a0b9a7ab7e9353feac7bd7bb57dde4acb2a03d1dfb94ffffff" + - "ff841e4ed900a6592687d7684ec0dd0893d7a3b86df07bb2626a1f51ae74ee0ba60c00" + - "00006a4730440220719b2baf0422e98d7462d120c3406e7e3657cceb8fabe84ddece55" + - "d34affb4620220168f87c8bcd57f903104b5effc0feffe3bd776dce6c09bfda8a3a36c" + - "95788ec701210263c387f7b141cf649a5296096976b6b622cd44a611c0e485724f9432" + - "9cf22b79ffffffff3e0ebec9b42f30657ef187bd263c57849541014700cc5480268dc6" + - "3ce6997552010000006a47304402206e62849a47494e9294bd7e47d87b9d32a6b69097" + - "067184b551350b7b34291d46022017ca461f6bcceb97432e9890700a68a4df4f4c5cfd" + - "272926af00a05dfdb3c07b0121024decad000f3aabfdac5ea8f132e457f0056b1f2cab" + - "0c946884d2ff0afbdd68b8ffffffff7e131188b66001c0f5a4b45bd3503d2ce17f40e4" + - "3e3803a7f30c20a8214abc95090000006b483045022100fa9c274892db835066eeac6c" + - "8f25d9507b43c93f4557b16560745e627387d7780220461025024a97816ff0567dcdca" + - "4702b5238894ee7028d04af48a7eb11bd3520301210391c94422cd25ee416178827513" + - "365ee49e4b2f17cdf0b07aed04353c10e34f6dffffffff06008c8647000000001976a9" + - "14e924c7d01f0201df5b1465aed50012c4dffa0ff988ac89ac7800000000001976a914" + - "3dc5ae7471acbba55107b651ce20177b3b44721388ac9f258509000000001976a91400" + - "656d12c4b48359f8637bcf96f9f02ceb2fad1c88ac4de9ec04000000001976a914f7de" + - "454e8b402cbde90f4a325de69236040f144088ac008c8647000000001976a914b7d9f8" + - "29f1bd6919728f91f8a10ce0ca87bde1aa88ac008c8647000000001976a914b877256a" + - "588a315e30b2c5aec38ff70cea71dd0c88ac00000000"), - ("4partycoinjoin", - "55eac9d4a4159d4ba355122c6c18f85293c19ae358306a3773ec3a5d053e2f1b", - "01000000108048bbbc26c394d45b514835d998ae2679c31cecd7623041372703f469aa" + - "ff8e040000006b4830450220293480c5975676ddbb8b66fc6e4bc53668ae4cc59d0da5" + - "377e520f3f360ad15e022100fe6aaf3d890570d66728210afb13e221f0405bbbf43262" + - "16371459877531a2d80121033392a42f2a8e4e93a5d7b50b4d26eabda42e04e110e77e" + - "22b514982a832dde92ffffffff587af588bbe3f911f8234a858cdc62a1c04c12cf09f5" + - "95f92a7292fd3f54d17e080000006b483045022100c47a07c3dd537e1e11216dcf3440" + - "619ec80a88d25f91f8b9de74b348947ba47902205e5a335e2c75b202b75ff8a50a2560" + - "f49542fe4e96598be6fe77b00bc7c33484012103e21d83d80666a020c3e4657f7155ac" + - "4572edf9fc79d59b5b69e27f23ff5f8df5ffffffff8048bbbc26c394d45b514835d998" + - "ae2679c31cecd7623041372703f469aaff8e0b0000006b48304502202a5403bfed1826" + - "c0d5b4143229518d6c0dc6b0881ad04e26039cef4981aa9487022100cecad7bd8fd476" + - "2257f83080bf2f56249bb36554137e9af9c1588f8f2ed170a9012103fd52e4f1e64cdd" + - "003528cf5f388ad455d001ea691bd02125d694117a1d481bbcffffffff8048bbbc26c3" + - "94d45b514835d998ae2679c31cecd7623041372703f469aaff8e110000006b48304502" + - "204e024b69b4a78ea9daa63ad1d8dba5e8729fbb61749dacc49be20053db8c02710221" + - "00c24555b43e6e838fd7cb797cbf96c278531af29493aaf719bdfe3354941d0b320121" + - "02a322a78564a976271153b0e289d631e5c6a217542d540c94262c40a83e48281affff" + - "ffff8048bbbc26c394d45b514835d998ae2679c31cecd7623041372703f469aaff8e05" + - "0000006b483045022100d3c7d326e233b2238576aecf53d5c3dfb88a3b41d815ff2cfd" + - "c3e2c5664aa52102204f0dc1cd110e4317ab435533a4440bbdabee75f360551bb46206" + - "79043058288801210310950ae84cfb02e24960c7227e40dedbc8804397a19e277233b1" + - "f3ecc797fdc2ffffffff9dd9d7a38ab7f29f17c4f32ae5be1426f80ff58c158fc7c74a" + - "afb9d53659cd05050000006b483045022100cc46ea21d4a1622cf7507ab9c96863b5fa" + - "3744c8c4b34a74c7bfdb92af41f40702200def9598dfcae25abaac047969f231c565c0" + - "6df6f7cb8f3c588e390049b0217801210202d1c96a5b459ca4fd430afc4d1b0e468a2a" + - "ba84c48ab5f5ad906ec09f55cfb3ffffffff84dc724e49f71d2a82cc46d781b1f47a21" + - "18c4056ab6dad2a83f03c7ed4a08c6040000006b483045022100ee5db129b8264b8a12" + - "e85fbf29566ad7edb84704214ced4dbea3e628149f08af02201149ab3c4e0aa6fa48ec" + - "4816ecefda6105a6df5838e9a6dbe235d156898d3b40012102da5240065576c7567c3e" + - "cc8dc09f3fda64a0c6b89c3974a7506bdeb7fb440704ffffffff68e787d07a48d3ba00" + - "d8160311ff0533684dd66c7ddba8f7d74af3862d98bd8a020000006b483045022100a8" + - "0edec0b7a6946fbcb877f95c48e22e26710f3f7a7ec9e5dd0e7622832fca3002204a8f" + - "ecc644f674cfa8c5ba0682318f5b6cd8cc5f0c15b70e5e423e3e0888e6550121032708" + - "198b4c046db76f9e0ff7350d05243950738b39c431f55d85d5ec886a1389ffffffff21" + - "57aecaef0b44114b1035d6630a436ee3a6ecbeebfca48b5ce8fa6dbc5e100700000000" + - "6b483045022100df2085a721c026328ea6dec9307c45d5991d51f696c15896ed14d40c" + - "dc69a3d302205a598476fa84a37c5eb70bab35d4a414fce63bc8bedec7883a724ea1f7" + - "d4b98401210271f768f5b16806df11ee81619c1371864707de25b27df94bd4516b6680" + - "2f0709ffffffff7cabc836092178f7325571627777476d1796e446ac8039dd706754b6" + - "824ee723040000006a473044022036eedb5259b74f8a99cba9f4866fafce94b5b7ebf7" + - "d248e973bdba27ac1161b202203d58951892c1680fafc77dd6ee1a9c27754018b96030" + - "a51c8891c1f5273c077f01210271f768f5b16806df11ee81619c1371864707de25b27d" + - "f94bd4516b66802f0709ffffffff84dc724e49f71d2a82cc46d781b1f47a2118c4056a" + - "b6dad2a83f03c7ed4a08c6000000006a473044022014a918294f28157288ab1e66e22b" + - "fe5e5cb92dde076e133b5d1d807bfd252048022020f8ef9d0c094b476b31f177fb083c" + - "b6e65f5167a086fd25bebe5012e46ae8b601210271f768f5b16806df11ee81619c1371" + - "864707de25b27df94bd4516b66802f0709ffffffff4c8670675a981c70935cbd0a0de4" + - "98c9b28c41784d590f1b9e59930e64dd53f5020000006a473044022067b08694963116" + - "f7d53522c2327b46150084b7b9cf5b279daa1461492dffd7010220453b1fd98cdc26fe" + - "d9385ec578da1dd11d1ca7367ebfd5586f4ddb913feabb6c01210397db984e478086ca" + - "f0bf90181c3c597905a52416e152bff007dffd8edf06e2deffffffff8048bbbc26c394" + - "d45b514835d998ae2679c31cecd7623041372703f469aaff8e070000006c4930460221" + - "0094e5123e70426ff6732bebbcaaac2d871b873880d77c71a97507a074fe1046000221" + - "00c67b7d7663817507dd03f00fd2b690d33daa645a13b6966ec1c76e5a8df0b0a40121" + - "035a9e5cdd37824ad390d73abf2c7bc902de6dd7e4ab92fcf4e84506747e8c6cdaffff" + - "ffff8048bbbc26c394d45b514835d998ae2679c31cecd7623041372703f469aaff8e01" + - "0000006b483045022100cfab7cb7b645e201515b756439ba943fecb77e68c76c482de5" + - "225bf7a80ca94a022061600e67eaf33b59c5863de617bdb79f63e69b310710cb2b7f4f" + - "63d9d64fb3bd012102dabd952ef6ebfe396dfd66436089f5df731dd8300ecb6d88eefe" + - "458bc73a4c8fffffffff11565b2030be339919eaa5d1c1b47f7d265485c2e230fe44d9" + - "5ce41f3f6d5bdc030000006b483045022021d350b215077c4b1aee83c747a0cabbe148" + - "2fc4ccc97657f07d481df3f89d45022100b16d629e35e857506af6e8dd74373f370f4b" + - "965f332a488878dc983f554378c40121022cde2859fb62f3b671a887fcce054626cdb2" + - "8ed4d20d0dfcf8e0b090e2d7fe4effffffff8048bbbc26c394d45b514835d998ae2679" + - "c31cecd7623041372703f469aaff8e0c0000006a47304402203adf6153783989ca7ec1" + - "fa12785e5abae5c6bd8a4e26bb0c2ae0cbc4635af981022056faa34a713bede42d0efe" + - "af84c35506f47d487bed59fba2de53167fdad64c570121025d3368bbb24f5980fc6081" + - "d2a1e0e04675d11751a53d13d896cff7b5940ac205ffffffff1cb00400000000000019" + - "76a9146418ea7a9cbcfbd9dfb4f552c00b834072ac833a88ac40420f00000000001976" + - "a91492f8428eae5228083b588b330f9dfc1cccd300fc88ac40420f00000000001976a9" + - "14871e08ec7dd94ef87fd7722ee703007badb7020d88ace8030000000000001976a914" + - "2ced0ea9992d7e5492b9053437702c83ea0750c088ac40420f00000000001976a91449" + - "464457f02609ba1d54153eaba614ab84d593cb88ac877fc30f000000001976a9141c20" + - "8af75037715bdca6c42f29e104af62204e4788ac540b0000000000001976a914cfccd5" + - "185a3c4ced7514fa584eee6ebd895c984888ac73037000000000001976a9149f0ddeb2" + - "473d013452bce71973d7515c0b4cd5cf88ac40420f00000000001976a9142cb4e342c4" + - "398a08397a8cedfd270e432e977da488ac09c2e005000000001976a914ca9ec96314a5" + - "f4842e3fe33a0cf6d047a4398d0a88ac40420f00000000001976a914cd0124b0969f16" + - "4132301032a0f99de5c8d8471e88ace9320400000000001976a9145c5f87677c7b50bc" + - "4fe88bdec9f2221a0ca3c35088ac40420f00000000001976a914fa40beb4e557e14d5e" + - "461c7aa66d674ea6b7ce5988ac40420f00000000001976a914564bdac0f31da4f35ce5" + - "3c9aaef7e796ed7acdcd88ace33c2b00000000001976a9142ef77132983b843b07396b" + - "731d213e3a60b4c6e488ac40420f00000000001976a9144aa2c6a81b4ad8c9414b1039" + - "d5b2c51d3416233888ac40420f00000000001976a914713667aa801d3d7cbf9c3d1667" + - "626131a9937a6288ac40420f00000000001976a9142fa24ea7272cbb3ee06beeb274b3" + - "486c3d83f60188ac40420f00000000001976a9147c2e0ee7484234b0a20fab40855dff" + - "93c3b195da88ac40420f00000000001976a914123d15de9b64aea49a9cf81d781fdfad" + - "00287acf88ac40420f00000000001976a91463f456082de5dbd63c7ab9cd511813d836" + - "a2a7cd88ac306a2e00000000001976a9144103b28ac18a4ad0ce075fe323ee3d38a95c" + - "4d2188acc0541d00000000001976a91484d1b7dadb4638895fc5eb79145dca39daea86" + - "ca88acb80b0000000000001976a9146baff4a7a65316aed38c032c6eded2e48111fb62" + - "88ac34080000000000001976a914a5e54586ea1626fc90359f55fa4a291c99cffae488" + - "accdc30100000000001976a914d547e4e5b7eb382f9309622bdf0271836083936988ac" + - "40420f00000000001976a914a0b17374b3085de303202b29439957ccd71ba13088acde" + - "af2600000000001976a914af4d27ced320ae73068d0e2bb2c3abfe6fbe2c6988ac0000" + - "0000"), - ("p2sh-in-to-p2sh-out", - "355a6090bfdb77d1d7fd7c67ba6a711b9f493344e372928fe79ca0206b34796b", - "0100000001b5bbd7adb1f9f6f599d8e547f7f789ba9b63da9f036e8b739477bb8295fd" + - "4a2600000000fdfd0000473044022062a3ca46f976d4719d94fa67c9360c65b0946e3a" + - "7877f7f2c4cc37b8c77dede302203af47e6c6ccd804726295eb99f90dc198c86414d85" + - "7e21bc617f26903d380b0901483045022100af39dc96cf32aed7165a7644dcdfa939cb" + - "7e6af8236126644be937ff8a10b3a10220204b03b1a6b8acc7748e3c76ee968f9e9997" + - "a81c6e944910d2dc626133aa82bf014c695221021de4261f30b149c3f14f93b1a06a5e" + - "96aa24780c789343f80c341043d46d700a21036daab402e66c56470eda26d981f80a1b" + - "8f224f21fdbe64329b0481b1b37d8a0b2103c72347824e9099dbdfad5a2dac56e5d056" + - "8c9c100caa7490f62f2731c23af27353aeffffffff028a7faf00000000001976a91472" + - "2a53477336bd4960bed86be7fc47b8ece07ba688acbda438010000000017a9148a6425" + - "ccc0cb64097c29579a40e733819d4e07c58700000000"), - ("op-return-output", - "5be05925b8ef04f326b19e2c57f9ce1f3e2024aa992dd0d744f7942e92a200a5", - "0100000001067c0bd822cdcfabf0978bdede3a7b395a5c175219fd704b1ee47f137858" + - "6362000000006b483045022100dffb89e46c734b6429b6322bce9b6343c98d0d9eed12" + - "059ac2b9dfcffaa5577702204350bc86aea9a7e3b2868a643d27c5a36bd3105c8a2a2c" + - "5daf6e8441c83c870f012102a1260627f8845765759454a2ee47603312bbd7fdad0fa0" + - "0f3ad0cea0c5602d2cffffffff03e9690700000000001976a91497d34e7a0c8082f180" + - "546040030afb925c0f2dd888ac0000000000000000166a146f6d6e6900000000000000" + - "0300000000000186a0aa0a0000000000001976a91488d924f51033b74a895863a5fb57" + - "fd545529df7d88ac00000000")]) -def test_serialization_roundtrip(tx_type, tx_id, tx_hex): - assert tx_hex == btc.serialize(btc.deserialize(tx_hex)) - -@pytest.mark.parametrize( - "ins, outs, txtype, valid", - [ - (4, 3, "p2pkh", True), - (4, 3, "p2sh", False), - ]) -def test_estimate_tx_size(ins, outs, txtype, valid): - #TODO: this function should throw on invalid number of ins or outs - if valid: - assert btc.estimate_tx_size(ins, outs, txtype)== 10 + 147*ins + 34*outs - else: - with pytest.raises(NotImplementedError) as e_info: - btc.estimate_tx_size(ins, outs, txtype) - - -def test_serialization_roundtrip2(): - #Data extracted from: - #https://github.com/bitcoin/bitcoin/blob/master/src/test/data/tx_valid.json - #These are a variety of rather strange edge case transactions, which are - #still valid. - #Note that of course this is only a serialization, not validity test, so - #only currently of very limited significance - with open(os.path.join(testdir,"tx_valid.json"), "r") as f: - json_data = f.read() - valid_txs = json.loads(json_data) - for j in valid_txs: - #ignore comment entries - if len(j) < 2: - continue - print(j) - deserialized = btc.deserialize(str(j[0])) - print(deserialized) - assert j[0] == btc.serialize(deserialized) diff --git a/jmbitcoin/test/test_tx_signing.py b/jmbitcoin/test/test_tx_signing.py new file mode 100644 index 0000000..4db7f3c --- /dev/null +++ b/jmbitcoin/test/test_tx_signing.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 + +import sys +import pytest + +import binascii +import hashlib +import jmbitcoin as btc + +@pytest.mark.parametrize( + "addrtype", + [("p2wpkh"), + ("p2sh-p2wpkh"), + ("p2pkh"), + ]) +def test_sign_standard_txs(addrtype): + # liberally copied from python-bitcoinlib tests, + # in particular see: + # https://github.com/petertodd/python-bitcoinlib/pull/227 + + # Create the (in)famous correct brainwallet secret key. + priv = hashlib.sha256(b'correct horse battery staple').digest() + b"\x01" + pub = btc.privkey_to_pubkey(priv) + + # Create an address from that private key. + # (note that the input utxo is fake so we are really only creating + # a destination here). + scriptPubKey = btc.CScript([btc.OP_0, btc.Hash160(pub)]) + address = btc.P2WPKHBitcoinAddress.from_scriptPubKey(scriptPubKey) + + # Create a dummy outpoint; use same 32 bytes for convenience + txid = priv[:32] + vout = 2 + amount = btc.coins_to_satoshi(float('0.12345')) + + # Calculate an amount for the upcoming new UTXO. Set a high fee to bypass + # bitcoind minfee setting. + amount_less_fee = int(amount - btc.coins_to_satoshi(0.01)) + + # Create a destination to send the coins. + destination_address = address + target_scriptPubKey = scriptPubKey + + # Create the unsigned transaction. + txin = btc.CTxIn(btc.COutPoint(txid[::-1], vout)) + txout = btc.CTxOut(amount_less_fee, target_scriptPubKey) + tx = btc.CMutableTransaction([txin], [txout]) + + # Calculate the signature hash for the transaction. This is then signed by the + # private key that controls the UTXO being spent here at this txin_index. + if addrtype == "p2wpkh": + sig, msg = btc.sign(tx, 0, priv, amount=amount, native=True) + elif addrtype == "p2sh-p2wpkh": + sig, msg = btc.sign(tx, 0, priv, amount=amount, native=False) + elif addrtype == "p2pkh": + sig, msg = btc.sign(tx, 0, priv) + else: + assert False + if not sig: + print(msg) + raise + print("created signature: ", binascii.hexlify(sig)) + print("serialized transaction: {}".format(btc.b2x(tx.serialize()))) + +def test_mk_shuffled_tx(): + # prepare two addresses for the outputs + pub = btc.privkey_to_pubkey(btc.Hash(b"priv") + b"\x01") + scriptPubKey = btc.CScript([btc.OP_0, btc.Hash160(pub)]) + addr1 = btc.P2WPKHBitcoinAddress.from_scriptPubKey(scriptPubKey) + scriptPubKey_p2sh = scriptPubKey.to_p2sh_scriptPubKey() + addr2 = btc.CCoinAddress.from_scriptPubKey(scriptPubKey_p2sh) + + ins = [(btc.Hash(b"blah"), 7), (btc.Hash(b"foo"), 15)] + # note the casts str() ; most calls to mktx will have addresses fed + # as strings, so this is enforced for simplicity. + outs = [{"address": str(addr1), "value": btc.coins_to_satoshi(float("0.1"))}, + {"address": str(addr2), "value": btc.coins_to_satoshi(float("45981.23331234"))}] + tx = btc.make_shuffled_tx(ins, outs, version=2, locktime=500000) + +def test_bip143_tv(): + # p2sh-p2wpkh case: + rawtx_hex = "0100000001db6b1b20aa0fd7b23880be2ecbd4a98130974cf4748fb66092ac4d3ceb1a54770100000000feffffff02b8b4eb0b000000001976a914a457b684d7f0d539a46a45bbc043f35b59d0d96388ac0008af2f000000001976a914fd270b1ee6abcaea97fea7ad0402e8bd8ad6d77c88ac92040000" + inp_spk_hex = "a9144733f37cf4db86fbc2efed2500b4f4e49f31202387" + value = 10 + redeemScript = "001479091972186c449eb1ded22b78e40d009bdf0089" + privkey_hex = "eb696a065ef48a2192da5b28b694f87544b30fae8327c4510137a922f32c6dcf01" + pubkey_hex = "03ad1d8e89212f0b92c74d23bb710c00662ad1470198ac48c43f7d6f93a2a26873" + tx = btc.CMutableTransaction.deserialize(btc.x(rawtx_hex)) + btc.sign(tx, 0, btc.x(privkey_hex), amount=btc.coins_to_satoshi(10), native=False) + expectedsignedtx = "01000000000101db6b1b20aa0fd7b23880be2ecbd4a98130974cf4748fb66092ac4d3ceb1a5477010000001716001479091972186c449eb1ded22b78e40d009bdf0089feffffff02b8b4eb0b000000001976a914a457b684d7f0d539a46a45bbc043f35b59d0d96388ac0008af2f000000001976a914fd270b1ee6abcaea97fea7ad0402e8bd8ad6d77c88ac02473044022047ac8e878352d3ebbde1c94ce3a10d057c24175747116f8288e5d794d12d482f0220217f36a485cae903c713331d877c1f64677e3622ad4010726870540656fe9dcb012103ad1d8e89212f0b92c74d23bb710c00662ad1470198ac48c43f7d6f93a2a2687392040000" + assert btc.b2x(tx.serialize()) == expectedsignedtx \ No newline at end of file diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index f4dd264..0b73d68 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -18,7 +18,8 @@ from .wallet import (Mnemonic, estimate_tx_fee, WalletError, BaseWallet, ImportW UTXOManager, WALLET_IMPLEMENTATIONS, compute_tx_locktime) from .storage import (Argon2Hash, Storage, StorageError, RetryableStorageError, StoragePasswordError, VolatileStorage) -from .cryptoengine import BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, EngineError +from .cryptoengine import (BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, EngineError, + TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH) from .configure import (load_test_config, load_program_config, get_p2pk_vbyte, jm_single, get_network, update_persist_config, validate_address, is_burn_destination, get_irc_mchannels, diff --git a/jmclient/jmclient/blockchaininterface.py b/jmclient/jmclient/blockchaininterface.py index 777530a..56ddff0 100644 --- a/jmclient/jmclient/blockchaininterface.py +++ b/jmclient/jmclient/blockchaininterface.py @@ -6,7 +6,7 @@ import time from decimal import Decimal import binascii from twisted.internet import reactor, task - +from jmbase import bintohex, hextobin import jmbitcoin as btc from jmclient.jsonrpc import JsonRpcConnectionError, JsonRpcError @@ -297,9 +297,8 @@ class BitcoinCoreInterface(BlockchainInterface): if not "hex" in rpcretval: log.info("Malformed gettransaction output") return None - #str cast for unicode - hexval = str(rpcretval["hex"]) - return btc.deserialize(hexval) + return btc.CMutableTransaction.deserialize( + hextobin(rpcretval["hex"])) def list_transactions(self, num, skip=0): """ Return a list of the last `num` transactions seen @@ -309,20 +308,22 @@ class BitcoinCoreInterface(BlockchainInterface): return self.rpc("listtransactions", ["*", num, skip, True]) def get_transaction(self, txid): - """ Returns a serialized transaction for txid txid, + """ Argument txid is passed in binary. + Returns a serialized transaction for txid txid, in hex as returned by Bitcoin Core rpc, or None if no transaction can be retrieved. Works also for watch-only wallets. """ + htxid = bintohex(txid) #changed syntax in 0.14.0; allow both syntaxes try: - res = self.rpc("gettransaction", [txid, True]) - except: + res = self.rpc("gettransaction", [htxid, True]) + except Exception as e: try: - res = self.rpc("gettransaction", [txid, 1]) + res = self.rpc("gettransaction", [htxid, 1]) except JsonRpcError as e: #This should never happen (gettransaction is a wallet rpc). - log.warn("Failed gettransaction call; JsonRpcError") + log.warn("Failed gettransaction call; JsonRpcError: " + repr(e)) return None except Exception as e: log.warn("Failed gettransaction call; unexpected error:") @@ -333,7 +334,11 @@ class BitcoinCoreInterface(BlockchainInterface): return None return res - def pushtx(self, txhex): + def pushtx(self, txbin): + """ Given a binary serialized valid bitcoin transaction, + broadcasts it to the network. + """ + txhex = bintohex(txbin) try: txid = self.rpc('sendrawtransaction', [txhex]) except JsonRpcConnectionError as e: @@ -345,9 +350,9 @@ class BitcoinCoreInterface(BlockchainInterface): return True def query_utxo_set(self, txout, includeconf=False, includeunconf=False): - """If txout is either (a) a single string in hex encoded txid:n form, + """If txout is either (a) a single utxo in (txidbin, n) form, or a list of the same, returns, as a list for each txout item, - the result of gettxout from the bitcoind rpc for those utxs; + the result of gettxout from the bitcoind rpc for those utxos; if any utxo is invalid, None is returned. includeconf: if this is True, the current number of confirmations of the prescribed utxo is included in the returned result dict. @@ -360,16 +365,18 @@ class BitcoinCoreInterface(BlockchainInterface): txout = [txout] result = [] for txo in txout: - if len(txo) < 66: + txo_hex = bintohex(txo[0]) + if len(txo_hex) != 64: + log.warn("Invalid utxo format, ignoring: {}".format(txo)) result.append(None) continue try: - txo_idx = int(txo[65:]) + txo_idx = int(txo[1]) except ValueError: log.warn("Invalid utxo format, ignoring: {}".format(txo)) result.append(None) continue - ret = self.rpc('gettxout', [txo[:64], txo_idx, includeunconf]) + ret = self.rpc('gettxout', [txo_hex, txo_idx, includeunconf]) if ret is None: result.append(None) else: @@ -380,7 +387,7 @@ class BitcoinCoreInterface(BlockchainInterface): result_dict = {'value': int(Decimal(str(ret['value'])) * Decimal('1e8')), 'address': address, - 'script': ret['scriptPubKey']['hex']} + 'script': hextobin(ret['scriptPubKey']['hex'])} if includeconf: result_dict['confirms'] = int(ret['confirmations']) result.append(result_dict) diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index 8783cfc..9711d9b 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -9,16 +9,17 @@ try: except ImportError: pass from jmbase import commands - +import binascii import json import hashlib import os import sys -from jmbase import get_log +from jmbase import (get_log, EXIT_FAILURE, hextobin, bintohex, + utxo_to_utxostr, dictchanger) from jmclient import (jm_single, get_irc_mchannels, RegtestBitcoinCoreInterface) import jmbitcoin as btc -from jmbase.support import EXIT_FAILURE + jlog = get_log() @@ -30,7 +31,8 @@ class JMClientProtocol(amp.AMP): self.client = client self.factory = factory if not nick_priv: - self.nick_priv = hashlib.sha256(os.urandom(16)).hexdigest() + '01' + self.nick_priv = hashlib.sha256( + os.urandom(16)).digest() + b"\x01" else: self.nick_priv = nick_priv @@ -59,10 +61,18 @@ class JMClientProtocol(amp.AMP): self.clientStart() def set_nick(self): - self.nick_pubkey = btc.privtopub(self.nick_priv) - self.nick_pkh_raw = btc.bin_sha256(self.nick_pubkey)[ - :self.nick_hashlen] - self.nick_pkh = btc.b58encode(self.nick_pkh_raw) + """ Algorithm: take pubkey and hex-serialized it; + then SHA2(hexpub) but truncate output to nick_hashlen. + Then encode to a base58 string (no check). + Then prepend J and version char (e.g. '5'). + Finally append padding to nick_maxencoded (+2). + """ + self.nick_pubkey = btc.privkey_to_pubkey(self.nick_priv) + # note we use binascii hexlify directly here because input + # to hashing must be encoded. + self.nick_pkh_raw = hashlib.sha256(binascii.hexlify( + self.nick_pubkey)).digest()[:self.nick_hashlen] + self.nick_pkh = btc.base58.encode(self.nick_pkh_raw) #right pad to maximum possible; b58 is not fixed length. #Use 'O' as one of the 4 not included chars in base58. self.nick_pkh += 'O' * (self.nick_maxencoded - len(self.nick_pkh)) @@ -93,7 +103,7 @@ class JMClientProtocol(amp.AMP): @commands.JMRequestMsgSig.responder def on_JM_REQUEST_MSGSIG(self, nick, cmd, msg, msg_to_be_signed, hostid): sig = btc.ecdsa_sign(str(msg_to_be_signed), self.nick_priv) - msg_to_return = str(msg) + " " + self.nick_pubkey + " " + sig + msg_to_return = str(msg) + " " + bintohex(self.nick_pubkey) + " " + sig d = self.callRemote(commands.JMMsgSignature, nick=nick, cmd=cmd, @@ -105,20 +115,21 @@ class JMClientProtocol(amp.AMP): @commands.JMRequestMsgSigVerify.responder def on_JM_REQUEST_MSGSIG_VERIFY(self, msg, fullmsg, sig, pubkey, nick, hashlen, max_encoded, hostid): + pubkey_bin = hextobin(pubkey) verif_result = True - if not btc.ecdsa_verify(str(msg), sig, pubkey): + if not btc.ecdsa_verify(str(msg), sig, pubkey_bin): # workaround for hostid, which sometimes is lowercase-only for some IRC connections - if not btc.ecdsa_verify(str(msg[:-len(hostid)] + hostid.lower()), sig, pubkey): + if not btc.ecdsa_verify(str(msg[:-len(hostid)] + hostid.lower()), sig, pubkey_bin): jlog.debug("nick signature verification failed, ignoring: " + str(nick)) verif_result = False #check that nick matches hash of pubkey - nick_pkh_raw = btc.bin_sha256(pubkey)[:hashlen] + nick_pkh_raw = hashlib.sha256(pubkey.encode("ascii")).digest()[:hashlen] nick_stripped = nick[2:2 + max_encoded] #strip right padding nick_unpadded = ''.join([x for x in nick_stripped if x != 'O']) - if not nick_unpadded == btc.b58encode(nick_pkh_raw): + if not nick_unpadded == btc.base58.encode(nick_pkh_raw): jlog.debug("Nick hash check failed, expected: " + str(nick_unpadded) - + ", got: " + str(btc.b58encode(nick_pkh_raw))) + + ", got: " + str(btc.base58.encode(nick_pkh_raw))) verif_result = False d = self.callRemote(commands.JMMsgSignatureVerify, verif_result=verif_result, @@ -234,10 +245,18 @@ class JMMakerClientProtocol(JMClientProtocol): jlog.info("Maker refuses to continue on receiving auth.") else: utxos, auth_pub, cj_addr, change_addr, btc_sig = retval[1:] + # json does not allow non-string keys: + utxos_strkeyed = {} + for k in utxos: + success, u = utxo_to_utxostr(k) + assert success + utxos_strkeyed[u] = {"value": utxos[k]["value"], + "address": utxos[k]["address"]} + auth_pub_hex = bintohex(auth_pub) d = self.callRemote(commands.JMIOAuth, nick=nick, - utxolist=json.dumps(utxos), - pubkey=auth_pub, + utxolist=json.dumps(utxos_strkeyed), + pubkey=auth_pub_hex, cjaddr=cj_addr, changeaddr=change_addr, pubkeysig=btc_sig) @@ -258,14 +277,14 @@ class JMMakerClientProtocol(JMClientProtocol): else: sigs = retval[1] self.finalized_offers[nick] = offer - tx = btc.deserialize(txhex) + tx = btc.CMutableTransaction.deserialize(hextobin(txhex)) self.finalized_offers[nick]["txd"] = tx - txid = btc.txhash(btc.serialize(tx)) + txid = tx.GetTxid()[::-1] # we index the callback by the out-set of the transaction, # because the txid is not known until all scriptSigs collected # (hence this is required for Makers, but not Takers). # For more info see WalletService.transaction_monitor(): - txinfo = tuple((x["script"], x["value"]) for x in tx["outs"]) + txinfo = tuple((x.scriptPubKey, x.nValue) for x in tx.vout) self.client.wallet_service.register_callbacks([self.unconfirm_callback], txinfo, "unconfirmed") self.client.wallet_service.register_callbacks([self.confirm_callback], @@ -285,8 +304,8 @@ class JMMakerClientProtocol(JMClientProtocol): def tx_match(self, txd): for k,v in iteritems(self.finalized_offers): - #Tx considered defined by its output set - if v["txd"]["outs"] == txd["outs"]: + # Tx considered defined by its output set + if v["txd"].vout == txd.vout: offerinfo = v break else: @@ -302,7 +321,7 @@ class JMMakerClientProtocol(JMClientProtocol): txid) self.client.modify_orders(to_cancel, to_announce) - txinfo = tuple((x["script"], x["value"]) for x in txd["outs"]) + txinfo = tuple((x.scriptPubKey, x.nValue) for x in txd.vout) confirm_timeout_sec = float(jm_single().config.get( "TIMEOUT", "confirm_timeout_hours")) * 3600 task.deferLater(reactor, confirm_timeout_sec, diff --git a/jmclient/jmclient/commitment_utils.py b/jmclient/jmclient/commitment_utils.py index 64a80c4..ab598ad 100644 --- a/jmclient/jmclient/commitment_utils.py +++ b/jmclient/jmclient/commitment_utils.py @@ -2,8 +2,10 @@ import sys import jmbitcoin as btc from jmbase import jmprint -from jmclient import jm_single, get_p2pk_vbyte, get_p2sh_vbyte -from jmbase.support import EXIT_FAILURE +from jmclient import (jm_single, get_p2pk_vbyte, get_p2sh_vbyte, + BTCEngine, TYPE_P2PKH, TYPE_P2SH_P2WPKH, + BTC_P2PKH, BTC_P2SH_P2WPKH) +from jmbase.support import EXIT_FAILURE, utxostr_to_utxo def quit(parser, errmsg): #pragma: no cover @@ -18,17 +20,16 @@ def get_utxo_info(upriv): u, priv = upriv.split(',') u = u.strip() priv = priv.strip() - txid, n = u.split(':') - assert len(txid)==64 - assert len(n) in range(1, 4) - n = int(n) - assert n in range(256) + success, utxo = utxostr_to_utxo(u) + assert success, utxo except: #not sending data to stdout in case privkey info jmprint("Failed to parse utxo information for utxo", "error") raise try: - hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte()) + # see note below for why keytype is ignored, and note that + # this calls read_privkey to validate. + raw, _ = BTCEngine.wif_to_privkey(priv) except: jmprint("failed to parse privkey, make sure it's WIF compressed format.", "error") raise @@ -40,16 +41,20 @@ def validate_utxo_data(utxo_datas, retrieve=False, segwit=False): then use the blockchain instance to look up the utxo and check that its address field matches. If retrieve is True, return the set of utxos and their values. + If segwit is true, assumes a p2sh wrapped p2wpkh, i.e. + native segwit is NOT currently supported here. If segwit + is false, p2pkh is assumed. """ results = [] for u, priv in utxo_datas: jmprint('validating this utxo: ' + str(u), "info") - hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte()) - if segwit: - addr = btc.pubkey_to_p2sh_p2wpkh_address( - btc.privkey_to_pubkey(hexpriv), get_p2sh_vbyte()) - else: - addr = btc.privkey_to_address(hexpriv, magicbyte=get_p2pk_vbyte()) + # as noted in `ImportWalletMixin` code comments, there is not + # yet a functional auto-detection of key type from WIF, so the + # second argument is ignored; we assume p2sh-p2wpkh if segwit, + # else we assume p2pkh. + engine = BTC_P2SH_P2WPKH if segwit else BTC_P2PKH + rawpriv, _ = BTCEngine.wif_to_privkey(priv) + addr = engine.privkey_to_address(rawpriv) jmprint('claimed address: ' + addr, "info") res = jm_single().bc_interface.query_utxo_set([u]) if len(res) != 1 or None in res: diff --git a/jmclient/jmclient/configure.py b/jmclient/jmclient/configure.py index 48372b2..45f5df2 100644 --- a/jmclient/jmclient/configure.py +++ b/jmclient/jmclient/configure.py @@ -377,33 +377,18 @@ def get_p2pk_vbyte(): def validate_address(addr): try: - assert len(addr) > 2 - if addr[:2].lower() in ['bc', 'tb']: - # Regtest special case - if addr[:4] == 'bcrt': - if btc.bech32addr_decode('bcrt', addr)[1]: - return True, 'address validated' - return False, 'Invalid bech32 regtest address' - #Else, enforce testnet/mainnet per config - if get_network() == "testnet": - hrpreq = 'tb' - else: - hrpreq = 'bc' - if btc.bech32addr_decode(hrpreq, addr)[1]: - return True, 'address validated' - return False, 'Invalid bech32 address' - #Not bech32; assume b58 from here - ver = btc.get_version_byte(addr) - except AssertionError: - return False, 'Checksum wrong. Typo in address?' - except Exception: - return False, "Invalid bitcoin address" - if ver != get_p2pk_vbyte() and ver != get_p2sh_vbyte(): - return False, 'Wrong address version. Testnet/mainnet confused?' - if len(btc.b58check_to_bin(addr)) != 20: - return False, "Address has correct checksum but wrong length." - return True, 'address validated' - + # automatically respects the network + # as set in btc.select_chain_params(...) + x = btc.CCoinAddress(addr) + except Exception as e: + return False, repr(e) + # additional check necessary because python-bitcointx + # does not check hash length on p2sh construction. + try: + x.to_scriptPubKey() + except Exception as e: + return False, repr(e) + return True, "address validated" _BURN_DESTINATION = "BURN" @@ -565,6 +550,7 @@ def get_blockchain_interface_instance(_config): source = _config.get("BLOCKCHAIN", "blockchain_source") network = get_network() testnet = network == 'testnet' + if source in ('bitcoin-rpc', 'regtest', 'bitcoin-rpc-no-history'): rpc_host = _config.get("BLOCKCHAIN", "rpc_host") rpc_port = _config.get("BLOCKCHAIN", "rpc_port") @@ -574,10 +560,20 @@ def get_blockchain_interface_instance(_config): rpc_wallet_file) if source == 'bitcoin-rpc': #pragma: no cover bc_interface = BitcoinCoreInterface(rpc, network) + if testnet: + btc.select_chain_params("bitcoin/testnet") + else: + btc.select_chain_params("bitcoin") elif source == 'regtest': bc_interface = RegtestBitcoinCoreInterface(rpc) + btc.select_chain_params("bitcoin/regtest") elif source == "bitcoin-rpc-no-history": bc_interface = BitcoinCoreNoHistoryInterface(rpc, network) + if testnet or network == "regtest": + # TODO will not work for bech32 regtest addresses: + btc.select_chain_params("bitcoin/testnet") + else: + btc.select_chain_params("bitcoin") else: assert 0 elif source == 'electrum': diff --git a/jmclient/jmclient/cryptoengine.py b/jmclient/jmclient/cryptoengine.py index 9b5d9f1..c24d108 100644 --- a/jmclient/jmclient/cryptoengine.py +++ b/jmclient/jmclient/cryptoengine.py @@ -19,20 +19,30 @@ NET_MAP = {'mainnet': NET_MAINNET, 'testnet': NET_TESTNET} WIF_PREFIX_MAP = {'mainnet': b'\x80', 'testnet': b'\xef'} BIP44_COIN_MAP = {'mainnet': 2**31, 'testnet': 2**31 + 1} -def detect_script_type(script): - if script.startswith(btc.P2PKH_PRE) and script.endswith(btc.P2PKH_POST) and\ - len(script) == 0x14 + len(btc.P2PKH_PRE) + len(btc.P2PKH_POST): +def detect_script_type(script_str): + """ Given a scriptPubKey, decide which engine + to use, one of: p2pkh, p2sh-p2wpkh, p2wpkh. + Note that for the p2sh case, we are assuming the nature + of the redeem script (p2wpkh wrapped) because that is what + we support; but we can't know for sure, from the sPK only. + Raises EngineError if the type cannot be detected, so + callers MUST handle this exception to avoid crashes. + """ + script = btc.CScript(script_str) + if not script.is_valid(): + raise EngineError("Unknown script type for script '{}'" + .format(hexlify(script_str))) + if script.is_p2pkh(): return TYPE_P2PKH - elif (script.startswith(btc.P2SH_P2WPKH_PRE) and - script.endswith(btc.P2SH_P2WPKH_POST) and - len(script) == 0x14 + len(btc.P2SH_P2WPKH_PRE) + len( - btc.P2SH_P2WPKH_POST)): + elif script.is_p2sh(): + # see note above. + # note that is_witness_v0_nested_keyhash does not apply, + # since that picks up scriptSigs not scriptPubKeys. return TYPE_P2SH_P2WPKH - elif script.startswith(btc.P2WPKH_PRE) and\ - len(script) == 0x14 + len(btc.P2WPKH_PRE): + elif script.is_witness_v0_keyhash(): return TYPE_P2WPKH raise EngineError("Unknown script type for script '{}'" - .format(hexlify(script))) + .format(hexlify(script_str))) class classproperty(object): """ @@ -97,28 +107,38 @@ class BTCEngine(object): @staticmethod def privkey_to_pubkey(privkey): - return btc.privkey_to_pubkey(privkey, False) + return btc.privkey_to_pubkey(privkey) @staticmethod def address_to_script(addr): - return unhexlify(btc.address_to_script(addr)) + return btc.CCoinAddress(addr).to_scriptPubKey() @classmethod def wif_to_privkey(cls, wif): - raw = btc.b58check_to_bin(wif) + raw = btc.b58check_to_bin(wif)[1] + # see note to `privkey_to_wif`; same applies here. + # We only handle valid private keys, not any byte string. + btc.read_privkey(raw) + vbyte = struct.unpack('B', btc.get_version_byte(wif))[0] - if (struct.unpack('B', btc.BTC_P2PK_VBYTE[get_network()])[0] + struct.unpack('B', cls.WIF_PREFIX)[0]) & 0xff == vbyte: + if (struct.unpack('B', btc.BTC_P2PK_VBYTE[get_network()])[0] + \ + struct.unpack('B', cls.WIF_PREFIX)[0]) & 0xff == vbyte: key_type = TYPE_P2PKH - elif (struct.unpack('B', btc.BTC_P2SH_VBYTE[get_network()])[0] + struct.unpack('B', cls.WIF_PREFIX)[0]) & 0xff == vbyte: + elif (struct.unpack('B', btc.BTC_P2SH_VBYTE[get_network()])[0] + \ + struct.unpack('B', cls.WIF_PREFIX)[0]) & 0xff == vbyte: key_type = TYPE_P2SH_P2WPKH else: key_type = None - return raw, key_type @classmethod def privkey_to_wif(cls, priv): + # refuse to WIF-ify something that we don't recognize + # as a private key; ignoring the return value of this + # function as we only want to raise whatever Exception + # it does: + btc.read_privkey(priv) return btc.bin_to_b58check(priv, cls.WIF_PREFIX) @classmethod @@ -166,12 +186,12 @@ class BTCEngine(object): @classmethod def privkey_to_address(cls, privkey): script = cls.key_to_script(privkey) - return btc.script_to_address(script, cls.VBYTE) + return str(btc.CCoinAddress.from_scriptPubKey(script)) @classmethod def pubkey_to_address(cls, pubkey): script = cls.pubkey_to_script(pubkey) - return btc.script_to_address(script, cls.VBYTE) + return str(btc.script_to_address(script, cls.VBYTE)) @classmethod def pubkey_has_address(cls, pubkey, addr): @@ -203,7 +223,12 @@ class BTCEngine(object): @classmethod def script_to_address(cls, script): - return btc.script_to_address(script, vbyte=cls.VBYTE) + """ a script passed in as binary converted to a + Bitcoin address of the appropriate type. + """ + s = btc.CScript(script) + assert s.is_valid() + return str(btc.CCoinAddress.from_scriptPubKey(s)) class BTC_P2PKH(BTCEngine): @@ -213,6 +238,7 @@ class BTC_P2PKH(BTCEngine): @classmethod def pubkey_to_script(cls, pubkey): + # this call does not enforce compressed: return btc.pubkey_to_p2pkh_script(pubkey) @classmethod @@ -222,7 +248,7 @@ class BTC_P2PKH(BTCEngine): @classmethod def sign_transaction(cls, tx, index, privkey, *args, **kwargs): hashcode = kwargs.get('hashcode') or btc.SIGHASH_ALL - return btc.sign(btc.serialize(tx), index, privkey, + return btc.sign(tx, index, privkey, hashcode=hashcode, amount=None, native=False) @@ -250,8 +276,9 @@ class BTC_P2SH_P2WPKH(BTCEngine): def sign_transaction(cls, tx, index, privkey, amount, hashcode=btc.SIGHASH_ALL, **kwargs): assert amount is not None - return btc.sign(btc.serialize(tx), index, privkey, + a, b = btc.sign(tx, index, privkey, hashcode=hashcode, amount=amount, native=False) + return a, b class BTC_P2WPKH(BTCEngine): @@ -286,7 +313,7 @@ class BTC_P2WPKH(BTCEngine): def sign_transaction(cls, tx, index, privkey, amount, hashcode=btc.SIGHASH_ALL, **kwargs): assert amount is not None - return btc.sign(btc.serialize(tx), index, privkey, + return btc.sign(tx, index, privkey, hashcode=hashcode, amount=amount, native=True) class BTC_Timelocked_P2WSH(BTCEngine): diff --git a/jmclient/jmclient/maker.py b/jmclient/jmclient/maker.py index c19c5a4..8ae4834 100644 --- a/jmclient/jmclient/maker.py +++ b/jmclient/jmclient/maker.py @@ -5,15 +5,13 @@ import pprint import random import sys import abc -from binascii import unhexlify - -from jmbitcoin import SerializationError, SerializationTruncationError import jmbitcoin as btc +from jmbase import (bintohex, hextobin, hexbin, + get_log, EXIT_SUCCESS, EXIT_FAILURE) from jmclient.wallet import estimate_tx_fee, compute_tx_locktime from jmclient.wallet_service import WalletService from jmclient.configure import jm_single -from jmbase.support import get_log, EXIT_SUCCESS, EXIT_FAILURE from jmclient.support import calc_cj_fee, select_one_utxo from jmclient.podle import verify_podle, PoDLE, PoDLEError from twisted.internet import task, reactor @@ -48,10 +46,15 @@ class Maker(object): sys.exit(EXIT_FAILURE) jlog.info('offerlist={}'.format(self.offerlist)) + @hexbin def on_auth_received(self, nick, offer, commitment, cr, amount, kphex): """Receives data on proposed transaction offer from daemon, verifies commitment, returns necessary data to send ioauth message (utxos etc) """ + # special case due to cjfee passed as string: it can accidentally parse + # as hex: + if not isinstance(offer["cjfee"], str): + offer["cjfee"] = bintohex(offer["cjfee"]) #check the validity of the proof of discrete log equivalence tries = jm_single().config.getint("POLICY", "taker_utxo_retries") def reject(msg): @@ -65,9 +68,8 @@ class Maker(object): reason = repr(e) return reject(reason) - if not verify_podle(str(cr_dict['P']), str(cr_dict['P2']), str(cr_dict['sig']), - str(cr_dict['e']), str(commitment), - index_range=range(tries)): + if not verify_podle(cr_dict['P'], cr_dict['P2'], cr_dict['sig'], + cr_dict['e'], commitment, index_range=range(tries)): reason = "verify_podle failed" return reject(reason) #finally, check that the proffered utxo is real, old enough, large enough, @@ -89,14 +91,14 @@ class Maker(object): try: if not self.wallet_service.pubkey_has_script( - unhexlify(cr_dict['P']), unhexlify(res[0]['script'])): + cr_dict['P'], res[0]['script']): raise EngineError() except EngineError: reason = "Invalid podle pubkey: " + str(cr_dict['P']) return reject(reason) # authorisation of taker passed - #Find utxos for the transaction now: + # Find utxos for the transaction now: utxos, cj_addr, change_addr = self.oid_to_order(offer, amount) if not utxos: #could not find funds @@ -109,21 +111,28 @@ class Maker(object): # Just choose the first utxo in self.utxos and retrieve key from wallet. auth_address = utxos[list(utxos.keys())[0]]['address'] auth_key = self.wallet_service.get_key_from_addr(auth_address) - auth_pub = btc.privtopub(auth_key) - btc_sig = btc.ecdsa_sign(kphex, auth_key) + auth_pub = btc.privkey_to_pubkey(auth_key) + # kphex was auto-converted by @hexbin but we actually need to sign the + # hex version to comply with pre-existing JM protocol: + btc_sig = btc.ecdsa_sign(bintohex(kphex), auth_key) return (True, utxos, auth_pub, cj_addr, change_addr, btc_sig) - def on_tx_received(self, nick, txhex, offerinfo): + @hexbin + def on_tx_received(self, nick, tx_from_taker, offerinfo): """Called when the counterparty has sent an unsigned transaction. Sigs are created and returned if and only if the transaction passes verification checks (see verify_unsigned_tx()). """ + # special case due to cjfee passed as string: it can accidentally parse + # as hex: + if not isinstance(offerinfo["offer"]["cjfee"], str): + offerinfo["offer"]["cjfee"] = bintohex(offerinfo["offer"]["cjfee"]) try: - tx = btc.deserialize(txhex) - except (IndexError, SerializationError, SerializationTruncationError) as e: + tx = btc.CMutableTransaction.deserialize(tx_from_taker) + except Exception as e: return (False, 'malformed txhex. ' + repr(e)) - jlog.info('obtained tx\n' + pprint.pformat(tx)) + jlog.info('obtained tx\n' + bintohex(tx.serialize())) goodtx, errmsg = self.verify_unsigned_tx(tx, offerinfo) if not goodtx: jlog.info('not a good tx, reason=' + errmsg) @@ -133,25 +142,26 @@ class Maker(object): utxos = offerinfo["utxos"] our_inputs = {} - for index, ins in enumerate(tx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + for index, ins in enumerate(tx.vin): + utxo = (ins.prevout.hash[::-1], ins.prevout.n) if utxo not in utxos: continue script = self.wallet_service.addr_to_script(utxos[utxo]['address']) amount = utxos[utxo]['value'] our_inputs[index] = (script, amount) - txs = self.wallet_service.sign_tx(btc.deserialize(unhexlify(txhex)), our_inputs) + success, msg = self.wallet_service.sign_tx(tx, our_inputs) + assert success, msg for index in our_inputs: - sigmsg = unhexlify(txs['ins'][index]['script']) - if 'txinwitness' in txs['ins'][index]: + sigmsg = tx.vin[index].scriptSig + if tx.has_witness(): # Note that this flag only implies that the transaction # *as a whole* is using segwit serialization; it doesn't # imply that this specific input is segwit type (to be # fully general, we allow that even our own wallet's # inputs might be of mixed type). So, we catch the EngineError # which is thrown by non-segwit types. This way the sigmsg - # will only contain the scriptSig field if the wallet object + # will only contain the scriptCode field if the wallet object # decides it's necessary/appropriate for this specific input # If it is segwit, we prepend the witness data since we want # (sig, pub, witnessprogram=scriptSig - note we could, better, @@ -160,16 +170,16 @@ class Maker(object): # transaction scriptSig), else (non-sw) the !sig message remains # unchanged as (sig, pub). try: - scriptSig = btc.pubkey_to_p2wpkh_script(txs['ins'][index]['txinwitness'][1]) - sigmsg = b''.join(btc.serialize_script_unit( - x) for x in txs['ins'][index]['txinwitness'] + [scriptSig]) - except IndexError: + sig, pub = [a for a in iter(tx.wit.vtxinwit[index].scriptWitness)] + scriptCode = btc.pubkey_to_p2wpkh_script(pub) + sigmsg = btc.CScript([sig]) + btc.CScript(pub) + scriptCode + except Exception as e: #the sigmsg was already set before the segwit check pass sigs.append(base64.b64encode(sigmsg).decode('ascii')) return (True, sigs) - def verify_unsigned_tx(self, txd, offerinfo): + def verify_unsigned_tx(self, tx, offerinfo): """This code is security-critical. Before signing the transaction the Maker must ensure that all details are as expected, and most importantly @@ -177,14 +187,13 @@ class Maker(object): in total. The data is taken from the offerinfo dict and compared with the serialized txhex. """ - tx_utxo_set = set(ins['outpoint']['hash'] + ':' + str( - ins['outpoint']['index']) for ins in txd['ins']) + tx_utxo_set = set((x.prevout.hash[::-1], x.prevout.n) for x in tx.vin) utxos = offerinfo["utxos"] cjaddr = offerinfo["cjaddr"] - cjaddr_script = btc.address_to_script(cjaddr) + cjaddr_script = btc.CCoinAddress(cjaddr).to_scriptPubKey() changeaddr = offerinfo["changeaddr"] - changeaddr_script = btc.address_to_script(changeaddr) + changeaddr_script = btc.CCoinAddress(changeaddr).to_scriptPubKey() #Note: this value is under the control of the Taker, #see comment below. amount = offerinfo["amount"] @@ -216,14 +225,14 @@ class Maker(object): #exactly once with the required amts, in the output. times_seen_cj_addr = 0 times_seen_change_addr = 0 - for outs in txd['outs']: - if outs['script'] == cjaddr_script: + for outs in tx.vout: + if outs.scriptPubKey == cjaddr_script: times_seen_cj_addr += 1 - if outs['value'] != amount: + if outs.nValue != amount: return (False, 'Wrong cj_amount. I expect ' + str(amount)) - if outs['script'] == changeaddr_script: + if outs.scriptPubKey == changeaddr_script: times_seen_change_addr += 1 - if outs['value'] != expected_change_value: + if outs.nValue != expected_change_value: return (False, 'wrong change, i expect ' + str( expected_change_value)) if times_seen_cj_addr != 1 or times_seen_change_addr != 1: @@ -392,7 +401,8 @@ class P2EPMaker(Maker): # will not be reached except in testing self.on_tx_unconfirmed(txd, txid) - def on_tx_received(self, nick, txhex): + @hexbin + def on_tx_received(self, nick, txser): """ Called when the sender-counterparty has sent a transaction proposal. 1. First we check for the expected destination and amount (this is sufficient to identify our cp, as this info was presumably passed @@ -418,29 +428,30 @@ class P2EPMaker(Maker): we broadcast the non-coinjoin fallback tx instead. """ try: - tx = btc.deserialize(txhex) - except (IndexError, SerializationError, SerializationTruncationError) as e: + tx = btc.CMutableTransaction.deserialize(txser) + except Exception as e: return (False, 'malformed txhex. ' + repr(e)) self.user_info('obtained proposed fallback (non-coinjoin) ' +\ - 'transaction from sender:\n' + pprint.pformat(tx)) + 'transaction from sender:\n' + str(tx)) - if len(tx["outs"]) != 2: + if len(tx.vout) != 2: return (False, "Transaction has more than 2 outputs; not supported.") dest_found = False destination_index = -1 change_index = -1 proposed_change_value = 0 - for index, out in enumerate(tx["outs"]): - if out["script"] == btc.address_to_script(self.destination_addr): + for index, out in enumerate(tx.vout): + if out.scriptPubKey == btc.CCoinAddress( + self.destination_addr).to_scriptPubKey(): # we found the expected destination; is the amount correct? - if not out["value"] == self.receiving_amount: + if not out.nValue == self.receiving_amount: return (False, "Wrong payout value in proposal from sender.") dest_found = True destination_index = index else: change_found = True - proposed_change_out = out["script"] - proposed_change_value = out["value"] + proposed_change_out = out.scriptPubKey + proposed_change_value = out.nValue change_index = index if not dest_found: @@ -450,9 +461,8 @@ class P2EPMaker(Maker): # batch retrieval of utxo data utxo = {} ctr = 0 - for index, ins in enumerate(tx['ins']): - utxo_for_checking = ins['outpoint']['hash'] + ':' + str(ins[ - 'outpoint']['index']) + for index, ins in enumerate(tx.vin): + utxo_for_checking = (ins.prevout.hash[::-1], ins.prevout.n) utxo[ctr] = [index, utxo_for_checking] ctr += 1 @@ -471,7 +481,7 @@ class P2EPMaker(Maker): btc_fee = total_sender_input - self.receiving_amount - proposed_change_value self.user_info("Network transaction fee of fallback tx is: " + str( btc_fee) + " satoshis.") - fee_est = estimate_tx_fee(len(tx['ins']), len(tx['outs']), + fee_est = estimate_tx_fee(len(tx.vin), len(tx.vout), txtype=self.wallet_service.get_txtype()) fee_ok = False if btc_fee > 0.3 * fee_est and btc_fee < 3 * fee_est: @@ -488,51 +498,28 @@ class P2EPMaker(Maker): # It has the advantage of (a) being simpler and (b) allowing for any # non standard coins. # - #res = jm_single().bc_interface.rpc('testmempoolaccept', [txhex]) + #res = jm_single().bc_interface.rpc('testmempoolaccept', [txser]) #print("Got this result from rpc call: ", res) #if not res["accepted"]: # return (False, "Proposed transaction was rejected from mempool.") - # Manual verification of the transaction signatures. Passing this - # test does imply that the transaction is valid (unless there is - # a double spend during the process), but is restricted to standard - # types: p2pkh, p2wpkh, p2sh-p2wpkh only. Double spend is not counted - # as a risk as this is a payment. + # Manual verification of the transaction signatures. + # TODO handle native segwit properly for i, u in iteritems(utxo): - if "txinwitness" in tx["ins"][u[0]]: - ver_amt = utxo_data[i]["value"] - try: - ver_sig, ver_pub = tx["ins"][u[0]]["txinwitness"] - except Exception as e: - self.user_info("Segwit error: " + repr(e)) - return (False, "Segwit input not of expected type, " - "either p2sh-p2wpkh or p2wpkh") - # note that the scriptCode is the same whether nested or not - # also note that the scriptCode has to be inferred if we are - # only given a transaction serialization. - scriptCode = "76a914" + btc.hash160(unhexlify(ver_pub)) + "88ac" - else: - scriptCode = None - ver_amt = None - scriptSig = btc.deserialize_script(tx["ins"][u[0]]["script"]) - if len(scriptSig) != 2: - return (False, - "Proposed transaction contains unsupported input type") - ver_sig, ver_pub = scriptSig - if not btc.verify_tx_input(txhex, u[0], - utxo_data[i]['script'], - ver_sig, ver_pub, - scriptCode=scriptCode, - amount=ver_amt): + if not btc.verify_tx_input(tx, i, + tx.vin[i].scriptSig, + btc.CScript(utxo_data[i]["script"]), + amount=utxo_data[i]["value"], + witness=tx.wit.vtxinwit[i].scriptWitness): return (False, "Proposed transaction is not correctly signed.") # At this point we are satisfied with the proposal. Record the fallback # in case the sender disappears and the payjoin tx doesn't happen: self.user_info("We'll use this serialized transaction to broadcast if your" " counterparty fails to broadcast the payjoin version:") - self.user_info(txhex) + self.user_info(bintohex(txser)) # Keep a local copy for broadcast fallback: - self.fallback_tx = txhex + self.fallback_tx = tx # Now we add our own inputs: # See the gist comment here: @@ -558,7 +545,7 @@ class P2EPMaker(Maker): self.user_info("Choosing one coin at random") try: my_utxos = self.wallet_service.select_utxos( - self.mixdepth, jm_single().DUST_THRESHOLD, + self.mixdepth, jm_single().DUST_THRESHOLD, select_fn=select_one_utxo) except: return self.no_coins_fallback() @@ -566,7 +553,7 @@ class P2EPMaker(Maker): else: # get an approximate required amount assuming 4 inputs, which is # fairly conservative (but guess by necessity). - fee_for_select = estimate_tx_fee(len(tx['ins']) + 4, 2, + fee_for_select = estimate_tx_fee(len(tx.vin) + 4, 2, txtype=self.wallet_service.get_txtype()) approx_sum = max_sender_amt - self.receiving_amount + fee_for_select try: @@ -592,7 +579,7 @@ class P2EPMaker(Maker): # adjust the output amount at the destination based on our contribution new_destination_amount = self.receiving_amount + my_total_in # estimate the required fee for the new version of the transaction - total_ins = len(tx["ins"]) + len(my_utxos.keys()) + total_ins = len(tx.vin) + len(my_utxos.keys()) est_fee = estimate_tx_fee(total_ins, 2, txtype=self.wallet_service.get_txtype()) self.user_info("We estimated a fee of: " + str(est_fee)) new_change_amount = total_sender_input + my_total_in - \ @@ -604,25 +591,27 @@ class P2EPMaker(Maker): new_outs = [{"address": self.destination_addr, "value": new_destination_amount}] if new_change_amount >= jm_single().BITCOIN_DUST_THRESHOLD: - new_outs.append({"script": proposed_change_out, - "value": new_change_amount}) + new_outs.append({"address": str(btc.CCoinAddress.from_scriptPubKey( + proposed_change_out)), "value": new_change_amount}) new_ins = [x[1] for x in utxo.values()] new_ins.extend(my_utxos.keys()) - new_tx = btc.make_shuffled_tx(new_ins, new_outs, False, 2, compute_tx_locktime()) - new_tx_deser = btc.deserialize(new_tx) + new_tx = btc.make_shuffled_tx(new_ins, new_outs, 2, compute_tx_locktime()) + # sign our inputs before transfer our_inputs = {} - for index, ins in enumerate(new_tx_deser['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + for index, ins in enumerate(new_tx.vin): + utxo = (ins.prevout.hash[::-1], ins.prevout.n) if utxo not in my_utxos: continue - script = self.wallet_service.addr_to_script(my_utxos[utxo]['address']) - amount = my_utxos[utxo]['value'] + script = my_utxos[utxo]["script"] + amount = my_utxos[utxo]["value"] our_inputs[index] = (script, amount) - txs = self.wallet_service.sign_tx(btc.deserialize(new_tx), our_inputs) - txinfo = tuple((x["script"], x["value"]) for x in txs["outs"]) + success, msg = self.wallet_service.sign_tx(new_tx, our_inputs) + if not success: + return (False, "Failed to sign new transaction, error: " + msg) + txinfo = tuple((x.scriptPubKey, x.nValue) for x in new_tx.vout) self.wallet_service.register_callbacks([self.on_tx_unconfirmed], txinfo, "unconfirmed") self.wallet_service.register_callbacks([self.on_tx_confirmed], txinfo, "confirmed") # The blockchain interface just abandons monitoring if the transaction @@ -630,7 +619,7 @@ class P2EPMaker(Maker): # action in this case, so we add an additional callback to the reactor: reactor.callLater(jm_single().config.getint("TIMEOUT", "unconfirm_timeout_sec"), self.broadcast_fallback) - return (True, nick, btc.serialize(txs)) + return (True, nick, bintohex(new_tx.serialize())) def no_coins_fallback(self): """ Broadcast, optionally, the fallback non-coinjoin transaction @@ -646,8 +635,8 @@ class P2EPMaker(Maker): def broadcast_fallback(self): self.user_info("Broadcasting non-coinjoin fallback transaction.") - txid = btc.txhash(self.fallback_tx) - success = jm_single().bc_interface.pushtx(self.fallback_tx) + txid = self.fallback_tx.GetTxid()[::-1] + success = jm_single().bc_interface.pushtx(self.fallback_tx.serialize()) if not success: self.user_info("ERROR: the fallback transaction did not broadcast. " "The payment has NOT been made.") diff --git a/jmclient/jmclient/output.py b/jmclient/jmclient/output.py index 7242aec..a6ffc4f 100644 --- a/jmclient/jmclient/output.py +++ b/jmclient/jmclient/output.py @@ -61,7 +61,7 @@ def generate_podle_error_string(priv_utxo_pairs, to, ts, wallet_service, cjamoun "with 'python add-utxo.py --help'\n\n") errmsg += ("***\nFor reference, here are the utxos in your wallet:\n") - for md, utxos in wallet_service.get_utxos_by_mixdepth(hexfmt=False).items(): + for md, utxos in wallet_service.get_utxos_by_mixdepth().items(): if not utxos: continue errmsg += ("\nmixdepth {}:\n{}".format( diff --git a/jmclient/jmclient/podle.py b/jmclient/jmclient/podle.py index eff5dcc..d7dd26f 100644 --- a/jmclient/jmclient/podle.py +++ b/jmclient/jmclient/podle.py @@ -7,11 +7,12 @@ import hashlib import json import binascii import struct +from pprint import pformat from jmbase import jmprint from jmbitcoin import multiply, add_pubkeys, getG, podle_PublicKey,\ - podle_PrivateKey, encode, decode, N, podle_PublicKey_class -from jmbase.support import EXIT_FAILURE - + podle_PrivateKey, N, podle_PublicKey_class +from jmbase import (EXIT_FAILURE, utxostr_to_utxo, + utxo_to_utxostr, hextobin, bintohex) PODLE_COMMIT_FILE = None @@ -45,33 +46,31 @@ class PoDLE(object): used=False): #This class allows storing of utxo in format "txid:n" only for #convenience of storage/access; it doesn't check or use the data. - #Arguments must be provided in hex. + #Arguments must be provided in binary not hex. self.u = u if not priv: if P: - #Construct a pubkey from raw hex - self.P = podle_PublicKey(binascii.unhexlify(P)) + self.P = podle_PublicKey(P) else: self.P = None else: if P: raise PoDLEError("Pubkey should not be provided with privkey") #any other formatting abnormality will just throw in PrivateKey - if len(priv) == 66 and priv[-2:] == '01': - priv = priv[:-2] - self.priv = podle_PrivateKey(binascii.unhexlify(priv)) + if len(priv) == 33 and priv[-1:] == b"\x01": + priv = priv[:-1] + self.priv = podle_PrivateKey(priv) self.P = self.priv.public_key if P2: - self.P2 = podle_PublicKey(binascii.unhexlify(P2)) + self.P2 = podle_PublicKey(P2) else: self.P2 = None - #These sig values should be passed in hex. self.s = None self.e = None if s: - self.s = binascii.unhexlify(s) + self.s = s if e: - self.e = binascii.unhexlify(e) + self.e = e #Optionally maintain usage state (boolean) self.used = used #the H(P2) value @@ -79,14 +78,13 @@ class PoDLE(object): def get_commitment(self): """Set the commitment to sha256(serialization of public key P2) - Return in hex to calling function """ if not self.P2: raise PoDLEError("Cannot construct commitment, no P2 available") if not isinstance(self.P2, podle_PublicKey_class): raise PoDLEError("Cannot construct commitment, P2 is not a pubkey") self.commitment = hashlib.sha256(self.P2.format()).digest() - return binascii.hexlify(self.commitment).decode('ascii') + return self.commitment def generate_podle(self, index=0, k=None): """Given a raw private key, in hex format, @@ -117,79 +115,85 @@ class PoDLE(object): and the associated signature data that can be used to open the commitment. """ + self.i = index #TODO nonce could be rfc6979? if not k: k = os.urandom(32) - J = getNUMS(index) + J = getNUMS(self.i) KG = podle_PrivateKey(k).public_key - KJ = multiply(k, J.format(), False, return_serialized=False) + KJ = multiply(k, J.format(), return_serialized=False) self.P2 = getP2(self.priv, J) self.get_commitment() self.e = hashlib.sha256(b''.join([x.format( ) for x in [KG, KJ, self.P, self.P2]])).digest() - k_int = decode(k, 256) - priv_int = decode(self.priv.secret, 256) - e_int = decode(self.e, 256) + k_int, priv_int, e_int = (int.from_bytes(x, + byteorder="big") for x in [k, self.priv.secret, self.e]) sig_int = (k_int + priv_int * e_int) % N - self.s = encode(sig_int, 256, minlen=32) + self.s = (sig_int).to_bytes(32, byteorder="big") return self.reveal() def reveal(self): """Encapsulate all the data representing the proof - in a dict for client functions. Data output in hex. + in a dict for client functions. """ if not all([self.u, self.P, self.P2, self.s, self.e]): raise PoDLEError("Cannot generate proof, data is missing") if not self.commitment: self.get_commitment() - Phex, P2hex, shex, ehex, commit = [ - binascii.hexlify(x).decode('ascii') - for x in [self.P.format(), self.P2.format(), self.s, self.e, - self.commitment] - ] - return {'used': str(self.used), + return {'used': self.used, 'utxo': self.u, - 'P': Phex, - 'P2': P2hex, - 'commit': commit, - 'sig': shex, - 'e': ehex} + 'P': self.P.format(), + 'P2': self.P2.format(), + 'commit': self.commitment, + 'sig': self.s, + 'e': self.e} def serialize_revelation(self, separator='|'): + """ Outputs the over-the-wire format as used in + Joinmarket communication protocol. + """ state_dict = self.reveal() - ser_list = [] - for k in ['utxo', 'P', 'P2', 'sig', 'e']: - ser_list += [state_dict[k]] + success, utxo = utxo_to_utxostr(state_dict["utxo"]) + assert success, "invalid utxo in PoDLE" + ser_list = [utxo] + ser_list += [bintohex(state_dict[x]) for x in ["P", "P2", "sig", "e"]] ser_string = separator.join(ser_list) return ser_string @classmethod def deserialize_revelation(cls, ser_rev, separator='|'): + """ Reads the over-the-wire format as used in + Joinmarket communication protocol. + """ ser_list = ser_rev.split(separator) if len(ser_list) != 5: raise PoDLEError("Failed to deserialize, wrong format") - utxo, P, P2, s, e = ser_list - return {'utxo': utxo, 'P': P, 'P2': P2, 'sig': s, 'e': e} + utxostr, P, P2, s, e = ser_list + success, utxo = utxostr_to_utxo(utxostr) + assert success, "invalid utxo format in PoDLE." + return {'utxo': utxo, 'P': hextobin(P), + 'P2': hextobin(P2), 'sig': hextobin(s), 'e': hextobin(e)} def verify(self, commitment, index_range): """For an object created without a private key, check that the opened commitment verifies for at least - one NUMS point as defined by the range in index_range + one NUMS point as defined by the range in index_range. """ if not all([self.P, self.P2, self.s, self.e]): raise PoDLEError("Verify called without sufficient data") if not self.get_commitment() == commitment: return False + for J in [getNUMS(i) for i in index_range]: sig_priv = podle_PrivateKey(self.s) sG = sig_priv.public_key - sJ = multiply(self.s, J.format(), False) - e_int = decode(self.e, 256) - minus_e = encode(-e_int % N, 256, minlen=32) - minus_e_P = multiply(minus_e, self.P.format(), False) - minus_e_P2 = multiply(minus_e, self.P2.format(), False) - KGser = add_pubkeys([sG.format(), minus_e_P], False) - KJser = add_pubkeys([sJ, minus_e_P2], False) + sJ = multiply(self.s, J.format()) + e_int = int.from_bytes(self.e, byteorder="big") + minus_e = (-e_int % N).to_bytes(32, byteorder="big") + minus_e_P = multiply(minus_e, self.P.format()) + minus_e_P2 = multiply(minus_e, self.P2.format()) + KGser = add_pubkeys([sG.format(), minus_e_P]) + KJser = add_pubkeys([sJ, minus_e_P2]) #check 2: e =?= H(K_G || K_J || P || P2) e_check = hashlib.sha256(KGser + KJser + self.P.format() + self.P2.format()).digest() @@ -198,6 +202,20 @@ class PoDLE(object): #commitment fails for any NUMS in the provided range return False + def __repr__(self): + """ Specified here to allow logging. + """ + # note: will throw if not fully initalised + r = self.reveal() + success, utxo = utxo_to_utxostr(r["utxo"]) + assert success, "invalid utxo in PoDLE." + return pformat({'used': r["used"], + 'utxo': utxo, + 'P': bintohex(r["P"]), + 'P2': bintohex(r["P2"]), + 'commit': bintohex(r["commit"]), + 'sig': bintohex(r["sig"]), + 'e': bintohex(r["e"])}) def getNUMS(index=0): """Taking secp256k1's G as a seed, @@ -244,7 +262,7 @@ def verify_all_NUMS(write=False): """ nums_points = {} for i in range(256): - nums_points[i] = binascii.hexlify(getNUMS(i).format()).decode('ascii') + nums_points[i] = bintohex(getNUMS(i).format()) if write: with open("nums_basepoints.txt", "wb") as f: from pprint import pformat @@ -263,13 +281,82 @@ def getP2(priv, nums_pt): priv_raw = priv.secret return multiply(priv_raw, nums_pt.format(), - False, return_serialized=False) +# functions which interact with the external persistence of podle data: + +def switch_external_dict_format(ed, utxo_converter, hexbinconverter): + """External dict has structure: + {txid:N:{'P':pubkey, 'reveal':{1:{'P2':P2,'s':s,'e':e}, 2:{..},..}}} + This function switches between readable/writable in file (strings, hex) + and that used internally. + """ + retval = {} + for u in ed: + success, u2 = utxo_converter(u) + assert success, "invalid utxo format in external dict parsing." + retval[u2] = {"P": hexbinconverter(ed[u]["P"])} + retval[u2]["reveal"] = {} + for i in ed[u]["reveal"]: + # hack: python json does not allow int dict keys, they must + # be str, so when reading from file, we convert back to int: + if not isinstance(i, int): + j = int(i) + else: + j = i + retval[u2]["reveal"][j] = { + "P2": hexbinconverter(ed[u]["reveal"][i]["P2"]), + "s": hexbinconverter(ed[u]["reveal"][i]["s"]), + "e": hexbinconverter(ed[u]["reveal"][i]["e"])} + return retval + +def external_dict_to_file(ed): + """ Converts internal format of dict to one writable/readable + in file. + """ + return switch_external_dict_format(ed, utxo_to_utxostr, bintohex) + +def external_dict_from_file(ed): + """ Takes the external dict extracted through json deserialization + from a file and converts it to internal format: + {txid:N:{'P':pubkey, 'reveal':{1:{'P2':P2,'s':s,'e':e}, 2:{..},..}}} + """ + return switch_external_dict_format(ed, utxostr_to_utxo, hextobin) + +def write_to_podle_file(used, external): + """ Update persisted commitment data in PODLE_COMMIT_FILE. + """ + to_write = {} + to_write['used'] = [bintohex(x) for x in used] + externalfmt = external_dict_to_file(external) + to_write['external'] = externalfmt + with open(PODLE_COMMIT_FILE, "wb") as f: + f.write(json.dumps(to_write, indent=4).encode('utf-8')) + +def read_from_podle_file(): + """ Returns used commitment list and external commitments dict + struct currently stored in PODLE_COMMIT_FILE. + """ + if os.path.isfile(PODLE_COMMIT_FILE): + with open(PODLE_COMMIT_FILE, "rb") as f: + try: + c = json.loads(f.read().decode('utf-8')) + except ValueError: #pragma: no cover + #Exit conditions cannot be included in tests. + jmprint("the file: " + PODLE_COMMIT_FILE + " is not valid json.", + "error") + sys.exit(EXIT_FAILURE) + if 'used' not in c.keys() or 'external' not in c.keys(): + raise PoDLEError("Incorrectly formatted file: " + PODLE_COMMIT_FILE) + + used = [hextobin(x) for x in c["used"]] + external = external_dict_from_file(c["external"]) + return (used, external) + return ([], {}) def get_podle_commitments(): """Returns set of commitments used as a list: - [H(P2),..] (hex) and a dict of all existing external commitments. + [H(P2),..] and a dict of all existing external commitments. It is presumed that each H(P2) can be used only once (this may not literally be true, but represents good joinmarket "citizenship"). @@ -280,18 +367,11 @@ def get_podle_commitments(): """ if not os.path.isfile(PODLE_COMMIT_FILE): return ([], {}) - with open(PODLE_COMMIT_FILE, "rb") as f: - c = json.loads(f.read().decode('utf-8')) - if 'used' not in c.keys() or 'external' not in c.keys(): - raise PoDLEError("Incorrectly formatted file: " + PODLE_COMMIT_FILE) - return (c['used'], c['external']) - + return read_from_podle_file() def add_external_commitments(ecs): """To allow external functions to add - PoDLE commitments that were calculated elsewhere; - the format of each entry in ecs must be: - {txid:N:{'P':pubkey, 'reveal':{1:{'P2':P2,'s':s,'e':e}, 2:{..},..}}} + PoDLE commitments that were calculated elsewhere. """ update_commitments(external_to_add=ecs) @@ -304,25 +384,7 @@ def update_commitments(commitment=None, whose key value is the utxo in external_to_remove, persist updated entries to disk. """ - c = {} - if os.path.isfile(PODLE_COMMIT_FILE): - with open(PODLE_COMMIT_FILE, "rb") as f: - try: - c = json.loads(f.read().decode('utf-8')) - except ValueError: #pragma: no cover - #Exit conditions cannot be included in tests. - jmprint("the file: " + PODLE_COMMIT_FILE + " is not valid json.", - "error") - sys.exit(EXIT_FAILURE) - - if 'used' in c: - commitments = c['used'] - else: - commitments = [] - if 'external' in c: - external = c['external'] - else: - external = {} + commitments, external = read_from_podle_file() if commitment: commitments.append(commitment) #remove repeats @@ -334,11 +396,7 @@ def update_commitments(commitment=None, } if external_to_add: external.update(external_to_add) - to_write = {} - to_write['used'] = commitments - to_write['external'] = external - with open(PODLE_COMMIT_FILE, "wb") as f: - f.write(json.dumps(to_write, indent=4).encode('utf-8')) + write_to_podle_file(commitments, external) def get_podle_tries(utxo, priv=None, max_tries=1, external=False): used_commitments, external_commitments = get_podle_commitments() @@ -349,9 +407,8 @@ def get_podle_tries(utxo, priv=None, max_tries=1, external=False): #use as many as were provided in the file, up to a max of max_tries m = min([len(ec['reveal'].keys()), max_tries]) for i in reversed(range(m)): - key = str(i) - p = PoDLE(u=utxo,P=ec['P'],P2=ec['reveal'][key]['P2'], - s=ec['reveal'][key]['s'], e=ec['reveal'][key]['e']) + p = PoDLE(u=utxo, P=ec["P"], P2=ec["reveal"][i]["P2"], + s=ec["reveal"][i]["s"], e=ec["reveal"][i]["e"]) if p.get_commitment() in used_commitments: return i+1 else: @@ -387,10 +444,10 @@ def generate_podle(priv_utxo_pairs, max_tries=1, allow_external=None, k=None): #which is still available. index = tries p = PoDLE(u=utxo, priv=priv) - c = p.generate_podle(index) + p.generate_podle(index) #persist for future checks - update_commitments(commitment=c['commit']) - return c + update_commitments(commitment=p.commitment) + return p if allow_external: for u in allow_external: tries = get_podle_tries(utxo=u, max_tries=max_tries, external=True) @@ -400,19 +457,17 @@ def generate_podle(priv_utxo_pairs, max_tries=1, allow_external=None, k=None): #remove this entry update_commitments(external_to_remove=u) continue - index = str(tries) ec = external_commitments[u] - p = PoDLE(u=u,P=ec['P'],P2=ec['reveal'][index]['P2'], - s=ec['reveal'][index]['s'], e=ec['reveal'][index]['e']) + ecri = ec["reveal"][tries] + p = PoDLE(u=u, P=ec["P"], P2=ecri["P2"], s=ecri["s"], e=ecri["e"]) update_commitments(commitment=p.get_commitment()) - return p.reveal() + return p #Failed to find any non-used valid commitment: return None def verify_podle(Pser, P2ser, sig, e, commitment, index_range=range(10)): verifying_podle = PoDLE(P=Pser, P2=P2ser, s=sig, e=e) - #check 1: Hash(P2ser) =?= commitment if not verifying_podle.verify(commitment, index_range): return False return True diff --git a/jmclient/jmclient/taker.py b/jmclient/jmclient/taker.py index 22d1e44..9ee185a 100644 --- a/jmclient/jmclient/taker.py +++ b/jmclient/jmclient/taker.py @@ -5,12 +5,10 @@ import base64 import pprint import random from twisted.internet import reactor, task -from binascii import hexlify, unhexlify -from jmbitcoin import SerializationError, SerializationTruncationError import jmbitcoin as btc from jmclient.configure import jm_single, validate_address -from jmbase.support import get_log +from jmbase import get_log, hextobin, bintohex, hexbin from jmclient.support import (calc_cj_fee, weighted_order_choose, choose_orders, choose_sweep_orders) from jmclient.wallet import estimate_tx_fee, compute_tx_locktime @@ -238,7 +236,6 @@ class Taker(object): #Initialization has been successful. We must set the nonrespondants #now to keep track of what changed when we receive the utxo data self.nonrespondants = list(self.orderbook.keys()) - return (True, self.cjamount, commitment, revelation, self.orderbook) def filter_orderbook(self, orderbook, sweep=False): @@ -344,6 +341,7 @@ class Taker(object): self.utxos = {None: list(self.input_utxos.keys())} return True + @hexbin def receive_utxos(self, ioauth_data): """Triggered when the daemon returns utxo data from makers who responded; this is the completion of phase 1 @@ -354,6 +352,7 @@ class Taker(object): #Temporary list used to aggregate all ioauth data that must be removed rejected_counterparties = [] + #Need to authorize against the btc pubkey first. for nick, nickdata in iteritems(ioauth_data): utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata @@ -376,10 +375,9 @@ class Taker(object): self.maker_utxo_data = {} for nick, nickdata in iteritems(ioauth_data): - utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata + utxo_list, auth_pub, cj_addr, change_addr, _, _ = nickdata + utxo_data = jm_single().bc_interface.query_utxo_set(utxo_list) self.utxos[nick] = utxo_list - utxo_data = jm_single().bc_interface.query_utxo_set(self.utxos[ - nick]) if None in utxo_data: jlog.warn(('ERROR outputs unconfirmed or already spent. ' 'utxo_data={}').format(pprint.pformat(utxo_data))) @@ -391,13 +389,12 @@ class Taker(object): #Extract the address fields from the utxos #Construct the Bitcoin address for the auth_pub field #Ensure that at least one address from utxos corresponds. - auth_pub_bin = unhexlify(auth_pub) for inp in utxo_data: try: if self.wallet_service.pubkey_has_script( - auth_pub_bin, unhexlify(inp['script'])): + auth_pub, inp['script']): break - except EngineError: + except EngineError as e: pass else: jlog.warn("ERROR maker's (" + nick + ")" @@ -500,81 +497,85 @@ class Taker(object): else: self.outputs.append({'address': self.my_change_addr, 'value': my_change_value}) - self.utxo_tx = [dict([('output', u)]) - for u in sum(self.utxos.values(), [])] + self.utxo_tx = [u for u in sum(self.utxos.values(), [])] self.outputs.append({'address': self.coinjoin_address(), 'value': self.cjamount}) - tx = btc.make_shuffled_tx(self.utxo_tx, self.outputs, False) - jlog.info('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) + self.latest_tx = btc.make_shuffled_tx(self.utxo_tx, self.outputs) + jlog.info('obtained tx\n' + bintohex(self.latest_tx.serialize())) - self.latest_tx = btc.deserialize(tx) - for index, ins in enumerate(self.latest_tx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + for index, ins in enumerate(self.latest_tx.vin): + utxo = (ins.prevout.hash[::-1], ins.prevout.n) if utxo not in self.input_utxos.keys(): continue # placeholders required - ins['script'] = 'deadbeef' + ins.scriptSig = btc.CScript.fromhex("deadbeef") self.taker_info_callback("INFO", "Built tx, sending to counterparties.") - return (True, list(self.maker_utxo_data.keys()), tx) + return (True, list(self.maker_utxo_data.keys()), + bintohex(self.latest_tx.serialize())) + @hexbin def auth_counterparty(self, btc_sig, auth_pub, maker_pk): """Validate the counterpartys claim to own the btc address/pubkey that will be used for coinjoining with an ecdsa verification. """ try: - if not btc.ecdsa_verify(maker_pk, btc_sig, auth_pub): + # maker pubkey as message is in hex format: + if not btc.ecdsa_verify(bintohex(maker_pk), btc_sig, auth_pub): jlog.debug('signature didnt match pubkey and message') return False except Exception as e: - jlog.info("Failed ecdsa verify for maker pubkey: " + str(maker_pk)) + jlog.info("Failed ecdsa verify for maker pubkey: " + bintohex(maker_pk)) jlog.info("Exception was: " + repr(e)) return False return True def on_sig(self, nick, sigb64): """Processes transaction signatures from counterparties. - Returns True if all signatures received correctly, else - returns False + If all signatures received correctly, returns the result + of self.self_sign_and_push() (i.e. we complete the signing + and broadcast); else returns False (thus returns False for + all but last signature). """ if self.aborted: return False if nick not in self.nonrespondants: jlog.debug(('add_signature => nick={} ' 'not in nonrespondants {}').format(nick, self.nonrespondants)) - return - sig = hexlify(base64.b64decode(sigb64)).decode('ascii') + return False + sig = base64.b64decode(sigb64) inserted_sig = False - txhex = btc.serialize(self.latest_tx) # batch retrieval of utxo data utxo = {} ctr = 0 - for index, ins in enumerate(self.latest_tx['ins']): - utxo_for_checking = ins['outpoint']['hash'] + ':' + str(ins[ - 'outpoint']['index']) - #'deadbeef' markers mean our own input scripts are not '' - if (ins['script'] != ''): + for index, ins in enumerate(self.latest_tx.vin): + utxo_for_checking = (ins.prevout.hash[::-1], ins.prevout.n) + # 'deadbeef' markers mean our own input scripts are not queried + if ins.scriptSig != b"": continue utxo[ctr] = [index, utxo_for_checking] ctr += 1 utxo_data = jm_single().bc_interface.query_utxo_set([x[ 1] for x in utxo.values()]) - # insert signatures for i, u in iteritems(utxo): if utxo_data[i] is None: continue - #Check if the sender serialize_scripted the scriptCode - #item into the sig message; if so, also pick up the amount - #from the utxo data retrieved from the blockchain to verify - #the segwit-style signature. Note that this allows a mixed - #SW/non-SW transaction as each utxo is interpreted separately. - sig_deserialized = btc.deserialize_script(sig) - #verify_tx_input will not even parse the script if it has integers or None, - #so abort in case we were given a junk sig: - if not all([not isinstance(x, int) and x for x in sig_deserialized]): - jlog.warn("Junk signature: " + str(sig_deserialized) + \ + # Check if the sender included the scriptCode in the sig message; + # if so, also pick up the amount from the utxo data retrieved + # from the blockchain to verify the segwit-style signature. + # Note that this allows a mixed SW/non-SW transaction as each utxo + # is interpreted separately. + try: + sig_deserialized = [a for a in iter(btc.CScript(sig))] + except Exception as e: + jlog.debug("Failed to parse junk sig message, ignoring.") + break + # abort in case we were given a junk sig (note this previously had + # to check to avoid crashes in verify_tx_input, no longer (Feb 2020)): + if not all([x for x in sig_deserialized]): + jlog.debug("Junk signature: " + str(sig_deserialized) + \ ", not attempting to verify") break if len(sig_deserialized) == 2: @@ -583,44 +584,48 @@ class Taker(object): elif len(sig_deserialized) == 3: ver_sig, ver_pub, scriptCode = sig_deserialized else: - jlog.debug("Invalid signature message - more than 3 items") + jlog.debug("Invalid signature message - not 2 or 3 items") break + ver_amt = utxo_data[i]['value'] if scriptCode else None - sig_good = btc.verify_tx_input(txhex, u[0], utxo_data[i]['script'], - ver_sig, ver_pub, scriptCode=scriptCode, amount=ver_amt) - - if ver_amt is not None and not sig_good: - # Special case to deal with legacy bots 0.5.0 or lower: - # the third field in the sigmessage was originally *not* the - # scriptCode, but the contents of tx['ins'][index]['script'], - # i.e. the witness program 0014... ; for this we can verify - # implicitly, as verify_tx_input used to, by reconstructing - # from the public key. For these cases, we can *assume* that - # the input is of type p2sh-p2wpkh; we call the jmbitcoin method - # directly, as we cannot assume that *our* wallet handles this. - scriptCode = hexlify(btc.pubkey_to_p2pkh_script( - ver_pub, True)).decode('ascii') - sig_good = btc.verify_tx_input(txhex, u[0], utxo_data[i]['script'], - ver_sig, ver_pub, scriptCode=scriptCode, amount=ver_amt) + witness = btc.CScriptWitness( + [ver_sig, ver_pub]) if scriptCode else None + + # don't attempt to parse `pub` as pubkey unless it's valid. + if scriptCode: + try: + s = btc.pubkey_to_p2wpkh_script(ver_pub) + except: + jlog.debug("Junk signature message, invalid pubkey, ignoring.") + break + scriptSig = btc.CScript([ver_sig, ver_pub]) if not scriptCode else btc.CScript([s]) + + # Pre-Feb 2020, we used the third field scriptCode differently in + # pre- and post-0.5.0; now the scriptCode is implicit (i.e. calculated + # by underlying library, so that exceptional case is covered. + sig_good = btc.verify_tx_input(self.latest_tx, u[0], scriptSig, + btc.CScript(utxo_data[i]['script']), amount=ver_amt, witness=witness) + + # verification for the native case is functionally identical but + # adds another flag; so we can allow it here: + if not sig_good: + sig_good = btc.verify_tx_input(self.latest_tx, u[0], scriptSig, + btc.CScript(utxo_data[i]['script']), amount=ver_amt, + witness=witness, native=True) + # if passes, below code executes, and we should change for native: + scriptSig = btc.CScript([b""]) if sig_good: jlog.debug('found good sig at index=%d' % (u[0])) + + # Note that, due to the complexity of handling multisig or other + # arbitrary script (considering sending multiple signatures OTW), + # there is an assumption of p2sh-p2wpkh or p2wpkh, for the segwit + # case. + self.latest_tx.vin[u[0]].scriptSig = scriptSig if ver_amt: - # Note that, due to the complexity of handling multisig or other - # arbitrary script (considering sending multiple signatures OTW), - # there is an assumption of p2sh-p2wpkh or p2wpkh, for the segwit - # case. - self.latest_tx["ins"][u[0]]["txinwitness"] = [ver_sig, ver_pub] - if btc.is_segwit_native_script(utxo_data[i]['script']): - scriptSig = "" - else: - scriptSig = btc.serialize_script_unit( - btc.pubkey_to_p2wpkh_script(ver_pub)) - self.latest_tx["ins"][u[0]]["script"] = scriptSig - else: - # Non segwit (as per above comments) is limited only to single key, - # p2pkh case. - self.latest_tx["ins"][u[0]]["script"] = sig + self.latest_tx.wit.vtxinwit[u[0]] = btc.CTxInWitness( + btc.CScriptWitness(witness)) inserted_sig = True # check if maker has sent everything possible @@ -643,8 +648,8 @@ class Taker(object): # other guy sent a failed signature tx_signed = True - for ins in self.latest_tx['ins']: - if ins['script'] == '': + for ins in self.latest_tx.vin: + if ins.scriptSig == b"": tx_signed = False if not tx_signed: return False @@ -667,7 +672,7 @@ class Taker(object): def filter_by_coin_age_amt(utxos, age, amt): results = jm_single().bc_interface.query_utxo_set(utxos, - includeconf=True) + includeconf=True) newresults = [] too_old = [] too_small = [] @@ -683,7 +688,6 @@ class Taker(object): too_small.append(utxos[i]) if valid_age and valid_amt: newresults.append(utxos[i]) - return newresults, too_old, too_small def priv_utxo_pairs_from_utxos(utxos, age, amt): @@ -695,7 +699,7 @@ class Taker(object): age, amt) new_utxos_dict = {k: v for k, v in utxos.items() if k in new_utxos} for k, v in iteritems(new_utxos_dict): - addr = v['address'] + addr = self.wallet_service.script_to_addr(v["script"]) priv = self.wallet_service.get_key_from_addr(addr) if priv: #can be null from create-unsigned priv_utxo_pairs.append((priv, k)) @@ -723,7 +727,7 @@ class Taker(object): #in the transaction, about to be consumed, rather than use #random utxos that will persist after. At this step we also #allow use of external utxos in the json file. - if any(self.wallet_service.get_utxos_by_mixdepth(hexfmt=False).values()): + if any(self.wallet_service.get_utxos_by_mixdepth().values()): utxos = {} for mdutxo in self.wallet_service.get_utxos_by_mixdepth().values(): utxos.update(mdutxo) @@ -739,13 +743,9 @@ class Taker(object): ext_valid = None podle_data = generate_podle(priv_utxo_pairs, tries, ext_valid) if podle_data: - jlog.debug("Generated PoDLE: " + pprint.pformat(podle_data)) - revelation = PoDLE(u=podle_data['utxo'], - P=podle_data['P'], - P2=podle_data['P2'], - s=podle_data['sig'], - e=podle_data['e']).serialize_revelation() - return (commit_type_byte + podle_data["commit"], revelation, + jlog.debug("Generated PoDLE: " + repr(podle_data)) + return (commit_type_byte + bintohex(podle_data.commitment), + podle_data.serialize_revelation(), "Commitment sourced OK") else: errmsgheader, errmsg = generate_podle_error_string(priv_utxo_pairs, @@ -771,25 +771,26 @@ class Taker(object): def self_sign(self): # now sign it ourselves our_inputs = {} - for index, ins in enumerate(self.latest_tx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + for index, ins in enumerate(self.latest_tx.vin): + utxo = (ins.prevout.hash[::-1], ins.prevout.n) if utxo not in self.input_utxos.keys(): continue - script = self.wallet_service.addr_to_script(self.input_utxos[utxo]['address']) + script = self.input_utxos[utxo]["script"] amount = self.input_utxos[utxo]['value'] our_inputs[index] = (script, amount) - self.latest_tx = self.wallet_service.sign_tx(self.latest_tx, our_inputs) + success, msg = self.wallet_service.sign_tx(self.latest_tx, our_inputs) + if not success: + jlog.error("Failed to sign transaction: " + msg) def push(self): - tx = btc.serialize(self.latest_tx) - jlog.debug('\n' + tx) - self.txid = btc.txhash(tx) + jlog.debug('\n' + bintohex(self.latest_tx.serialize())) + self.txid = bintohex(self.latest_tx.GetTxid()[::-1]) jlog.info('txid = ' + self.txid) #If we are sending to a bech32 address, in case of sweep, will #need to use that bech32 for address import, which requires #converting to script (Core does not allow import of bech32) if self.my_cj_addr.lower()[:2] in ['bc', 'tb']: - notify_addr = btc.address_to_script(self.my_cj_addr) + notify_addr = btc.CCoinAddress(self.my_cj_addr).to_scriptPubKey() else: notify_addr = self.my_cj_addr #add the callbacks *before* pushing to ensure triggering; @@ -810,7 +811,7 @@ class Taker(object): tx_broadcast = jm_single().config.get('POLICY', 'tx_broadcast') nick_to_use = None if tx_broadcast == 'self': - pushed = jm_single().bc_interface.pushtx(tx) + pushed = jm_single().bc_interface.pushtx(self.latest_tx.serialize()) elif tx_broadcast in ['random-peer', 'not-self']: n = len(self.maker_utxo_data) if tx_broadcast == 'random-peer': @@ -818,14 +819,14 @@ class Taker(object): else: i = random.randrange(n) if i == n: - pushed = jm_single().bc_interface.pushtx(tx) + pushed = jm_single().bc_interface.pushtx(self.latest_tx.serialize()) else: nick_to_use = list(self.maker_utxo_data.keys())[i] pushed = True else: jlog.info("Only self, random-peer and not-self broadcast " "methods supported. Reverting to self-broadcast.") - pushed = jm_single().bc_interface.pushtx(tx) + pushed = jm_single().bc_interface.pushtx(self.latest_tx.serialize()) if not pushed: self.on_finished_callback(False, fromtx=True) else: @@ -841,7 +842,7 @@ class Taker(object): # Takers process only in series, so this should not occur: assert self.latest_tx is not None # check if the transaction matches our created tx: - if txd['outs'] != self.latest_tx['outs']: + if txd.vout != self.latest_tx.vout: return False return True @@ -928,12 +929,12 @@ class P2EPTaker(Taker): # For the p2ep taker, the variable 'my_cj_addr' is the destination: self.my_cj_addr = si[3] if isinstance(self.cjamount, float): - raise JMTakerError("P2EP coinjoin must use amount in satoshis") + raise JMTakerError("Payjoin must use amount in satoshis") if self.cjamount == 0: # Note that we don't allow sweep, currently, since the coin # choosing algo would not apply in that case (we'd have to rewrite # prepare_my_bitcoin_data for that case). - raise JMTakerError("P2EP coinjoin does not currently support sweep") + raise JMTakerError("Payjoin does not currently support sweep") # Next we prepare our coins with the inherited method # for this purpose; for this we must set the @@ -1004,27 +1005,27 @@ class P2EPTaker(Taker): if self.my_change_addr is not None: self.outputs.append({'address': self.my_change_addr, 'value': my_change_value}) - # As for JM coinjoins, the `None` key is used for our own inputs - # to the transaction; this preparatory version contains only those. - tx = btc.make_shuffled_tx(self.utxos[None], self.outputs, - False, 2, compute_tx_locktime()) - jlog.info('Created proposed fallback tx\n' + pprint.pformat( - btc.deserialize(tx))) + # Oour own inputs to the transaction; this preparatory version + # contains only those. + tx = btc.make_shuffled_tx(self.input_utxos, self.outputs, + version=2, locktime=compute_tx_locktime()) + jlog.info('Created proposed fallback tx\n' + pprint.pformat(str(tx))) # We now sign as a courtesy, because if we disappear the recipient # can still claim his coins with this. # sign our inputs before transfer our_inputs = {} - dtx = btc.deserialize(tx) - for index, ins in enumerate(dtx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - script = self.wallet_service.addr_to_script(self.input_utxos[utxo]['address']) - amount = self.input_utxos[utxo]['value'] - our_inputs[index] = (script, amount) - self.signed_noncj_tx = btc.serialize(self.wallet_service.sign_tx(dtx, our_inputs)) + for index, ins in enumerate(tx.vin): + utxo = (ins.prevout.hash[::-1], ins.prevout.n) + our_inputs[index] = (self.input_utxos[utxo]["script"], + self.input_utxos[utxo]['value']) + success, msg = self.wallet_service.sign_tx(tx, our_inputs) + if not success: + jlog.error("Failed to create backup transaction; error: " + msg) self.taker_info_callback("INFO", "Built tx proposal, sending to receiver.") - return (True, [self.p2ep_receiver_nick], self.signed_noncj_tx) + return (True, [self.p2ep_receiver_nick], bintohex(tx.serialize())) - def on_tx_received(self, nick, txhex): + @hexbin + def on_tx_received(self, nick, txser): """ Here the taker (payer) retrieves a version of the transaction from the maker (receiver) which should have the following properties: @@ -1041,12 +1042,14 @@ class P2EPTaker(Taker): and then broadcast (TODO broadcast delay or don't broadcast). """ try: - tx = btc.deserialize(txhex) - except (IndexError, SerializationError, SerializationTruncationError) as e: + tx = btc.CMutableTransaction.deserialize(txser) + except Exception as e: return (False, "malformed txhex. " + repr(e)) - jlog.info("Obtained tx from receiver:\n" + pprint.pformat(tx)) - cjaddr_script = btc.address_to_script(self.my_cj_addr) - changeaddr_script = btc.address_to_script(self.my_change_addr) + jlog.info("Obtained tx from receiver:\n" + pprint.pformat(str(tx))) + cjaddr_script = btc.CCoinAddress( + self.my_cj_addr).to_scriptPubKey() + changeaddr_script = btc.CCoinAddress( + self.my_change_addr).to_scriptPubKey() # We ensure that the coinjoin address and our expected change # address are still in the outputs, once (with the caveat that @@ -1054,19 +1057,19 @@ class P2EPTaker(Taker): # of dust change, which we assess after). times_seen_cj_addr = 0 times_seen_change_addr = 0 - for outs in tx['outs']: - if outs['script'] == cjaddr_script: + for outs in tx.vout: + if outs.scriptPubKey == cjaddr_script: times_seen_cj_addr += 1 - new_cj_amount = outs['value'] + new_cj_amount = outs.nValue if new_cj_amount < self.cjamount: # This is a violation of protocol; # receiver must be providing extra bitcoin # as input, so his receiving amount should have increased. return (False, 'Wrong cj_amount. I expect at least' + str(self.cjamount)) - if outs['script'] == changeaddr_script: + if outs.scriptPubKey == changeaddr_script: times_seen_change_addr += 1 - new_change_amount = outs['value'] + new_change_amount = outs.nValue if times_seen_cj_addr != 1: fmt = ('cj addr not in tx outputs once, #cjaddr={}').format return (False, (fmt(times_seen_cj_addr))) @@ -1077,8 +1080,7 @@ class P2EPTaker(Taker): new_change_amount = 0 # Check that our inputs are present. - tx_utxo_set = set(ins['outpoint']['hash'] + ':' + str( - ins['outpoint']['index']) for ins in tx['ins']) + tx_utxo_set = set((ins.prevout.hash[::-1], ins.prevout.n) for ins in tx.vin) if not tx_utxo_set.issuperset(set(self.utxos[None])): return (False, "my utxos are not contained") # Check that the sequence numbers of all inputs are unaltered @@ -1087,10 +1089,10 @@ class P2EPTaker(Taker): # Note that this is hacky and is most elegantly addressed by # use of PSBT (although any object encapsulation of tx input # would serve the same purpose). - if tx["locktime"] == 0: + if tx.nLockTime == 0: return (False, "Invalid PayJoin v0 transaction: locktime 0") - for i in tx["ins"]: - if i["sequence"] != 0xffffffff - 1: + for i in tx.vin: + if i.nSequence != 0xffffffff - 1: return (False, "Invalid PayJoin v0 transaction: "+\ "sequence is not 0xffffffff -1") @@ -1099,7 +1101,7 @@ class P2EPTaker(Taker): # not) of PayJoin to sweep utxos at no cost. # (TODO This is very kludgy, more sophisticated approach # should be used in future): - if len(tx["ins"]) - len (self.utxos[None]) > 5: + if len(tx.vin) - len (self.utxos[None]) > 5: return (False, "proposed tx has more than 5 inputs from " "the recipient, which is too expensive.") @@ -1122,9 +1124,8 @@ class P2EPTaker(Taker): # checking input validity and transaction balance. retrieve_utxos = {} ctr = 0 - for index, ins in enumerate(tx['ins']): - utxo_for_checking = ins['outpoint']['hash'] + ':' + str( - ins['outpoint']['index']) + for index, ins in enumerate(tx.vin): + utxo_for_checking = (ins.prevout.hash[::-1], ins.prevout.n) if utxo_for_checking in self.utxos[None]: continue retrieve_utxos[ctr] = [index, utxo_for_checking] @@ -1140,32 +1141,11 @@ class P2EPTaker(Taker): if utxo_data[i] is None: return (False, "Proposed transaction contains invalid utxos") total_receiver_input += utxo_data[i]["value"] - scriptCode = None - ver_amt = None idx = retrieve_utxos[i][0] - if "txinwitness" in tx["ins"][idx]: - ver_amt = utxo_data[i]["value"] - try: - ver_sig, ver_pub = tx["ins"][idx]["txinwitness"] - except Exception as e: - print("Segwit error: ", repr(e)) - return (False, "Segwit input not of expected type, " - "either p2sh-p2wpkh or p2wpkh") - # note that the scriptCode is the same whether nested or not - # also note that the scriptCode has to be inferred if we are - # only given a transaction serialization. - scriptCode = "76a914" + btc.hash160(unhexlify(ver_pub)) + "88ac" - else: - scriptSig = btc.deserialize_script(tx["ins"][idx]["script"]) - if len(scriptSig) != 2: - return (False, - "Proposed transaction contains unsupported input type") - ver_sig, ver_pub = scriptSig - if not btc.verify_tx_input(txhex, idx, - utxo_data[i]['script'], - ver_sig, ver_pub, - scriptCode=scriptCode, - amount=ver_amt): + if not btc.verify_tx_input(tx, idx, tx.vin[idx].scriptSig, + btc.CScript(utxo_data[i]['script']), + amount=utxo_data[i]["value"], + witness=tx.wit.vtxinwit[idx].scriptWitness): return (False, "Proposed transaction is not correctly signed.") payment = new_cj_amount - total_receiver_input @@ -1185,7 +1165,7 @@ class P2EPTaker(Taker): # our fee estimator. Its return value will be governed by our own fee settings # in joinmarket.cfg; allow either (a) automatic agreement for any value within # a range of 0.3 to 3x this figure, or (b) user to agree on prompt. - fee_est = estimate_tx_fee(len(tx['ins']), len(tx['outs']), + fee_est = estimate_tx_fee(len(tx.vin), len(tx.vout), txtype=self.wallet_service.get_txtype()) fee_ok = False if btc_fee > 0.3 * fee_est and btc_fee < 3 * fee_est: diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index 8524018..4f32544 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -5,15 +5,13 @@ import os import sys import time import numbers - -from jmbase import get_log, jmprint +from jmbase import get_log, jmprint, bintohex from .configure import jm_single, validate_address, is_burn_destination from .schedule import human_readable_schedule_entry, tweak_tumble_schedule,\ schedule_to_text from .wallet import BaseWallet, estimate_tx_fee, compute_tx_locktime, \ FidelityBondMixin -from jmbitcoin import deserialize, make_shuffled_tx, serialize, txhash,\ - amount_to_str, mk_burn_script, bin_hash160 +from jmbitcoin import make_shuffled_tx, amount_to_str, mk_burn_script from jmbase.support import EXIT_SUCCESS log = get_log() @@ -136,13 +134,20 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, log.info("Using a fee of : " + amount_to_str(fee_est) + ".") if amount != 0: log.info("Using a change value of: " + amount_to_str(changeval) + ".") - txsigned = sign_tx(wallet_service, make_shuffled_tx( - list(utxos.keys()), outs, False, 2, tx_locktime), utxos) + tx = make_shuffled_tx(list(utxos.keys()), outs, 2, compute_tx_locktime()) + list(utxos.keys()), outs, 2, tx_locktime), utxos) + inscripts = {} + for i, txinp in enumerate(tx.vin): + u = (txinp.prevout.hash[::-1], txinp.prevout.n) + inscripts[i] = (utxos[u]["script"], utxos[u]["value"]) + success, msg = wallet_service.sign_tx(tx, inscripts) + if not success: + log.error("Failed to sign transaction, quitting. Error msg: " + msg) + return log.info("Got signed transaction:\n") - log.info(pformat(txsigned)) - tx = serialize(txsigned) + log.info(pformat(str(tx))) log.info("In serialized form (for copy-paste):") - log.info(tx) + log.info(bintohex(tx.serialize())) actual_amount = amount if amount != 0 else total_inputs_val - fee_est log.info("Sends: " + amount_to_str(actual_amount) + " to destination: " + destination) if not answeryes: @@ -151,12 +156,12 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, log.info("You chose not to broadcast the transaction, quitting.") return False else: - accepted = accept_callback(pformat(txsigned), destination, actual_amount, + accepted = accept_callback(pformat(str(tx)), destination, actual_amount, fee_est) if not accepted: return False - jm_single().bc_interface.pushtx(tx) - txid = txhash(tx) + jm_single().bc_interface.pushtx(tx.serialize()) + txid = bintohex(tx.GetTxid()[::-1]) successmsg = "Transaction sent: " + txid cb = log.info if not info_callback else info_callback cb(successmsg) diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index 5159a0d..2a62a28 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -27,7 +27,7 @@ from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH,\ from .support import get_random_bytes from . import mn_encode, mn_decode import jmbitcoin as btc -from jmbase import JM_WALLET_NAME_PREFIX +from jmbase import JM_WALLET_NAME_PREFIX, bintohex """ @@ -198,7 +198,8 @@ class UTXOManager(object): assert isinstance(index, numbers.Integral) assert isinstance(mixdepth, numbers.Integral) - return self._utxo[mixdepth].pop((txid, index)) + x = self._utxo[mixdepth].pop((txid, index)) + return x def add_utxo(self, txid, index, path, value, mixdepth, height=None): # Assumed: that we add a utxo only if we want it enabled, @@ -433,19 +434,22 @@ class BaseWallet(object): Add signatures to transaction for inputs referenced by scripts. args: - tx: transaction dict + tx: CMutableTransaction object scripts: {input_index: (output_script, amount)} kwargs: additional arguments for engine.sign_transaction returns: - input transaction dict with added signatures, hex-encoded. + True, None if success. + False, msg if signing failed, with error msg. """ for index, (script, amount) in scripts.items(): assert amount > 0 path = self.script_to_path(script) privkey, engine = self._get_key_from_path(path) - tx = btc.deserialize(engine.sign_transaction(tx, index, privkey, - amount, **kwargs)) - return tx + sig, msg = engine.sign_transaction(tx, index, privkey, + amount, **kwargs) + if not sig: + return False, msg + return True, None @deprecated def get_key_from_addr(self, addr): @@ -455,7 +459,7 @@ class BaseWallet(object): script = self._ENGINE.address_to_script(addr) path = self.script_to_path(script) privkey = self._get_key_from_path(path)[0] - return hexlify(privkey).decode('ascii') + return privkey def _get_addr_int_ext(self, address_type, mixdepth): if address_type == self.ADDRESS_TYPE_EXTERNAL: @@ -578,32 +582,19 @@ class BaseWallet(object): """ self.save() - @deprecated def remove_old_utxos(self, tx): - tx = deepcopy(tx) - for inp in tx['ins']: - inp['outpoint']['hash'] = unhexlify(inp['outpoint']['hash']) - - ret = self.remove_old_utxos_(tx) - - removed_utxos = {} - for (txid, index), val in ret.items(): - val['address'] = self.get_address_from_path(val['path']) - removed_utxos[hexlify(txid).decode('ascii') + ':' + str(index)] = val - return removed_utxos - - def remove_old_utxos_(self, tx): """ Remove all own inputs of tx from internal utxo list. args: - tx: transaction dict + tx: CMutableTransaction returns: {(txid, index): {'script': bytes, 'path': str, 'value': int} for all removed utxos """ removed_utxos = {} - for inp in tx['ins']: - txid, index = inp['outpoint']['hash'], inp['outpoint']['index'] + for inp in tx.vin: + txid = inp.prevout.hash[::-1] + index = inp.prevout.n md = self._utxos.have_utxo(txid, index) if md is False: continue @@ -614,46 +605,32 @@ class BaseWallet(object): 'value': value} return removed_utxos - @deprecated - def add_new_utxos(self, tx, txid, height=None): - tx = deepcopy(tx) - for out in tx['outs']: - out['script'] = unhexlify(out['script']) - - ret = self.add_new_utxos_(tx, unhexlify(txid), height=height) - - added_utxos = {} - for (txid_bin, index), val in ret.items(): - addr = self.get_address_from_path(val['path']) - val['address'] = addr - added_utxos[txid + ':' + str(index)] = val - return added_utxos - - def add_new_utxos_(self, tx, txid, height=None): + def add_new_utxos(self, tx, height=None): """ Add all outputs of tx for this wallet to internal utxo list. - + They are also returned in standard dict form. args: - tx: transaction dict + tx: CMutableTransaction height: blockheight in which tx was included, or None if unconfirmed. returns: - {(txid, index): {'script': bytes, 'path': tuple, 'value': int} + {(txid, index): {'script': bytes, 'path': tuple, 'value': int, + 'address': str} for all added utxos """ - assert isinstance(txid, bytes) and len(txid) == self._utxos.TXID_LEN added_utxos = {} - for index, outs in enumerate(tx['outs']): + txid = tx.GetTxid()[::-1] + for index, outs in enumerate(tx.vout): + spk = outs.scriptPubKey + val = outs.nValue try: - self.add_utxo(txid, index, outs['script'], outs['value'], - height=height) + self.add_utxo(txid, index, spk, val, height=height) except WalletError: continue - path = self.script_to_path(outs['script']) - added_utxos[(txid, index)] = {'script': outs['script'], - 'path': path, - 'value': outs['value']} + path = self.script_to_path(spk) + added_utxos[(txid, index)] = {'script': spk, 'path': path, 'value': val, + 'address': self._ENGINE.script_to_address(spk)} return added_utxos def add_utxo(self, txid, index, script, value, height=None): @@ -669,9 +646,10 @@ class BaseWallet(object): mixdepth = self._get_mixdepth_from_path(path) self._utxos.add_utxo(txid, index, path, value, mixdepth, height=height) - def process_new_tx(self, txd, txid, height=None): - """ Given a newly seen transaction, deserialized as txd and - with transaction id, process its inputs and outputs and update + def process_new_tx(self, txd, height=None): + """ Given a newly seen transaction, deserialized as + CMutableTransaction txd, + process its inputs and outputs and update the utxo contents of this wallet accordingly. NOTE: this should correctly handle transactions that are not actually related to the wallet; it will not add (or remove, @@ -679,30 +657,15 @@ class BaseWallet(object): functions check this condition. """ removed_utxos = self.remove_old_utxos(txd) - added_utxos = self.add_new_utxos(txd, txid, height=height) + added_utxos = self.add_new_utxos(txd, height=height) return (removed_utxos, added_utxos) - @deprecated - def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None, - maxheight=None): - utxo_filter_new = None - if utxo_filter: - utxo_filter_new = [(unhexlify(utxo[:64]), int(utxo[65:])) - for utxo in utxo_filter] - ret = self.select_utxos_(mixdepth, amount, utxo_filter_new, select_fn, - maxheight=maxheight) - ret_conv = {} - for utxo, data in ret.items(): - addr = self.get_address_from_path(data['path']) - utxo_txt = hexlify(utxo[0]).decode('ascii') + ':' + str(utxo[1]) - ret_conv[utxo_txt] = {'address': addr, 'value': data['value']} - return ret_conv - - def select_utxos_(self, mixdepth, amount, utxo_filter=None, - select_fn=None, maxheight=None): + def select_utxos(self, mixdepth, amount, utxo_filter=None, + select_fn=None, maxheight=None, includeaddr=False): """ Select a subset of available UTXOS for a given mixdepth whose value is - greater or equal to amount. + greater or equal to amount. If `includeaddr` is True, adds an `address` + key to the returned dict. args: mixdepth: int, mixdepth to select utxos from, must be smaller or @@ -713,6 +676,7 @@ class BaseWallet(object): returns: {(txid, index): {'script': bytes, 'path': tuple, 'value': int}} + """ assert isinstance(mixdepth, numbers.Integral) assert isinstance(amount, numbers.Integral) @@ -728,7 +692,8 @@ class BaseWallet(object): for data in ret.values(): data['script'] = self.get_script_from_path(data['path']) - + if includeaddr: + data["address"] = self.get_address_from_path(data["path"]) return ret def disable_utxo(self, txid, index, disable=True): @@ -758,21 +723,7 @@ class BaseWallet(object): include_disabled=include_disabled, maxheight=maxheight) - @deprecated - def get_utxos_by_mixdepth(self, verbose=True, includeheight=False): - # TODO: verbose - ret = self.get_utxos_by_mixdepth_(includeheight=includeheight) - - utxos_conv = collections.defaultdict(dict) - for md, utxos in ret.items(): - for utxo, data in utxos.items(): - utxo_str = hexlify(utxo[0]).decode('ascii') + ':' + str(utxo[1]) - addr = self.get_address_from_path(data['path']) - data['address'] = addr - utxos_conv[md][utxo_str] = data - return utxos_conv - - def get_utxos_by_mixdepth_(self, include_disabled=False, includeheight=False): + def get_utxos_by_mixdepth(self, include_disabled=False, includeheight=False): """ Get all UTXOs for active mixdepths. @@ -791,9 +742,11 @@ class BaseWallet(object): if not include_disabled and self._utxos.is_disabled(*utxo): continue script = self.get_script_from_path(path) + addr = self.get_address_from_path(path) script_utxos[md][utxo] = {'script': script, 'path': path, - 'value': value} + 'value': value, + 'address': addr} if includeheight: script_utxos[md][utxo]['height'] = height return script_utxos @@ -1101,7 +1054,6 @@ class ImportWalletMixin(object): raise WalletError("Unsupported key type for imported keys.") privkey, key_type_wif = self._ENGINE.wif_to_privkey(wif) - # FIXME: there is no established standard for encoding key type in wif #if key_type is not None and key_type_wif is not None and \ # key_type != key_type_wif: diff --git a/jmclient/jmclient/wallet_service.py b/jmclient/jmclient/wallet_service.py index ef186b2..3552105 100644 --- a/jmclient/jmclient/wallet_service.py +++ b/jmclient/jmclient/wallet_service.py @@ -16,8 +16,9 @@ from jmclient.output import fmt_tx_data from jmclient.blockchaininterface import (INF_HEIGHT, BitcoinCoreInterface, BitcoinCoreNoHistoryInterface) from jmclient.wallet import FidelityBondMixin -from jmbase.support import jmprint, EXIT_SUCCESS -import jmbitcoin as btc +from jmbase.support import jmprint, EXIT_SUCCESS, utxo_to_utxostr, bintohex, hextobin +from jmbitcoin import lx + """Wallet service The purpose of this independent service is to allow @@ -173,9 +174,9 @@ class WalletService(Service): and return type boolean. `txinfo` - either a txid expected for the transaction, if known, or a tuple of the ordered output set, of the form - (('script': script), ('value': value), ..). This can be - constructed from jmbitcoin.deserialize output, key "outs", - using tuple(). See WalletService.transaction_monitor(). + ((CScript, int), ..). This is be constructed from the + CMutableTransaction vout list. + See WalletService.transaction_monitor(). `cb_type` - must be one of "all", "unconfirmed", "confirmed"; the first type will be called back once for every new transaction, the second only once when the number of @@ -223,7 +224,9 @@ class WalletService(Service): self.bci.import_addresses([address], self.EXTERNAL_WALLET_LABEL, restart_cb=self.restart_callback) - def default_autofreeze_warning_cb(self, utxostr): + def default_autofreeze_warning_cb(self, utxo): + success, utxostr = utxo_to_utxostr(utxo) + assert success, "Autofreeze warning cb called with invalid utxo." jlog.warning("WARNING: new utxo has been automatically " "frozen to prevent forced address reuse: ") jlog.warning(utxostr) @@ -267,9 +270,7 @@ class WalletService(Service): utxo]["value"] <= freeze_threshold: # freezing of coins must be communicated to user: self.autofreeze_warning_cb(utxo) - # process_new_tx returns added utxos in str format: - txidstr, idx = utxo.split(":") - self.disable_utxo(binascii.unhexlify(txidstr), int(idx)) + self.disable_utxo(*utxo) def transaction_monitor(self): """Keeps track of any changes in the wallet (new transactions). @@ -298,7 +299,7 @@ class WalletService(Service): for tx in new_txs: txid = tx["txid"] - res = self.bci.get_transaction(txid) + res = self.bci.get_transaction(hextobin(txid)) if not res: continue confs = res["confirmations"] @@ -317,7 +318,7 @@ class WalletService(Service): txd = self.bci.get_deser_from_gettransaction(res) if txd is None: continue - removed_utxos, added_utxos = self.wallet.process_new_tx(txd, txid, height) + removed_utxos, added_utxos = self.wallet.process_new_tx(txd, height) if txid not in self.processed_txids: # apply checks to disable/freeze utxos to reused addrs if needed: self.check_for_reuse(added_utxos) @@ -339,16 +340,9 @@ class WalletService(Service): f(txd, txid) # The tuple given as the second possible key for the dict - # is such because dict keys must be hashable types, so a simple - # replication of the entries in the list tx["outs"], where tx - # was generated via jmbitcoin.deserialize, is unacceptable to - # Python, since they are dicts. However their keyset is deterministic - # so it is sufficient to convert these dicts to tuples with fixed - # ordering, thus it can be used as a key into the self.callbacks - # dicts. (This is needed because txid is not always available + # is such because txid is not always available # at the time of callback registration). - possible_keys = [txid, tuple( - (x["script"], x["value"]) for x in txd["outs"])] + possible_keys = [txid, tuple((x.scriptPubKey, x.nValue) for x in txd.vout)] # note that len(added_utxos) > 0 is not a sufficient condition for # the tx being new, since wallet.add_new_utxos will happily re-add @@ -406,8 +400,9 @@ class WalletService(Service): def report_changed(x, utxos): if len(utxos.keys()) > 0: jlog.info(x + ' utxos=\n{}'.format('\n'.join( - '{} - {}'.format(u, fmt_tx_data(tx_data, self)) - for u, tx_data in utxos.items()))) + '{} - {}'.format(utxo_to_utxostr(u)[1], + fmt_tx_data(tx_data, self)) for u, + tx_data in utxos.items()))) report_changed("Removed", removed_utxos) report_changed("Added", added_utxos) @@ -752,14 +747,15 @@ class WalletService(Service): def _add_unspent_txo(self, utxo, height): """ Add a UTXO as returned by rpc's listunspent call to the wallet. - + Note that these are returned as little endian outpoint txids, so + are converted. params: utxo: single utxo dict as returned by listunspent current_blockheight: blockheight as integer, used to set the block in which a confirmed utxo is included. """ - txid = binascii.unhexlify(utxo['txid']) - script = binascii.unhexlify(utxo['scriptPubKey']) + txid = hextobin(utxo['txid']) + script = hextobin(utxo['scriptPubKey']) value = int(Decimal(str(utxo['amount'])) * Decimal('1e8')) self.add_utxo(txid, int(utxo['vout']), script, value, height) @@ -774,11 +770,9 @@ class WalletService(Service): self.wallet.save() def get_utxos_by_mixdepth(self, include_disabled=False, - verbose=False, hexfmt=True, includeconfs=False): + verbose=False, includeconfs=False): """ Returns utxos by mixdepth in a dict, optionally including information about how many confirmations each utxo has. - TODO clean up underlying wallet.get_utxos_by_mixdepth (including verbosity - and formatting options) to make this less confusing. """ def height_to_confs(x): # convert height entries to confirmations: @@ -793,36 +787,30 @@ class WalletService(Service): confs = self.current_blockheight - h + 1 ubym_conv[m][u]["confs"] = confs return ubym_conv + ubym = self.wallet.get_utxos_by_mixdepth( + include_disabled=include_disabled, includeheight=includeconfs) + if not includeconfs: + return ubym + else: + return height_to_confs(ubym) - if hexfmt: - ubym = self.wallet.get_utxos_by_mixdepth(verbose=verbose, - includeheight=includeconfs) - if not includeconfs: - return ubym - else: - return height_to_confs(ubym) + def minconfs_to_maxheight(self, minconfs): + if minconfs is None: + return None else: - ubym = self.wallet.get_utxos_by_mixdepth_( - include_disabled=include_disabled, includeheight=includeconfs) - if not includeconfs: - return ubym - else: - return height_to_confs(ubym) + return self.current_blockheight - minconfs + 1 def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None, - minconfs=None): + minconfs=None, includeaddr=False): """ Request utxos from the wallet in a particular mixdepth to satisfy a certain total amount, optionally set the selector function (or use the currently configured function set by the wallet, and optionally require a minimum of minconfs confirmations (default none means unconfirmed are allowed). """ - if minconfs is None: - maxheight = None - else: - maxheight = self.current_blockheight - minconfs + 1 return self.wallet.select_utxos(mixdepth, amount, utxo_filter=utxo_filter, - select_fn=select_fn, maxheight=maxheight) + select_fn=select_fn, maxheight=self.minconfs_to_maxheight(minconfs), + includeaddr=includeaddr) def get_balance_by_mixdepth(self, verbose=True, include_disabled=False, diff --git a/jmclient/jmclient/wallet_utils.py b/jmclient/jmclient/wallet_utils.py index 6770a9e..8aa3818 100644 --- a/jmclient/jmclient/wallet_utils.py +++ b/jmclient/jmclient/wallet_utils.py @@ -16,7 +16,8 @@ from jmclient import (get_network, WALLET_IMPLEMENTATIONS, Storage, podle, LegacyWallet, SegwitWallet, FidelityBondMixin, FidelityBondWatchonlyWallet, is_native_segwit_mode, load_program_config, add_base_options, check_regtest) from jmclient.wallet_service import WalletService -from jmbase.support import get_password, jmprint, EXIT_FAILURE, EXIT_ARGERROR +from jmbase.support import (get_password, jmprint, EXIT_FAILURE, + EXIT_ARGERROR, utxo_to_utxostr) from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH, \ TYPE_SEGWIT_LEGACY_WALLET_FIDELITY_BONDS @@ -369,22 +370,26 @@ def wallet_showutxos(wallet, showprivkey): utxos = wallet.get_utxos_by_mixdepth(includeconfs=True) for md in utxos: for u, av in utxos[md].items(): + success, us = utxo_to_utxostr(u) + assert success key = wallet.get_key_from_addr(av['address']) tries = podle.get_podle_tries(u, key, max_tries) tries_remaining = max(0, max_tries - tries) - unsp[u] = {'address': av['address'], 'value': av['value'], + unsp[us] = {'address': av['address'], 'value': av['value'], 'tries': tries, 'tries_remaining': tries_remaining, 'external': False, 'confirmations': av['confs']} if showprivkey: - unsp[u]['privkey'] = wallet.get_wif_path(av['path']) + unsp[us]['privkey'] = wallet.get_wif_path(av['path']) used_commitments, external_commitments = podle.get_podle_commitments() for u, ec in iteritems(external_commitments): + success, us = utxo_to_utxostr(u) + assert success tries = podle.get_podle_tries(utxo=u, max_tries=max_tries, external=True) tries_remaining = max(0, max_tries - tries) - unsp[u] = {'tries': tries, 'tries_remaining': tries_remaining, + unsp[us] = {'tries': tries, 'tries_remaining': tries_remaining, 'external': True} return json.dumps(unsp, indent=4) @@ -431,7 +436,7 @@ def wallet_display(wallet_service, showprivkey, displayall=False, acctlist = [] # TODO - either optionally not show disabled utxos, or # mark them differently in display (labels; colors) - utxos = wallet_service.get_utxos_by_mixdepth(include_disabled=True, hexfmt=False) + utxos = wallet_service.get_utxos_by_mixdepth(include_disabled=True) for m in range(wallet_service.mixdepth + 1): branchlist = [] for address_type in [0, 1]: diff --git a/jmclient/jmclient/yieldgenerator.py b/jmclient/jmclient/yieldgenerator.py index b18eb90..f89af36 100644 --- a/jmclient/jmclient/yieldgenerator.py +++ b/jmclient/jmclient/yieldgenerator.py @@ -132,7 +132,8 @@ class YieldGeneratorBasic(YieldGenerator): change_addr = self.wallet_service.get_internal_addr(mixdepth) - utxos = self.wallet_service.select_utxos(mixdepth, total_amount, minconfs=1) + utxos = self.wallet_service.select_utxos(mixdepth, total_amount, + minconfs=1, includeaddr=True) my_total_in = sum([va['value'] for va in utxos.values()]) real_cjfee = calc_cj_fee(offer["ordertype"], offer["cjfee"], amount) change_value = my_total_in - amount - offer["txfee"] + real_cjfee @@ -141,7 +142,8 @@ class YieldGeneratorBasic(YieldGenerator): 'finding new utxos').format(change_value)) try: utxos = self.wallet_service.select_utxos(mixdepth, - total_amount + jm_single().DUST_THRESHOLD, minconfs=1) + total_amount + jm_single().DUST_THRESHOLD, + minconfs=1, includeaddr=True) except Exception: jlog.info('dont have the required UTXOs to make a ' 'output above the dust threshold, quitting') diff --git a/jmclient/test/commontest.py b/jmclient/test/commontest.py index 01ce37a..36ed7c4 100644 --- a/jmclient/test/commontest.py +++ b/jmclient/test/commontest.py @@ -6,11 +6,13 @@ import binascii import random from decimal import Decimal -from jmbase import get_log +from jmbase import (get_log, hextobin, utxostr_to_utxo, + utxo_to_utxostr, listchanger, dictchanger) + from jmclient import ( jm_single, open_test_wallet_maybe, estimate_tx_fee, BlockchainInterface, get_p2sh_vbyte, BIP32Wallet, - SegwitLegacyWallet, WalletService) + SegwitLegacyWallet, WalletService, BTC_P2SH_P2WPKH) from jmbase.support import chunks import jmbitcoin as btc @@ -45,9 +47,11 @@ class DummyBlockchainInterface(BlockchainInterface): pass def is_address_imported(self, addr): pass - + + def get_current_block_height(self): + return 10**6 + def pushtx(self, txhex): - print("pushing: " + str(txhex)) return True def insert_fake_query_results(self, fqr): @@ -56,7 +60,7 @@ class DummyBlockchainInterface(BlockchainInterface): def setQUSFail(self, state): self.qusfail = state - def query_utxo_set(self, txouts,includeconf=False): + def query_utxo_set(self, txouts, includeconf=False): if self.qusfail: #simulate failure to find the utxo return [None] @@ -72,12 +76,14 @@ class DummyBlockchainInterface(BlockchainInterface): known_outs = {"03243f4a659e278a1333f8308f6aaf32db4692ee7df0340202750fd6c09150f6:1": "03a2d1cbe977b1feaf8d0d5cc28c686859563d1520b28018be0c2661cf1ebe4857", "498faa8b22534f3b443c6b0ce202f31e12f21668b4f0c7a005146808f250d4c3:0": "02b4b749d54e96b04066b0803e372a43d6ffa16e75a001ae0ed4b235674ab286be", "3f3ea820d706e08ad8dc1d2c392c98facb1b067ae4c671043ae9461057bd2a3c:1": "023bcbafb4f68455e0d1d117c178b0e82a84e66414f0987453d78da034b299c3a9"} + known_outs = dictchanger(known_outs) #our wallet utxos, faked, for podle tests: utxos are doctored (leading 'f'), #and the lists are (amt, age) wallet_outs = {'f34b635ed8891f16c4ec5b8236ae86164783903e8e8bb47fa9ef2ca31f3c2d7a:0': [10000000, 2], 'f780d6e5e381bff01a3519997bb4fcba002493103a198fde334fd264f9835d75:1': [20000000, 6], 'fe574db96a4d43a99786b3ea653cda9e4388f377848f489332577e018380cff1:0': [50000000, 3], 'fd9711a2ef340750db21efb761f5f7d665d94b312332dc354e252c77e9c48349:0': [50000000, 6]} + wallet_outs = dictchanger(wallet_outs) if includeconf and set(txouts).issubset(set(wallet_outs)): #includeconf used as a trigger for a podle check; @@ -88,16 +94,16 @@ class DummyBlockchainInterface(BlockchainInterface): 'confirms': wallet_outs[to][1]}) return results if txouts[0] in known_outs: - addr = btc.pubkey_to_p2sh_p2wpkh_address( - known_outs[txouts[0]], get_p2sh_vbyte()) + scr = BTC_P2SH_P2WPKH.pubkey_to_script(known_outs[txouts[0]]) + addr = btc.CCoinAddress.from_scriptPubKey(scr) return [{'value': 200000000, 'address': addr, - 'script': btc.address_to_script(addr), + 'script': scr, 'confirms': 20}] for t in txouts: result_dict = {'value': 10000000000, 'address': "mrcNu71ztWjAQA6ww9kHiW3zBWSQidHXTQ", - 'script': '76a91479b000887626b294a914501a4cd226b58b23598388ac'} + 'script': hextobin('76a91479b000887626b294a914501a4cd226b58b23598388ac')} if includeconf: result_dict['confirms'] = 20 result.append(result_dict) @@ -110,18 +116,10 @@ class DummyBlockchainInterface(BlockchainInterface): def create_wallet_for_sync(wallet_structure, a, **kwargs): #We need a distinct seed for each run so as not to step over each other; #make it through a deterministic hash - seedh = btc.sha256("".join([str(x) for x in a]))[:32] + seedh = btc.b2x(btc.Hash("".join([str(x) for x in a]).encode("utf-8")))[:32] return make_wallets( 1, [wallet_structure], fixed_seeds=[seedh], **kwargs)[0]['wallet'] - -def binarize_tx(tx): - for o in tx['outs']: - o['script'] = binascii.unhexlify(o['script']) - for i in tx['ins']: - i['outpoint']['hash'] = binascii.unhexlify(i['outpoint']['hash']) - - def make_sign_and_push(ins_full, wallet_service, amount, @@ -130,7 +128,12 @@ def make_sign_and_push(ins_full, hashcode=btc.SIGHASH_ALL, estimate_fee = False): """Utility function for easily building transactions - from wallets + from wallets. + `ins_full` should be a list of dicts in format returned + by wallet.select_utxos: + {(txid, index): {"script":..,"value":..,"path":..}} + ... although the path is not used. + The "script" and "value" data is used to allow signing. """ assert isinstance(wallet_service, WalletService) total = sum(x['value'] for x in ins_full.values()) @@ -143,22 +146,21 @@ def make_sign_and_push(ins_full, 'address': output_addr}, {'value': total - amount - fee_est, 'address': change_addr}] - de_tx = btc.deserialize(btc.mktx(ins, outs)) + tx = btc.mktx(ins, outs) scripts = {} - for index, ins in enumerate(de_tx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - script = wallet_service.addr_to_script(ins_full[utxo]['address']) - scripts[index] = (script, ins_full[utxo]['value']) - binarize_tx(de_tx) - de_tx = wallet_service.sign_tx(de_tx, scripts, hashcode=hashcode) + for i, j in enumerate(ins): + scripts[i] = (ins_full[j]["script"], ins_full[j]["value"]) + + success, msg = wallet_service.sign_tx(tx, scripts, hashcode=hashcode) + if not success: + return False #pushtx returns False on any error - push_succeed = jm_single().bc_interface.pushtx(btc.serialize(de_tx)) + push_succeed = jm_single().bc_interface.pushtx(tx.serialize()) if push_succeed: - txid = btc.txhash(btc.serialize(de_tx)) # in normal operation this happens automatically # but in some tests there is no monitoring loop: - wallet_service.process_new_tx(de_tx, txid) - return txid + wallet_service.process_new_tx(tx) + return tx.GetTxid()[::-1] else: return False diff --git a/jmclient/test/test_client_protocol.py b/jmclient/test/test_client_protocol.py index 0ced712..7583a55 100644 --- a/jmclient/test/test_client_protocol.py +++ b/jmclient/test/test_client_protocol.py @@ -1,7 +1,8 @@ #! /usr/bin/env python '''test client-protocol interfacae.''' -from jmbase import get_log +from jmbase import get_log, bintohex, hextobin +from jmbase.commands import * from jmclient import load_test_config, Taker,\ JMClientProtocolFactory, jm_single, Maker, WalletService from jmclient.client_protocol import JMTakerClientProtocol @@ -14,12 +15,10 @@ from twisted.protocols.amp import UnknownRemoteError from twisted.protocols import amp from twisted.trial import unittest from twisted.test import proto_helpers -from jmbase.commands import * from taker_test_data import t_raw_signed_tx from commontest import default_max_cj_fee import json import jmbitcoin as bitcoin - import twisted twisted.internet.base.DelayedCall.debug = True @@ -97,7 +96,7 @@ class DummyMaker(Maker): def on_auth_received(self, nick, offer, commitment, cr, amount, kphex): # success, utxos, auth_pub, cj_addr, change_addr, btc_sig - return True, [], '', '', '', '' + return True, [], b"", '', '', '' def on_tx_received(self, nick, txhex, offerinfo): # success, sigs @@ -227,8 +226,8 @@ class JMTestServerProtocol(JMBaseProtocol): self.defaultCallbacks(d2) #To test, this must include a valid ecdsa sig fullmsg = "fullmsgforverify" - priv = "aa"*32 + "01" - pub = bitcoin.privkey_to_pubkey(priv) + priv = b"\xaa"*32 + b"\x01" + pub = bintohex(bitcoin.privkey_to_pubkey(priv)) sig = bitcoin.ecdsa_sign(fullmsg, priv) d3 = self.callRemote(JMRequestMsgSigVerify, msg="msgforverify", @@ -262,7 +261,7 @@ class JMTestServerProtocolFactory(protocol.ServerFactory): class DummyClientProtocolFactory(JMClientProtocolFactory): def buildProtocol(self, addr): - return JMTakerClientProtocol(self, self.client, nick_priv="aa"*32) + return JMTakerClientProtocol(self, self.client, nick_priv=b"\xaa"*32 + b"\x01") class TrialTestJMClientProto(unittest.TestCase): @@ -364,8 +363,8 @@ class TestMakerClientProtocol(unittest.TestCase): @inlineCallbacks def test_JMRequestMsgSigVerify(self): fullmsg = 'fullmsgforverify' - priv = 'aa'*32 + '01' - pub = bitcoin.privkey_to_pubkey(priv) + priv = b"\xaa"*32 + b"\x01" + pub = bintohex(bitcoin.privkey_to_pubkey(priv)) sig = bitcoin.ecdsa_sign(fullmsg, priv) yield self.init_client() yield self.callClient( diff --git a/jmclient/test/test_coinjoin.py b/jmclient/test/test_coinjoin.py index 6428a2b..810bf71 100644 --- a/jmclient/test/test_coinjoin.py +++ b/jmclient/test/test_coinjoin.py @@ -6,14 +6,15 @@ Test doing full coinjoins, bypassing IRC import os import sys import pytest +import copy from twisted.internet import reactor -from jmbase import get_log +from jmbase import get_log, hextobin, bintohex from jmclient import load_test_config, jm_single,\ YieldGeneratorBasic, Taker, LegacyWallet, SegwitLegacyWallet,\ NO_ROUNDING from jmclient.podle import set_commitment_file -from commontest import make_wallets, binarize_tx, default_max_cj_fee +from commontest import make_wallets, default_max_cj_fee from test_taker import dummy_filter_orderbook import jmbitcoin as btc @@ -85,14 +86,14 @@ def init_coinjoin(taker, makers, orderbook, cj_amount): ioauth_data = list(response[1:]) ioauth_data[0] = list(ioauth_data[0].keys()) # maker_pk which is set up by jmdaemon - ioauth_data.append(None) + ioauth_data.append("00") maker_data[mid] = ioauth_data # this is handled by jmdaemon active_orders[mid]['utxos'] = response[1] active_orders[mid]['cjaddr'] = ioauth_data[2] active_orders[mid]['changeaddr'] = ioauth_data[3] - active_orders[mid]['offer'] = m.offerlist[0] + active_orders[mid]['offer'] = copy.deepcopy(m.offerlist[0]) active_orders[mid]['amount'] = cj_amount return active_orders, maker_data @@ -194,13 +195,12 @@ def test_coinjoin_mixdepth_wrap_taker(monkeypatch, tmpdir, setup_cj): taker_final_result = do_tx_signing(taker, makers, active_orders, txdata) assert taker_final_result is not False - tx = btc.deserialize(txdata[2]) - binarize_tx(tx) + tx = btc.CMutableTransaction.deserialize(hextobin(txdata[2])) wallet_service = wallet_services[-1] # TODO change for new tx monitoring: - wallet_service.remove_old_utxos_(tx) - wallet_service.add_new_utxos_(tx, b'\x00' * 32) # fake txid + wallet_service.remove_old_utxos(tx) + wallet_service.add_new_utxos(tx) balances = wallet_service.get_balance_by_mixdepth() assert balances[0] == cj_amount @@ -250,14 +250,13 @@ def test_coinjoin_mixdepth_wrap_maker(monkeypatch, tmpdir, setup_cj): taker_final_result = do_tx_signing(taker, makers, active_orders, txdata) assert taker_final_result is not False - tx = btc.deserialize(txdata[2]) - binarize_tx(tx) + tx = btc.CMutableTransaction.deserialize(hextobin(txdata[2])) for i in range(MAKER_NUM): wallet_service = wallet_services[i] # TODO as above re: monitoring - wallet_service.remove_old_utxos_(tx) - wallet_service.add_new_utxos_(tx, b'\x00' * 32) # fake txid + wallet_service.remove_old_utxos(tx) + wallet_service.add_new_utxos(tx) balances = wallet_service.get_balance_by_mixdepth() assert balances[0] == cj_amount diff --git a/jmclient/test/test_commitment_utils.py b/jmclient/test/test_commitment_utils.py index 31000ab..070e7e8 100644 --- a/jmclient/test/test_commitment_utils.py +++ b/jmclient/test/test_commitment_utils.py @@ -4,11 +4,14 @@ import pytest from jmclient import (load_test_config, jm_single) from jmclient.commitment_utils import get_utxo_info, validate_utxo_data - +from jmbitcoin import select_chain_params def test_get_utxo_info(): load_test_config() + # this test tests mainnet keys, so temporarily switch network + select_chain_params("bitcoin") jm_single().config.set("BLOCKCHAIN", "network", "mainnet") + dbci = DummyBlockchainInterface() privkey = "L1RrrnXkcKut5DEMwtDthjwRcTTwED36thyL1DebVrKuwvohjMNi" #to verify use from_wif_privkey and privkey_to_address @@ -28,7 +31,7 @@ def test_get_utxo_info(): with pytest.raises(Exception) as e_info: u, priv = get_utxo_info(fakeutxo + privkey) #invalid index - fu2 = "ab"*32 + ":00004" + fu2 = "ab"*32 + ":-1" with pytest.raises(Exception) as e_info: u, priv = get_utxo_info(fu2 + "," + privkey) #invalid privkey @@ -53,3 +56,5 @@ def test_get_utxo_info(): retval = validate_utxo_data(utxodatas, False) assert not retval dbci.setQUSFail(False) + select_chain_params("bitcoin/regtest") + jm_single().config.set("BLOCKCHAIN", "network", "mainnet") diff --git a/jmclient/test/test_maker.py b/jmclient/test/test_maker.py index 1bc53e9..a7046f2 100644 --- a/jmclient/test/test_maker.py +++ b/jmclient/test/test_maker.py @@ -29,8 +29,7 @@ def construct_tx_offerlist(cjaddr, changeaddr, maker_utxos, maker_utxos_value, 'txfee': 0 } - utxos = { utxo['outpoint']['hash'] + ':' + str(utxo['outpoint']['index']): - {'utxo': utxo, 'value': maker_utxos_value} for utxo in maker_utxos } + utxos = { utxo: {'utxo': utxo, 'value': maker_utxos_value} for utxo in maker_utxos } offerlist = { 'utxos': utxos, @@ -46,59 +45,56 @@ def construct_tx_offerlist(cjaddr, changeaddr, maker_utxos, maker_utxos_value, def create_tx_inputs(count=1): inp = [] for i in range(count): - inp.append({'outpoint': {'hash': '0'*64, 'index': i}, - 'script': '', - 'sequence': 4294967295}) + inp.append((b"\x00"*32, i)) return inp -def create_tx_outputs(*scripts_amount): +def create_tx_outputs(*addrs_amount): outp = [] - for script, amount in scripts_amount: - outp.append({'script': script, 'value': amount}) + for addr, amount in addrs_amount: + outp.append({'address': addr, 'value': amount}) return outp def address_p2pkh_generator(): - return get_address_generator(b'\x76\xa9\x14', b'\x88\xac', get_p2pk_vbyte()) + return get_address_generator(b'\x76\xa9\x14', b'\x88\xac') def address_p2sh_generator(): - return get_address_generator(b'\xa9\x14', b'\x87', get_p2sh_vbyte()) + return get_address_generator(b'\xa9\x14', b'\x87', p2sh=True) -def get_address_generator(script_pre, script_post, vbyte): +def get_address_generator(script_pre, script_post, p2sh=False): counter = 0 while True: script = script_pre + struct.pack(b'=LQQ', 0, 0, counter) + script_post - addr = btc.script_to_address(script, vbyte) - yield addr, binascii.hexlify(script).decode('ascii') + if p2sh: + addr = btc.CCoinAddress.from_scriptPubKey( + btc.CScript(script).to_p2sh_scriptPubKey()) + else: + addr = btc.CCoinAddress.from_scriptPubKey(btc.CScript(script)) + yield str(addr), binascii.hexlify(script).decode('ascii') counter += 1 -def create_tx_and_offerlist(cj_addr, cj_change_addr, other_output_scripts, - cj_script=None, cj_change_script=None, offertype='swreloffer'): - assert len(other_output_scripts) % 2 == 0, "bug in test" +def create_tx_and_offerlist(cj_addr, cj_change_addr, other_output_addrs, + offertype='swreloffer'): + assert len(other_output_addrs) % 2 == 0, "bug in test" cj_value = 100000000 maker_total_value = cj_value*3 - if cj_script is None: - cj_script = btc.address_to_script(cj_addr) - if cj_change_script is None: - cj_change_script = btc.address_to_script(cj_change_addr) - inputs = create_tx_inputs(3) outputs = create_tx_outputs( - (cj_script, cj_value), - (cj_change_script, maker_total_value - cj_value), # cjfee=0, txfee=0 - *((script, cj_value + (i%2)*(50000000+i)) \ - for i, script in enumerate(other_output_scripts)) + (cj_addr, cj_value), + (cj_change_addr, maker_total_value - cj_value), # cjfee=0, txfee=0 + *((addr, cj_value + (i%2)*(50000000+i)) \ + for i, addr in enumerate(other_output_addrs)) ) maker_utxos = [inputs[0]] - tx = btc.deserialize(btc.mktx(inputs, outputs)) + tx = btc.mktx(inputs, outputs) offerlist = construct_tx_offerlist(cj_addr, cj_change_addr, maker_utxos, maker_total_value, cj_value, offertype) @@ -119,21 +115,20 @@ def test_verify_unsigned_tx_sw_valid(setup_env_nodeps): # test standard cj tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr, - [next(p2sh_gen)[1] for s in range(4)], cj_script, cj_change_script) + [next(p2sh_gen)[0] for s in range(4)]) assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "standard sw cj" # test cj with mixed outputs tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr, - list(chain((next(p2sh_gen)[1] for s in range(3)), - (next(p2pkh_gen)[1] for s in range(1)))), - cj_script, cj_change_script) + list(chain((next(p2sh_gen)[0] for s in range(3)), + (next(p2pkh_gen)[0] for s in range(1))))) assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "sw cj with p2pkh output" # test cj with only p2pkh outputs tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr, - [next(p2pkh_gen)[1] for s in range(4)], cj_script, cj_change_script) + [next(p2pkh_gen)[0] for s in range(4)]) assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "sw cj with only p2pkh outputs" @@ -152,21 +147,20 @@ def test_verify_unsigned_tx_nonsw_valid(setup_env_nodeps): # test standard cj tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr, - [next(p2pkh_gen)[1] for s in range(4)], cj_script, cj_change_script, 'reloffer') + [next(p2pkh_gen)[0] for s in range(4)], offertype='reloffer') assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "standard nonsw cj" # test cj with mixed outputs tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr, - list(chain((next(p2sh_gen)[1] for s in range(1)), - (next(p2pkh_gen)[1] for s in range(3)))), - cj_script, cj_change_script, 'reloffer') + list(chain((next(p2sh_gen)[0] for s in range(1)), + (next(p2pkh_gen)[0] for s in range(3)))), offertype='reloffer') assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "nonsw cj with p2sh output" # test cj with only p2sh outputs tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr, - [next(p2sh_gen)[1] for s in range(4)], cj_script, cj_change_script, 'reloffer') + [next(p2sh_gen)[0] for s in range(4)], offertype='reloffer') assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "nonsw cj with only p2sh outputs" @@ -175,4 +169,5 @@ def test_verify_unsigned_tx_nonsw_valid(setup_env_nodeps): def setup_env_nodeps(monkeypatch): monkeypatch.setattr(jmclient.configure, 'get_blockchain_interface_instance', lambda x: DummyBlockchainInterface()) + btc.select_chain_params("bitcoin/regtest") load_test_config() diff --git a/jmclient/test/test_payjoin.py b/jmclient/test/test_payjoin.py index 3670100..6d0ba14 100644 --- a/jmclient/test/test_payjoin.py +++ b/jmclient/test/test_payjoin.py @@ -75,14 +75,11 @@ def final_checks(wallet_services, amount, txfee, tsb, msb, source_mixdepth=0): return True @pytest.mark.parametrize('wallet_cls, wallet_structures, mean_amt', - [([LegacyWallet, LegacyWallet], - [[4, 0, 0, 0, 0]] * 2, 1.0), + [ # note we have removed LegacyWallet test cases. ([SegwitLegacyWallet, SegwitLegacyWallet], [[1, 3, 0, 0, 0]] * 2, 2.0), ([SegwitWallet, SegwitWallet], [[1, 0, 0, 0, 0]] * 2, 4.0), - ([LegacyWallet, SegwitWallet], - [[4, 0, 0, 0, 0]] * 2, 1.0), ([SegwitLegacyWallet, SegwitWallet], [[1, 3, 0, 0, 0]] * 2, 2.0), ([SegwitWallet, SegwitLegacyWallet], diff --git a/jmclient/test/test_podle.py b/jmclient/test/test_podle.py index d49fe80..a643b7e 100644 --- a/jmclient/test/test_podle.py +++ b/jmclient/test/test_podle.py @@ -28,8 +28,8 @@ def test_commitment_retries(setup_podle): """ allowed = jm_single().config.getint("POLICY", "taker_utxo_retries") #make some pretend commitments - dummy_priv_utxo_pairs = [(bitcoin.sha256(os.urandom(10)), - bitcoin.sha256(os.urandom(10))+":0") for _ in range(10)] + dummy_priv_utxo_pairs = [(bitcoin.Hash(os.urandom(10)), + bitcoin.b2x(bitcoin.Hash(os.urandom(10)))+":0") for _ in range(10)] #test a single commitment request of all 10 for x in dummy_priv_utxo_pairs: p = generate_podle([x], allowed) @@ -46,15 +46,15 @@ def generate_single_podle_sig(priv, i): This calls the underlying 'raw' code based on the class PoDLE, not the library 'generate_podle' which intelligently searches and updates commitments. """ - dummy_utxo = bitcoin.sha256(priv) + ":3" - podle = PoDLE(dummy_utxo, binascii.hexlify(priv).decode('ascii')) + dummy_utxo = bitcoin.b2x(bitcoin.Hash(priv)) + ":3" + podle = PoDLE(dummy_utxo, priv) r = podle.generate_podle(i) return (r['P'], r['P2'], r['sig'], r['e'], r['commit']) def test_rand_commitments(setup_podle): for i in range(20): - priv = os.urandom(32) + priv = os.urandom(32)+b"\x01" Pser, P2ser, s, e, commitment = generate_single_podle_sig(priv, 1 + i%5) assert verify_podle(Pser, P2ser, s, e, commitment) #tweak commitments to verify failure @@ -90,7 +90,7 @@ def test_external_commitments(setup_podle): tries = jm_single().config.getint("POLICY","taker_utxo_retries") for i in range(10): priv = os.urandom(32) - dummy_utxo = bitcoin.sha256(priv)+":2" + dummy_utxo = (bitcoin.Hash(priv), 2) ecs[dummy_utxo] = {} ecs[dummy_utxo]['reveal']={} for j in range(tries): @@ -104,16 +104,16 @@ def test_external_commitments(setup_podle): assert external[u]['P'] == ecs[u]['P'] for i in range(tries): for x in ['P2', 's', 'e']: - assert external[u]['reveal'][str(i)][x] == ecs[u]['reveal'][i][x] + assert external[u]['reveal'][i][x] == ecs[u]['reveal'][i][x] #add a dummy used commitment, then try again - update_commitments(commitment="ab"*32) + update_commitments(commitment=b"\xab"*32) ecs = {} known_commits = [] known_utxos = [] tries = 3 for i in range(1, 6): - u = binascii.hexlify(struct.pack(b'B', i)*32).decode('ascii') + u = (struct.pack(b'B', i)*32, i+3) known_utxos.append(u) priv = struct.pack(b'B', i)*32+b"\x01" ecs[u] = {} @@ -131,8 +131,9 @@ def test_external_commitments(setup_podle): #this should find the remaining one utxo and return from it assert generate_podle([], max_tries=tries, allow_external=known_utxos) #test commitment removal - to_remove = ecs[binascii.hexlify(struct.pack(b'B', 3)*32).decode('ascii')] - update_commitments(external_to_remove={binascii.hexlify(struct.pack(b'B', 3)*32).decode('ascii'):to_remove}) + tru = (struct.pack(b"B", 3)*32, 3+3) + to_remove = {tru: ecs[tru]} + update_commitments(external_to_remove=to_remove) #test that an incorrectly formatted file raises with open(get_commitment_file(), "rb") as f: validjson = json.loads(f.read().decode('utf-8')) @@ -152,14 +153,14 @@ def test_podle_constructor(setup_podle): """Tests rules about construction of PoDLE object are conformed to. """ - priv = "aa"*32 + priv = b"\xaa"*32 #pub and priv together not allowed with pytest.raises(PoDLEError) as e_info: p = PoDLE(priv=priv, P="dummypub") #no pub or priv is allowed, i forget if this is useful for something p = PoDLE() #create from priv - p = PoDLE(priv=priv+"01", u="dummyutxo") + p = PoDLE(priv=priv+b"\x01", u=(struct.pack(b"B", 7)*32, 4)) pdict = p.generate_podle(2) assert all([k in pdict for k in ['used', 'utxo', 'P', 'P2', 'commit', 'sig', 'e']]) #using the valid data, serialize/deserialize test @@ -181,7 +182,7 @@ def test_podle_constructor(setup_podle): with pytest.raises(PoDLEError) as e_info: p.generate_podle(0) #Test construction from pubkey - pub = bitcoin.privkey_to_pubkey(priv+"01") + pub = bitcoin.privkey_to_pubkey(priv+b"\x01") p = PoDLE(P=pub) with pytest.raises(PoDLEError) as e_info: p.get_commitment() diff --git a/jmbitcoin/test/test_keys.py b/jmclient/test/test_privkeys.py similarity index 75% rename from jmbitcoin/test/test_keys.py rename to jmclient/test/test_privkeys.py index 06c48fd..2fb6884 100644 --- a/jmbitcoin/test/test_keys.py +++ b/jmclient/test/test_privkeys.py @@ -2,13 +2,16 @@ '''Public and private key validity and formatting tests.''' import jmbitcoin as btc +from jmclient import (BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, + jm_single, load_test_config) import binascii +import struct import json import pytest import os testdir = os.path.dirname(os.path.realpath(__file__)) -def test_read_raw_privkeys(): +def test_read_raw_privkeys(setup_keys): badkeys = [b'', b'\x07'*31,b'\x07'*34, b'\x07'*33] for b in badkeys: with pytest.raises(Exception) as e_info: @@ -18,7 +21,7 @@ def test_read_raw_privkeys(): c, k = btc.read_privkey(g[0]) assert c == g[1] -def test_wif_privkeys_invalid(): +def test_wif_privkeys_invalid(setup_keys): #first try to create wif privkey from key of wrong length bad_privs = [b'\x01\x02'*17] #some silly private key but > 33 bytes @@ -27,7 +30,7 @@ def test_wif_privkeys_invalid(): for priv in bad_privs: with pytest.raises(Exception) as e_info: - fake_wif = btc.wif_compressed_privkey(binascii.hexlify(priv).decode('ascii')) + fake_wif = BTCEngine.privkey_to_wif(priv) #Create a wif with wrong length bad_wif1 = btc.bin_to_b58check(b'\x01\x02'*34, b'\x80') @@ -35,7 +38,7 @@ def test_wif_privkeys_invalid(): bad_wif2 = btc.bin_to_b58check(b'\x07'*33, b'\x80') for bw in [bad_wif1, bad_wif2]: with pytest.raises(Exception) as e_info: - fake_priv = btc.from_wif_privkey(bw) + fake_priv, keytype = BTCEngine.wif_to_privkey(bw) #Some invalid b58 from bitcoin repo; #none of these are valid as any kind of key or address @@ -49,15 +52,14 @@ def test_wif_privkeys_invalid(): print('testing this key: ' + bad_key) #should throw exception with pytest.raises(Exception) as e_info: - from_wif_key = btc.from_wif_privkey(bad_key, - btc.get_version_byte(bad_key)) + from_wif_key, keytype = BTCEngine.wif_to_privkey(bad_key) #in case the b58 check encoding is valid, we should #also check if the leading version byte is in the #expected set, and throw an error if not. if chr(btc.get_version_byte(bad_key)) not in b'\x80\xef': raise Exception("Invalid version byte") -def test_wif_privkeys_valid(): +def test_wif_privkeys_valid(setup_keys): with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f: json_data = f.read() valid_keys_list = json.loads(json_data) @@ -65,16 +67,20 @@ def test_wif_privkeys_valid(): key, hex_key, prop_dict = a if prop_dict["isPrivkey"]: netval = "testnet" if prop_dict["isTestnet"] else "mainnet" + jm_single().config.set("BLOCKCHAIN", "network", netval) print('testing this key: ' + key) assert btc.get_version_byte( key) in b'\x80\xef', "not valid network byte" comp = prop_dict["isCompressed"] - from_wif_key = btc.from_wif_privkey( - key, - compressed=comp, - vbyte=btc.from_int_to_byte(btc.from_byte_to_int(btc.get_version_byte(key))-128)) - expected_key = hex_key - if comp: expected_key += '01' + if not comp: + # we only handle compressed keys + continue + from_wif_key, keytype = BTCEngine.wif_to_privkey(key) + expected_key = binascii.unhexlify(hex_key) + b"\x01" assert from_wif_key == expected_key, "Incorrect key decoding: " + \ str(from_wif_key) + ", should be: " + str(expected_key) + jm_single().config.set("BLOCKCHAIN", "network", "testnet") +@pytest.fixture(scope='module') +def setup_keys(): + load_test_config() \ No newline at end of file diff --git a/jmclient/test/test_taker.py b/jmclient/test/test_taker.py index bc00d78..3cb0bd3 100644 --- a/jmclient/test/test_taker.py +++ b/jmclient/test/test_taker.py @@ -10,13 +10,21 @@ import pytest import json import struct from base64 import b64encode +from jmbase import (utxostr_to_utxo, utxo_to_utxostr, hextobin, + dictchanger, listchanger) from jmclient import load_test_config, jm_single, set_commitment_file,\ get_commitment_file, SegwitLegacyWallet, Taker, VolatileStorage,\ - get_network, WalletService, NO_ROUNDING + get_network, WalletService, NO_ROUNDING, BTC_P2PKH from taker_test_data import t_utxos_by_mixdepth, t_orderbook,\ t_maker_response, t_chosen_orders, t_dummy_ext from commontest import default_max_cj_fee +def convert_utxos(utxodict): + return_dict = {} + for uk, val in utxodict.items(): + return_dict[utxostr_to_utxo(uk)[1]] = val + return return_dict + class DummyWallet(SegwitLegacyWallet): def __init__(self): storage = VolatileStorage() @@ -36,32 +44,30 @@ class DummyWallet(SegwitLegacyWallet): script = self._ENGINE.address_to_script(data['address']) self._script_map[script] = path - def get_utxos_by_mixdepth(self, verbose=True, includeheight=False): - return t_utxos_by_mixdepth - - def get_utxos_by_mixdepth_(self, verbose=True, include_disabled=False, - includeheight=False): - utxos = self.get_utxos_by_mixdepth(verbose) - - utxos_conv = {} - for md, utxo_data in utxos.items(): - md_utxo = utxos_conv.setdefault(md, {}) - for i, (utxo_hex, data) in enumerate(utxo_data.items()): - utxo, index = utxo_hex.split(':') - data_conv = { - 'script': self._ENGINE.address_to_script(data['address']), - 'path': (b'dummy', md, i), - 'value': data['value'] - } - md_utxo[(binascii.unhexlify(utxo), int(index))] = data_conv - - return utxos_conv + def get_utxos_by_mixdepth(self, include_disabled=False, verbose=True, + includeheight=False): + # utxostr conversion routines because taker_test_data uses hex: + retval = {} + for mixdepth, v in t_utxos_by_mixdepth.items(): + retval[mixdepth] = {} + for i, (utxo, val) in enumerate(v.items()): + retval[mixdepth][utxostr_to_utxo(utxo)[1]] = val + val["script"] = self._ENGINE.address_to_script(val['address']) + val["path"] = (b'dummy', mixdepth, i) + return retval def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None, - maxheight=None): + maxheight=None, includeaddr=False): if amount > self.get_balance_by_mixdepth()[mixdepth]: raise Exception("Not enough funds") - return t_utxos_by_mixdepth[mixdepth] + # comment as for get_utxos_by_mixdepth: + retval = {} + for k, v in t_utxos_by_mixdepth[mixdepth].items(): + success, u = utxostr_to_utxo(k) + assert success + retval[u] = v + retval[u]["script"] = self.addr_to_script(retval[u]["address"]) + return retval def get_internal_addr(self, mixing_depth, bci=None): if self.inject_addr_get_failure: @@ -70,7 +76,7 @@ class DummyWallet(SegwitLegacyWallet): def sign_tx(self, tx, addrs): print("Pretending to sign on addresses: " + str(addrs)) - return tx + return True, None def sign(self, tx, i, priv, amount): """Sign a transaction; the amount field @@ -97,10 +103,10 @@ class DummyWallet(SegwitLegacyWallet): musGZczug3BAbqobmYherywCwL9REgNaNm """ for p in privs: - addrs[p] = bitcoin.privkey_to_address(p, False, magicbyte=0x6f) + addrs[p] = BTC_P2PKH.privkey_to_address(p) for p, a in iteritems(addrs): if a == addr: - return binascii.hexlify(p).decode('ascii') + return p raise ValueError("No such keypair") def _is_my_bip32_path(self, path): @@ -170,7 +176,7 @@ def test_make_commitment(setup_taker, failquery, external): amount = 110000000 taker = get_taker([(mixdepth, amount, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", NO_ROUNDING)]) taker.cjamount = amount - taker.input_utxos = t_utxos_by_mixdepth[0] + taker.input_utxos = convert_utxos(t_utxos_by_mixdepth[0]) if failquery: jm_single().bc_interface.setQUSFail(True) taker.make_commitment() @@ -194,12 +200,13 @@ def test_auth_pub_not_found(setup_taker): res = taker.initialize(orderbook) taker.orderbook = copy.deepcopy(t_chosen_orders) #total_cjfee unaffected, all same maker_response = copy.deepcopy(t_maker_response) - utxos = ["03243f4a659e278a1333f8308f6aaf32db4692ee7df0340202750fd6c09150f6:1", - "498faa8b22534f3b443c6b0ce202f31e12f21668b4f0c7a005146808f250d4c3:0", - "3f3ea820d706e08ad8dc1d2c392c98facb1b067ae4c671043ae9461057bd2a3c:1"] + utxos = [utxostr_to_utxo(x)[1] for x in [ + "03243f4a659e278a1333f8308f6aaf32db4692ee7df0340202750fd6c09150f6:1", + "498faa8b22534f3b443c6b0ce202f31e12f21668b4f0c7a005146808f250d4c3:0", + "3f3ea820d706e08ad8dc1d2c392c98facb1b067ae4c671043ae9461057bd2a3c:1"]] fake_query_results = [{'value': 200000000, 'address': "mrKTGvFfYUEqk52qPKUroumZJcpjHLQ6pn", - 'script': '76a914767c956efe6092a775fea39a06d1cac9aae956d788ac', + 'script': hextobin('76a914767c956efe6092a775fea39a06d1cac9aae956d788ac'), 'utxo': utxos[i], 'confirms': 20} for i in range(3)] jm_single().bc_interface.insert_fake_query_results(fake_query_results) @@ -223,8 +230,8 @@ def test_auth_pub_not_found(setup_taker): ([(0, 199850001, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False, 2, False, None, None), #trigger sub dust change for taker #edge case triggers that do fail - ([(0, 199850000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False, - 2, False, None, None), #trigger negative change + ([(0, 199851000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False, + 2, False, None, None), #trigger negative change ([(0, 199599800, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False, 2, False, None, None), #trigger sub dust change for maker ([(0, 20000000, 3, "INTERNAL", 0, NO_ROUNDING)], True, False, @@ -232,7 +239,7 @@ def test_auth_pub_not_found(setup_taker): ([(0, 20000000, 3, "INTERNAL", 0, NO_ROUNDING)], False, False, 7, False, None, None), #test not enough cp ([(0, 80000000, 3, "INTERNAL", 0, NO_ROUNDING)], False, False, - 2, False, None, "30000"), #test failed commit + 2, False, None, "30000"), #test failed commit ([(0, 20000000, 3, "INTERNAL", 0, NO_ROUNDING)], False, False, 2, True, None, None), #test unauthed response ([(0, 5000000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, True, @@ -282,14 +289,15 @@ def test_taker_init(setup_taker, schedule, highfee, toomuchcoins, minmakers, if notauthed: #Doctor one of the maker response data fields maker_response["J659UPUSLLjHJpaB"][1] = "xx" #the auth pub - if schedule[0][1] == 199850000: + if schedule[0][1] == 199851000: #triggers negative change - #makers offer 3000 txfee; we estimate ~ 147*10 + 2*34 + 10=1548 bytes - #times 30k = 46440, so we pay 43440, plus maker fees = 3*0.0002*200000000 - #roughly, gives required selected = amt + 163k, hence the above = - #2btc - 150k sats = 199850000 (tweaked because of aggressive coin selection) + #((109 + 4*64)*ins + 34 * outs + 8)/4. plug in 9 ins and 8 outs gives + #tx size estimate = 1101 bytes. Times 30 ~= 33030. + #makers offer 3000 txfee, so we pay 30030, plus maker fees = 3*0.0002*200000000 + #roughly, gives required selected = amt + 120k+30k, hence the above = + #2btc - 140k sats = 199851000 (tweaked because of aggressive coin selection) #simulate the effect of a maker giving us a lot more utxos - taker.utxos["dummy_for_negative_change"] = ["a", "b", "c", "d", "e"] + taker.utxos["dummy_for_negative_change"] = [(struct.pack(b"B", a) *32, a+1) for a in range(7,12)] with pytest.raises(ValueError) as e_info: res = taker.receive_utxos(maker_response) return clean_up() @@ -361,7 +369,7 @@ def test_taker_init(setup_taker, schedule, highfee, toomuchcoins, minmakers, [ (7), ]) -def test_unconfirm_confirm(schedule_len): +def test_unconfirm_confirm(setup_taker, schedule_len): """These functions are: do-nothing by default (unconfirm, for Taker), and merely update schedule index for confirm (useful for schedules/tumbles). This tests that the on_finished callback correctly reports the fromtx @@ -369,20 +377,26 @@ def test_unconfirm_confirm(schedule_len): The exception to the above is that the txd passed in must match self.latest_tx, so we use a dummy value here for that. """ + class DummyTx(object): + pass test_unconfirm_confirm.txflag = True def finished_for_confirms(res, fromtx=False, waittime=0, txdetails=None): assert res #confirmed should always send true test_unconfirm_confirm.txflag = fromtx taker = get_taker(schedule_len=schedule_len, on_finished=finished_for_confirms) - taker.latest_tx = {"outs": "blah"} - taker.unconfirm_callback({"ins": "foo", "outs": "blah"}, "b") + taker.latest_tx = DummyTx() + taker.latest_tx.vout = "blah" + fake_txd = DummyTx() + fake_txd.vin = "foo" + fake_txd.vout = "blah" + taker.unconfirm_callback(fake_txd, "b") for i in range(schedule_len-1): taker.schedule_index += 1 - fromtx = taker.confirm_callback({"ins": "foo", "outs": "blah"}, "b", 1) + fromtx = taker.confirm_callback(fake_txd, "b", 1) assert test_unconfirm_confirm.txflag taker.schedule_index += 1 - fromtx = taker.confirm_callback({"ins": "foo", "outs": "blah"}, "b", 1) + fromtx = taker.confirm_callback(fake_txd, "b", 1) assert not test_unconfirm_confirm.txflag @pytest.mark.parametrize( @@ -396,16 +410,13 @@ def test_on_sig(setup_taker, dummyaddr, schedule): #then, create a signature with various inputs, pass in in b64 to on_sig. #in order for it to verify, the DummyBlockchainInterface will have to #return the right values in query_utxo_set - + utxos = [(struct.pack(b"B", x) * 32, 1) for x in range(5)] #create 2 privkey + utxos that are to be ours privs = [x*32 + b"\x01" for x in [struct.pack(b'B', y) for y in range(1,6)]] - utxos = [str(x)*64+":1" for x in range(5)] - fake_query_results = [{'value': 200000000, - 'utxo': utxos[x], - 'address': bitcoin.privkey_to_address(privs[x], False, magicbyte=0x6f), - 'script': bitcoin.mk_pubkey_script( - bitcoin.privkey_to_address(privs[x], False, magicbyte=0x6f)), - 'confirms': 20} for x in range(5)] + scripts = [BTC_P2PKH.privkey_to_script(privs[x]) for x in range(5)] + addrs = [BTC_P2PKH.privkey_to_address(privs[x]) for x in range(5)] + fake_query_results = [{'value': 200000000, 'utxo': utxos[x], 'address': addrs[x], + 'script': scripts[x], 'confirms': 20} for x in range(5)] dbci = DummyBlockchainInterface() dbci.insert_fake_query_results(fake_query_results) @@ -414,47 +425,52 @@ def test_on_sig(setup_taker, dummyaddr, schedule): outs = [{'value': 100000000, 'address': dummyaddr}, {'value': 899990000, 'address': dummyaddr}] tx = bitcoin.mktx(utxos, outs) - - de_tx = bitcoin.deserialize(tx) + # since tx will be updated as it is signed, unlike in real life + # (where maker signing operation doesn't happen here), we'll create + # a second copy without the signatures: + tx2 = bitcoin.mktx(utxos, outs) + #prepare the Taker with the right intermediate data taker = get_taker(schedule=schedule) taker.nonrespondants=["cp1", "cp2", "cp3"] - taker.latest_tx = de_tx + taker.latest_tx = tx #my inputs are the first 2 utxos taker.input_utxos = {utxos[0]: - {'address': bitcoin.privkey_to_address(privs[0], False, magicbyte=0x6f), - 'script': bitcoin.mk_pubkey_script( - bitcoin.privkey_to_address(privs[0], False, magicbyte=0x6f)), + {'address': addrs[0], + 'script': scripts[0], 'value': 200000000}, utxos[1]: - {'address': bitcoin.privkey_to_address(privs[1], False, magicbyte=0x6f), - 'script': bitcoin.mk_pubkey_script( - bitcoin.privkey_to_address(privs[1], False, magicbyte=0x6f)), + {'address': addrs[1], + 'script': scripts[1], 'value': 200000000}} taker.utxos = {None: utxos[:2], "cp1": [utxos[2]], "cp2": [utxos[3]], "cp3":[utxos[4]]} for i in range(2): # placeholders required for my inputs - taker.latest_tx['ins'][i]['script'] = 'deadbeef' + taker.latest_tx.vin[i].scriptSig = bitcoin.CScript(hextobin('deadbeef')) + tx2.vin[i].scriptSig = bitcoin.CScript(hextobin('deadbeef')) #to prepare for my signing, need to mark cjaddr: taker.my_cj_addr = dummyaddr #make signatures for the last 3 fake utxos, considered as "not ours": - tx3 = bitcoin.sign(tx, 2, privs[2]) - sig3 = b64encode(binascii.unhexlify(bitcoin.deserialize(tx3)['ins'][2]['script'])) + sig, msg = bitcoin.sign(tx2, 2, privs[2]) + assert sig, "Failed to sign: " + msg + sig3 = b64encode(tx2.vin[2].scriptSig) taker.on_sig("cp1", sig3) #try sending the same sig again; should be ignored taker.on_sig("cp1", sig3) - tx4 = bitcoin.sign(tx, 3, privs[3]) - sig4 = b64encode(binascii.unhexlify(bitcoin.deserialize(tx4)['ins'][3]['script'])) + sig, msg = bitcoin.sign(tx2, 3, privs[3]) + assert sig, "Failed to sign: " + msg + sig4 = b64encode(tx2.vin[3].scriptSig) #try sending junk instead of cp2's correct sig - taker.on_sig("cp2", str("junk")) + assert not taker.on_sig("cp2", str("junk")), "incorrectly accepted junk signature" taker.on_sig("cp2", sig4) - tx5 = bitcoin.sign(tx, 4, privs[4]) + sig, msg = bitcoin.sign(tx2, 4, privs[4]) + assert sig, "Failed to sign: " + msg #Before completing with the final signature, which will trigger our own #signing, try with an injected failure of query utxo set, which should #prevent this signature being accepted. dbci.setQUSFail(True) - sig5 = b64encode(binascii.unhexlify(bitcoin.deserialize(tx5)['ins'][4]['script'])) - taker.on_sig("cp3", sig5) + sig5 = b64encode(tx2.vin[4].scriptSig) + assert not taker.on_sig("cp3", sig5), "incorrectly accepted sig5" #allow it to succeed, and try again dbci.setQUSFail(False) #this should succeed and trigger the we-sign code @@ -465,7 +481,7 @@ def test_on_sig(setup_taker, dummyaddr, schedule): [ ([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw")]), ]) -def test_auth_counterparty(schedule): +def test_auth_counterparty(setup_taker, schedule): taker = get_taker(schedule=schedule) first_maker_response = t_maker_response["J659UPUSLLjHJpaB"] utxo, auth_pub, cjaddr, changeaddr, sig, maker_pub = first_maker_response diff --git a/jmclient/test/test_tx_creation.py b/jmclient/test/test_tx_creation.py index d251b43..7c1e75c 100644 --- a/jmclient/test/test_tx_creation.py +++ b/jmclient/test/test_tx_creation.py @@ -1,6 +1,9 @@ #! /usr/bin/env python '''Test of unusual transaction types creation and push to -network to check validity.''' + network to check validity. + Note as of Feb 2020: earlier versions included multisig + p2(w)sh tests, these have been removed since Joinmarket + does not use this feature.''' import time import binascii @@ -24,62 +27,12 @@ vpubs = ["03e9a06e539d6bf5cf1ca5c41b59121fa3df07a338322405a312c67b6349a707e9", "028a2f126e3999ff66d01dcb101ab526d3aa1bf5cbdc4bde14950a4cead95f6fcb", "02bea84d70e74f7603746b62d79bf035e16d982b56e6a1ee07dfd3b9130e8a2ad9"] - -@pytest.mark.parametrize( - "nw, wallet_structures, mean_amt, sdev_amt, amount, pubs, k", [ - (1, [[2, 1, 4, 0, 0]], 4, 1.4, 600000000, vpubs[1:4], 2), - (1, [[3, 3, 0, 0, 3]], 4, 1.4, 100000000, vpubs[:4], 3), - ]) -def test_create_p2sh_output_tx(setup_tx_creation, nw, wallet_structures, - mean_amt, sdev_amt, amount, pubs, k): - wallets = make_wallets(nw, wallet_structures, mean_amt, sdev_amt) - for w in wallets.values(): - w['wallet'].sync_wallet(fast=True) - for k, w in enumerate(wallets.values()): - wallet_service = w['wallet'] - ins_full = wallet_service.select_utxos(0, amount) - script = bitcoin.mk_multisig_script(pubs, k) - output_addr = bitcoin.p2sh_scriptaddr(bitcoin.safe_from_hex(script), - magicbyte=196) - txid = make_sign_and_push(ins_full, - wallet_service, - amount, - output_addr=output_addr) - assert txid - -def test_script_to_address(setup_tx_creation): - sample_script = "a914307f099a3bfedec9a09682238db491bade1b467f87" - assert bitcoin.script_to_address( - sample_script, vbyte=5) == "367SYUMqo1Fi4tQsycnmCtB6Ces1Z7EZLH" - assert bitcoin.script_to_address( - sample_script, vbyte=196) == "2MwfecDHsQTm4Gg3RekQdpqAMR15BJrjfRF" - -def test_mktx(setup_tx_creation): - """Testing exceptional conditions; not guaranteed - to create valid tx objects""" - #outpoint structure must be {"outpoint":{"hash":hash, "index": num}} - ins = [{'outpoint': {"hash":x*64, "index":0}, - "script": "", "sequence": 4294967295} for x in ["a", "b", "c"]] - pub = vpubs[0] - addr = bitcoin.pubkey_to_address(pub, magicbyte=get_p2pk_vbyte()) - script = bitcoin.address_to_script(addr) - outs = [script + ":1000", addr+":2000",{"script":script, "value":3000}] - tx = bitcoin.mktx(ins, outs) - print(tx) - #rewrite with invalid output - outs.append({"foo": "bar"}) - with pytest.raises(Exception) as e_info: - tx = bitcoin.mktx(ins, outs) - -def test_bintxhash(setup_tx_creation): - tx = "abcdef1234" - x = bitcoin.bin_txhash(tx) - assert binascii.hexlify(x).decode('ascii') == "121480fc2cccd5103434a9c88b037e08ef6c4f9f95dfb85b56f7043a344613fe" - def test_all_same_priv(setup_tx_creation): #recipient - priv = "aa"*32 + "01" - addr = bitcoin.privkey_to_address(priv, magicbyte=get_p2pk_vbyte()) + priv = b"\xaa"*32 + b"\x01" + pub = bitcoin.privkey_to_pubkey(priv) + addr = str(bitcoin.CCoinAddress.from_scriptPubKey( + bitcoin.CScript([bitcoin.OP_0, bitcoin.Hash160(pub)]))) wallet_service = make_wallets(1, [[1,0,0,0,0]], 1)[0]['wallet'] #make another utxo on the same address addrinwallet = wallet_service.get_addr(0,0,0) @@ -89,52 +42,41 @@ def test_all_same_priv(setup_tx_creation): outs = [{"address": addr, "value": 1000000}] ins = list(insfull.keys()) tx = bitcoin.mktx(ins, outs) - tx = bitcoin.signall(tx, wallet_service.get_key_from_addr(addrinwallet)) - -@pytest.mark.parametrize( - "signall", - [ - (True,), - (False,), - ]) -def test_verify_tx_input(setup_tx_creation, signall): - priv = "aa"*32 + "01" - addr = bitcoin.privkey_to_address(priv, magicbyte=get_p2pk_vbyte()) + scripts = {} + for i, j in enumerate(ins): + scripts[i] = (insfull[j]["script"], insfull[j]["value"]) + success, msg = wallet_service.sign_tx(tx, scripts) + assert success, msg + +def test_verify_tx_input(setup_tx_creation): + priv = b"\xaa"*32 + b"\x01" + pub = bitcoin.privkey_to_pubkey(priv) + script = bitcoin.pubkey_to_p2sh_p2wpkh_script(pub) + addr = str(bitcoin.CCoinAddress.from_scriptPubKey(script)) wallet_service = make_wallets(1, [[2,0,0,0,0]], 1)[0]['wallet'] wallet_service.sync_wallet(fast=True) insfull = wallet_service.select_utxos(0, 110000000) - print(insfull) outs = [{"address": addr, "value": 1000000}] ins = list(insfull.keys()) tx = bitcoin.mktx(ins, outs) - desertx = bitcoin.deserialize(tx) - print(desertx) - if signall: - privdict = {} - for index, ins in enumerate(desertx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - ad = insfull[utxo]['address'] - priv = wallet_service.get_key_from_addr(ad) - privdict[utxo] = priv - tx = bitcoin.signall(tx, privdict) - else: - for index, ins in enumerate(desertx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - ad = insfull[utxo]['address'] - priv = wallet_service.get_key_from_addr(ad) - tx = bitcoin.sign(tx, index, priv) - desertx2 = bitcoin.deserialize(tx) - print(desertx2) - sig, pub = bitcoin.deserialize_script(desertx2['ins'][0]['script']) - print(sig, pub) - pubscript = bitcoin.address_to_script(bitcoin.pubkey_to_address( - pub, magicbyte=get_p2pk_vbyte())) - sig = binascii.unhexlify(sig) - pub = binascii.unhexlify(pub) - sig_good = bitcoin.verify_tx_input(tx, 0, pubscript, - sig, pub) - assert sig_good - + scripts = {0: (insfull[ins[0]]["script"], bitcoin.coins_to_satoshi(1))} + success, msg = wallet_service.sign_tx(tx, scripts) + assert success, msg + # testing Joinmarket's ability to verify transaction inputs + # of others: pretend we don't have a wallet owning the transaction, + # and instead verify an input using the (sig, pub, scriptCode) data + # that is sent by counterparties: + cScrWit = tx.wit.vtxinwit[0].scriptWitness + sig = cScrWit.stack[0] + pub = cScrWit.stack[1] + scriptSig = tx.vin[0].scriptSig + tx2 = bitcoin.mktx(ins, outs) + res = bitcoin.verify_tx_input(tx2, 0, scriptSig, + bitcoin.pubkey_to_p2sh_p2wpkh_script(pub), + amount = bitcoin.coins_to_satoshi(1), + witness = bitcoin.CScript([sig, pub])) + assert res + def test_absurd_fees(setup_tx_creation): """Test triggering of ValueError exception if the transaction fees calculated from the blockchain @@ -157,7 +99,6 @@ def test_create_sighash_txs(setup_tx_creation): wallet_service.sync_wallet(fast=True) amount = 350000000 ins_full = wallet_service.select_utxos(0, amount) - print("using hashcode: " + str(sighash)) txid = make_sign_and_push(ins_full, wallet_service, amount, hashcode=sighash) assert txid @@ -165,118 +106,39 @@ def test_create_sighash_txs(setup_tx_creation): with pytest.raises(Exception) as e_info: fake_utxos = wallet_service.select_utxos(4, 1000000000) - -def test_spend_p2sh_utxos(setup_tx_creation): - #make a multisig address from 3 privs - privs = [struct.pack(b'B', x) * 32 + b'\x01' for x in range(1, 4)] - pubs = [bitcoin.privkey_to_pubkey(binascii.hexlify(priv).decode('ascii')) for priv in privs] - script = bitcoin.mk_multisig_script(pubs, 2) - msig_addr = bitcoin.p2sh_scriptaddr(script, magicbyte=196) - #pay into it - wallet_service = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet'] - wallet_service.sync_wallet(fast=True) - amount = 350000000 - ins_full = wallet_service.select_utxos(0, amount) - txid = make_sign_and_push(ins_full, wallet_service, amount, output_addr=msig_addr) - assert txid - #wait for mining - time.sleep(1) - #spend out; the input can be constructed from the txid of previous - msig_in = txid + ":0" - ins = [msig_in] - #random output address and change addr - output_addr = wallet_service.get_internal_addr(1) - amount2 = amount - 50000 - outs = [{'value': amount2, 'address': output_addr}] - tx = bitcoin.mktx(ins, outs) - sigs = [] - for priv in privs[:2]: - sigs.append(bitcoin.get_p2sh_signature(tx, 0, script, binascii.hexlify(priv).decode('ascii'))) - tx = bitcoin.apply_multisignatures(tx, 0, script, sigs) - txid = jm_single().bc_interface.pushtx(tx) - assert txid - def test_spend_p2wpkh(setup_tx_creation): #make 3 p2wpkh outputs from 3 privs privs = [struct.pack(b'B', x) * 32 + b'\x01' for x in range(1, 4)] - pubs = [bitcoin.privkey_to_pubkey( - binascii.hexlify(priv).decode('ascii')) for priv in privs] + pubs = [bitcoin.privkey_to_pubkey(priv) for priv in privs] scriptPubKeys = [bitcoin.pubkey_to_p2wpkh_script(pub) for pub in pubs] - addresses = [bitcoin.pubkey_to_p2wpkh_address(pub) for pub in pubs] + addresses = [str(bitcoin.CCoinAddress.from_scriptPubKey( + spk)) for spk in scriptPubKeys] #pay into it wallet_service = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet'] wallet_service.sync_wallet(fast=True) amount = 35000000 p2wpkh_ins = [] - for addr in addresses: + for i, addr in enumerate(addresses): ins_full = wallet_service.select_utxos(0, amount) txid = make_sign_and_push(ins_full, wallet_service, amount, output_addr=addr) assert txid - p2wpkh_ins.append(txid + ":0") + p2wpkh_ins.append((txid, 0)) + txhex = jm_single().bc_interface.get_transaction(txid) #wait for mining - time.sleep(1) + jm_single().bc_interface.tick_forward_chain(1) #random output address output_addr = wallet_service.get_internal_addr(1) amount2 = amount*3 - 50000 outs = [{'value': amount2, 'address': output_addr}] tx = bitcoin.mktx(p2wpkh_ins, outs) - sigs = [] - for i, priv in enumerate(privs): - # sign each of 3 inputs - tx = bitcoin.p2wpkh_sign(tx, i, binascii.hexlify(priv), - amount, native=True) - # check that verify_tx_input correctly validates; - # to do this, we need to extract the signature and get the scriptCode - # of this pubkey - scriptCode = bitcoin.pubkey_to_p2pkh_script(pubs[i]) - witness = bitcoin.deserialize(tx)['ins'][i]['txinwitness'] - assert len(witness) == 2 - assert witness[1] == pubs[i] - sig = witness[0] - assert bitcoin.verify_tx_input(tx, i, scriptPubKeys[i], sig, - pubs[i], scriptCode=scriptCode, amount=amount) - txid = jm_single().bc_interface.pushtx(tx) - assert txid - -def test_spend_p2wsh(setup_tx_creation): - #make 2 x 2 of 2multisig outputs; will need 4 privs - privs = [struct.pack(b'B', x) * 32 + b'\x01' for x in range(1, 5)] - privs = [binascii.hexlify(priv).decode('ascii') for priv in privs] - pubs = [bitcoin.privkey_to_pubkey(priv) for priv in privs] - redeemScripts = [bitcoin.mk_multisig_script(pubs[i:i+2], 2) for i in [0, 2]] - scriptPubKeys = [bitcoin.pubkeys_to_p2wsh_multisig_script(pubs[i:i+2]) for i in [0, 2]] - addresses = [bitcoin.pubkeys_to_p2wsh_multisig_address(pubs[i:i+2]) for i in [0, 2]] - #pay into it - wallet_service = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet'] - wallet_service.sync_wallet(fast=True) - amount = 35000000 - p2wsh_ins = [] - for addr in addresses: - ins_full = wallet_service.select_utxos(0, amount) - txid = make_sign_and_push(ins_full, wallet_service, amount, output_addr=addr) - assert txid - p2wsh_ins.append(txid + ":0") - #wait for mining - time.sleep(1) - #random output address and change addr - output_addr = wallet_service.get_internal_addr(1) - amount2 = amount*2 - 50000 - outs = [{'value': amount2, 'address': output_addr}] - tx = bitcoin.mktx(p2wsh_ins, outs) - sigs = [] - for i in range(2): - sigs = [] - for priv in privs[i*2:i*2+2]: - # sign input j with each of 2 keys - sig = bitcoin.get_p2sh_signature(tx, i, redeemScripts[i], priv, amount=amount) - sigs.append(sig) - # check that verify_tx_input correctly validates; - assert bitcoin.verify_tx_input(tx, i, scriptPubKeys[i], sig, - bitcoin.privkey_to_pubkey(priv), - scriptCode=redeemScripts[i], amount=amount) - tx = bitcoin.apply_p2wsh_multisignatures(tx, i, redeemScripts[i], sigs) - txid = jm_single().bc_interface.pushtx(tx) + for i, priv in enumerate(privs): + # sign each of 3 inputs; note that bitcoin.sign + # automatically validates each signature it creates. + sig, msg = bitcoin.sign(tx, i, priv, amount=amount, native=True) + if not sig: + assert False, msg + txid = jm_single().bc_interface.pushtx(tx.serialize()) assert txid def test_spend_freeze_script(setup_tx_creation): @@ -320,7 +182,6 @@ def test_spend_freeze_script(setup_tx_creation): push_success = jm_single().bc_interface.pushtx(tx) assert push_success == required_success - @pytest.fixture(scope="module") def setup_tx_creation(): load_test_config() diff --git a/jmclient/test/test_valid_addresses.py b/jmclient/test/test_valid_addresses.py index a3d8238..fcacc64 100644 --- a/jmclient/test/test_valid_addresses.py +++ b/jmclient/test/test_valid_addresses.py @@ -1,10 +1,18 @@ from jmclient.configure import validate_address, load_test_config from jmclient import jm_single +import jmbitcoin as btc import json import pytest import os testdir = os.path.dirname(os.path.realpath(__file__)) +def address_valid_somewhere(addr): + for x in ["bitcoin", "bitcoin/testnet", "bitcoin/regtest"]: + btc.select_chain_params(x) + if validate_address(addr)[0]: + return True + return False + def test_non_addresses(setup_addresses): #could flesh this out with other examples res, msg = validate_address(2) @@ -17,9 +25,8 @@ def test_b58_invalid_addresses(setup_addresses): invalid_key_list = json.loads(json_data) for k in invalid_key_list: bad_key = k[0] - res, message = validate_address(bad_key) - assert res == False, "Incorrectly validated address: " + bad_key + " with message: " + message - + res = address_valid_somewhere(bad_key) + assert res == False, "Incorrectly validated address: " + bad_key def test_b58_valid_addresses(): with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f: @@ -30,30 +37,36 @@ def test_b58_valid_addresses(): if not prop_dict["isPrivkey"]: if prop_dict["isTestnet"]: jm_single().config.set("BLOCKCHAIN", "network", "testnet") + btc.select_chain_params("bitcoin/testnet") else: jm_single().config.set("BLOCKCHAIN", "network", "mainnet") + btc.select_chain_params("bitcoin") #if using pytest -s ; sanity check to see what's actually being tested - print('testing this address: ', addr) res, message = validate_address(addr) assert res == True, "Incorrectly failed to validate address: " + addr + " with message: " + message jm_single().config.set("BLOCKCHAIN", "network", "testnet") + btc.select_chain_params("bitcoin/regtest") def test_valid_bech32_addresses(): valids = ["BC1QW508D6QEJXTDG4Y5R3ZARVARY0C5XW7KV8F3T4", "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sl5k7", - "bc1pw508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7k7grplx", - "BC1SW50QA3JX3S", - "bc1zw508d6qejxtdg4y5r3zarvaryvg6kdaj", + # TODO these are valid bech32 addresses but rejected by bitcointx + # because they are not witness version 0; add others. + #"bc1pw508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7k7grplx", + #"BC1SW50QA3JX3S", + #"bc1zw508d6qejxtdg4y5r3zarvaryvg6kdaj", "tb1qqqqqp399et2xygdj5xreqhjjvcmzhxw4aywxecjdzew6hylgvsesrxh6hy"] for va in valids: - print("Testing this address: ", va) if va.lower()[:2] == "bc": jm_single().config.set("BLOCKCHAIN", "network", "mainnet") + btc.select_chain_params("bitcoin") else: jm_single().config.set("BLOCKCHAIN", "network", "testnet") + btc.select_chain_params("bitcoin/testnet") res, message = validate_address(va) assert res == True, "Incorrect failed to validate address: " + va + " with message: " + message jm_single().config.set("BLOCKCHAIN", "network", "testnet") + btc.select_chain_params("bitcoin/regtest") def test_invalid_bech32_addresses(): invalids = [ @@ -68,8 +81,7 @@ def test_invalid_bech32_addresses(): "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3pjxtptv", "bc1gmk9yu"] for iva in invalids: - print("Testing this address: ", iva) - res, message = validate_address(iva) + res = address_valid_somewhere(iva) assert res == False, "Incorrectly validated address: " + iva @pytest.fixture(scope="module") diff --git a/jmclient/test/test_wallet.py b/jmclient/test/test_wallet.py index 0926947..e0e0e4a 100644 --- a/jmclient/test/test_wallet.py +++ b/jmclient/test/test_wallet.py @@ -6,12 +6,14 @@ from binascii import hexlify, unhexlify import pytest import jmbitcoin as btc -from commontest import binarize_tx, ensure_bip65_activated -from jmbase import get_log +from commontest import ensure_bip65_activated +from jmbase import (get_log, utxostr_to_utxo, utxo_to_utxostr, + hextobin, bintohex) from jmclient import load_test_config, jm_single, \ SegwitLegacyWallet,BIP32Wallet, BIP49Wallet, LegacyWallet,\ VolatileStorage, get_network, cryptoengine, WalletError,\ SegwitWallet, WalletService, SegwitLegacyWalletFidelityBonds,\ + BTC_P2PKH, BTC_P2SH_P2WPKH,\ FidelityBondMixin, FidelityBondWatchonlyWallet, wallet_gettimelockaddress from test_blockchaininterface import sync_test_wallet @@ -20,10 +22,7 @@ log = get_log() def signed_tx_is_segwit(tx): - for inp in tx['ins']: - if 'txinwitness' not in inp: - return False - return True + return tx.has_witness() def assert_segwit(tx): @@ -47,10 +46,11 @@ def get_populated_wallet(amount=10**8, num=3): def fund_wallet_addr(wallet, addr, value_btc=1): - txin_id = jm_single().bc_interface.grab_coins(addr, value_btc) + # special case, grab_coins returns hex from rpc: + txin_id = hextobin(jm_single().bc_interface.grab_coins(addr, value_btc)) txinfo = jm_single().bc_interface.get_transaction(txin_id) - txin = btc.deserialize(unhexlify(txinfo['hex'])) - utxo_in = wallet.add_new_utxos_(txin, unhexlify(txin_id), 1) + txin = btc.CMutableTransaction.deserialize(btc.x(txinfo["hex"])) + utxo_in = wallet.add_new_utxos(txin, 1) assert len(utxo_in) == 1 return list(utxo_in.keys())[0] @@ -367,13 +367,15 @@ def test_signing_imported(setup_wallet, wif, keytype, type_check): path = wallet.import_private_key(MIXDEPTH, wif, keytype) utxo = fund_wallet_addr(wallet, wallet.get_address_from_path(path)) # The dummy output is constructed as an unspendable p2sh: - tx = btc.deserialize(btc.mktx(['{}:{}'.format( - hexlify(utxo[0]).decode('ascii'), utxo[1])], - [btc.p2sh_scriptaddr(b"\x00",magicbyte=196) + ':' + str(10**8 - 9000)])) + tx = btc.mktx([utxo], + [{"address": str(btc.CCoinAddress.from_scriptPubKey( + btc.CScript(b"\x00").to_p2sh_scriptPubKey())), + "value": 10**8 - 9000}]) script = wallet.get_script_from_path(path) - tx = wallet.sign_tx(tx, {0: (script, 10**8)}) + success, msg = wallet.sign_tx(tx, {0: (script, 10**8)}) + assert success, msg type_check(tx) - txout = jm_single().bc_interface.pushtx(btc.serialize(tx)) + txout = jm_single().bc_interface.pushtx(tx.serialize()) assert txout @@ -385,17 +387,19 @@ def test_signing_imported(setup_wallet, wif, keytype, type_check): def test_signing_simple(setup_wallet, wallet_cls, type_check): jm_single().config.set('BLOCKCHAIN', 'network', 'testnet') storage = VolatileStorage() - wallet_cls.initialize(storage, get_network()) + wallet_cls.initialize(storage, get_network(), entropy=b"\xaa"*16) wallet = wallet_cls(storage) utxo = fund_wallet_addr(wallet, wallet.get_internal_addr(0)) # The dummy output is constructed as an unspendable p2sh: - tx = btc.deserialize(btc.mktx(['{}:{}'.format( - hexlify(utxo[0]).decode('ascii'), utxo[1])], - [btc.p2sh_scriptaddr(b"\x00",magicbyte=196) + ':' + str(10**8 - 9000)])) + tx = btc.mktx([utxo], + [{"address": str(btc.CCoinAddress.from_scriptPubKey( + btc.CScript(b"\x00").to_p2sh_scriptPubKey())), + "value": 10**8 - 9000}]) script = wallet.get_script(0, 1, 0) - tx = wallet.sign_tx(tx, {0: (script, 10**8)}) + success, msg = wallet.sign_tx(tx, {0: (script, 10**8)}) + assert success, msg type_check(tx) - txout = jm_single().bc_interface.pushtx(btc.serialize(tx)) + txout = jm_single().bc_interface.pushtx(tx.serialize()) assert txout def test_timelocked_output_signing(setup_wallet): @@ -450,18 +454,18 @@ def test_add_utxos(setup_wallet): for md in range(1, wallet.max_mixdepth + 1): assert balances[md] == 0 - utxos = wallet.get_utxos_by_mixdepth_() + utxos = wallet.get_utxos_by_mixdepth() assert len(utxos[0]) == num_tx for md in range(1, wallet.max_mixdepth + 1): assert not utxos[md] with pytest.raises(Exception): # no funds in mixdepth - wallet.select_utxos_(1, amount) + wallet.select_utxos(1, amount) with pytest.raises(Exception): # not enough funds - wallet.select_utxos_(0, amount * (num_tx + 1)) + wallet.select_utxos(0, amount * (num_tx + 1)) wallet.reset_utxos() assert wallet.get_balance_by_mixdepth()[0] == 0 @@ -472,12 +476,12 @@ def test_select_utxos(setup_wallet): amount = 10**8 wallet = get_populated_wallet(amount) - utxos = wallet.select_utxos_(0, amount // 2) + utxos = wallet.select_utxos(0, amount // 2) assert len(utxos) == 1 utxos = list(utxos.keys()) - more_utxos = wallet.select_utxos_(0, int(amount * 1.5), utxo_filter=utxos) + more_utxos = wallet.select_utxos(0, int(amount * 1.5), utxo_filter=utxos) assert len(more_utxos) == 2 assert utxos[0] not in more_utxos @@ -488,14 +492,11 @@ def test_add_new_utxos(setup_wallet): scripts = [wallet.get_new_script(x, True) for x in range(3)] tx_scripts = list(scripts) - tx_scripts.append(b'\x22'*17) - - tx = btc.deserialize(btc.mktx( - ['0'*64 + ':2'], [{'script': hexlify(s).decode('ascii'), 'value': 10**8} - for s in tx_scripts])) - binarize_tx(tx) - txid = b'\x01' * 32 - added = wallet.add_new_utxos_(tx, txid, 1) + tx = btc.mktx( + [(b"\x00"*32, 2)], + [{"address": wallet.script_to_addr(s), + "value": 10**8} for s in tx_scripts]) + added = wallet.add_new_utxos(tx, 1) assert len(added) == len(scripts) added_scripts = {x['script'] for x in added.values()} @@ -517,21 +518,20 @@ def test_remove_old_utxos(setup_wallet): for i in range(3): txin = jm_single().bc_interface.grab_coins( wallet.get_internal_addr(1), 1) - wallet.add_utxo(unhexlify(txin), 0, wallet.get_script(1, 1, i), 10**8, 1) + wallet.add_utxo(btc.x(txin), 0, wallet.get_script(1, 1, i), 10**8, 1) - inputs = wallet.select_utxos_(0, 10**8) - inputs.update(wallet.select_utxos_(1, 2 * 10**8)) + inputs = wallet.select_utxos(0, 10**8) + inputs.update(wallet.select_utxos(1, 2 * 10**8)) assert len(inputs) == 3 tx_inputs = list(inputs.keys()) tx_inputs.append((b'\x12'*32, 6)) - tx = btc.deserialize(btc.mktx( - ['{}:{}'.format(hexlify(txid).decode('ascii'), i) for txid, i in tx_inputs], - ['0' * 36 + ':' + str(3 * 10**8 - 1000)])) - binarize_tx(tx) + tx = btc.mktx(tx_inputs, + [{"address": "2N9gfkUsFW7Kkb1Eurue7NzUxUt7aNJiS1U", + "value": 3 * 10**8 - 1000}]) - removed = wallet.remove_old_utxos_(tx) + removed = wallet.remove_old_utxos(tx) assert len(removed) == len(inputs) for txid in removed: @@ -760,6 +760,14 @@ def test_wallet_mixdepth_decrease(setup_wallet): max_mixdepth = wallet.max_mixdepth assert max_mixdepth >= 1, "bad default value for mixdepth for this test" utxo = fund_wallet_addr(wallet, wallet.get_internal_addr(max_mixdepth), 1) + bci = jm_single().bc_interface + unspent_list = bci.rpc('listunspent', [0]) + # filter on label, but note (a) in certain circumstances (in- + # wallet transfer) it is possible for the utxo to be labeled + # with the external label, and (b) the wallet will know if it + # belongs or not anyway (is_known_addr): + our_unspent_list = [x for x in unspent_list if ( + bci.is_address_labeled(x, wallet.get_wallet_name()))] assert wallet.get_balance_by_mixdepth()[max_mixdepth] == 10**8 wallet.close() storage_data = wallet._storage.file_data @@ -777,7 +785,7 @@ def test_wallet_mixdepth_decrease(setup_wallet): # wallet.select_utxos will still return utxos from higher mixdepths # because we explicitly ask for a specific mixdepth - assert utxo in new_wallet.select_utxos_(max_mixdepth, 10**7) + assert utxo in new_wallet.select_utxos(max_mixdepth, 10**7) def test_watchonly_wallet(setup_wallet): jm_single().config.set('BLOCKCHAIN', 'network', 'testnet') @@ -819,6 +827,7 @@ def test_watchonly_wallet(setup_wallet): @pytest.fixture(scope='module') def setup_wallet(): load_test_config() + btc.select_chain_params("bitcoin/regtest") #see note in cryptoengine.py: cryptoengine.BTC_P2WPKH.VBYTE = 100 jm_single().bc_interface.tick_forward_chain_interval = 2 diff --git a/jmclient/test/test_wallets.py b/jmclient/test/test_wallets.py index 66d6058..7344ead 100644 --- a/jmclient/test/test_wallets.py +++ b/jmclient/test/test_wallets.py @@ -8,7 +8,8 @@ from commontest import create_wallet_for_sync, make_sign_and_push import json import pytest -from jmbase import get_log +from jmbase import get_log, hextobin +import jmbitcoin as btc from jmclient import ( load_test_config, jm_single, estimate_tx_fee, BitcoinCoreInterface, Mnemonic) @@ -43,9 +44,10 @@ def test_query_utxo_set(setup_wallets): txid = do_tx(wallet_service, 90000000) txid2 = do_tx(wallet_service, 20000000) print("Got txs: ", txid, txid2) - res1 = jm_single().bc_interface.query_utxo_set(txid + ":0", includeunconf=True) + res1 = jm_single().bc_interface.query_utxo_set( + (txid, 0), includeunconf=True) res2 = jm_single().bc_interface.query_utxo_set( - [txid + ":0", txid2 + ":1"], + [(txid, 0), (txid2, 1)], includeconf=True, includeunconf=True) assert len(res1) == 1 assert len(res2) == 2 @@ -53,7 +55,7 @@ def test_query_utxo_set(setup_wallets): assert not 'confirms' in res1[0] assert 'confirms' in res2[0] assert 'confirms' in res2[1] - res3 = jm_single().bc_interface.query_utxo_set("ee" * 32 + ":25") + res3 = jm_single().bc_interface.query_utxo_set((b"\xee" * 32, 25)) assert res3 == [None] @@ -69,11 +71,11 @@ def test_wrong_network_bci(setup_wallets): def test_pushtx_errors(setup_wallets): """Ensure pushtx fails return False """ - badtxhex = "aaaa" - assert not jm_single().bc_interface.pushtx(badtxhex) + badtx = b"\xaa\xaa" + assert not jm_single().bc_interface.pushtx(badtx) #Break the authenticated jsonrpc and try again jm_single().bc_interface.jsonRpc.port = 18333 - assert not jm_single().bc_interface.pushtx(t_raw_signed_tx) + assert not jm_single().bc_interface.pushtx(hextobin(t_raw_signed_tx)) #rebuild a valid jsonrpc inside the bci load_test_config() diff --git a/jmclient/test/test_walletservice.py b/jmclient/test/test_walletservice.py index a652a40..f46238b 100644 --- a/jmclient/test/test_walletservice.py +++ b/jmclient/test/test_walletservice.py @@ -39,7 +39,6 @@ def test_address_reuse_freezing(setup_walletservice): """ context = {'cb_called': 0} def reuse_callback(utxostr): - print("Address reuse freezing callback on utxo: ", utxostr) context['cb_called'] += 1 # we must fund after initial sync (for imports), hence # "populated" with no coins diff --git a/jmclient/test/test_yieldgenerator.py b/jmclient/test/test_yieldgenerator.py index 2fd2539..332c50b 100644 --- a/jmclient/test/test_yieldgenerator.py +++ b/jmclient/test/test_yieldgenerator.py @@ -1,6 +1,6 @@ import unittest - +from jmbitcoin import CMutableTxOut, CMutableTransaction from jmclient import load_test_config, jm_single,\ SegwitLegacyWallet, VolatileStorage, YieldGeneratorBasic, \ get_network, WalletService @@ -28,14 +28,12 @@ class CustomUtxoWallet(SegwitLegacyWallet): self.add_utxo_at_mixdepth(m, b) def add_utxo_at_mixdepth(self, mixdepth, balance): - tx = {'outs': [{'script': self.get_internal_script(mixdepth), - 'value': balance}]} - # We need to generate a fake "txid" that has to be unique for all the - # UTXOs that are added to the wallet. For that, we simply use the - # script, and make it fit the required length (32 bytes). - txid = tx['outs'][0]['script'] + b'x' * 32 - txid = txid[:32] - self.add_new_utxos_(tx, txid, 1) + txout = CMutableTxOut(balance, self.get_internal_script(mixdepth)) + tx = CMutableTransaction() + tx.vout = [txout] + # (note: earlier requirement that txid be generated uniquely is now + # automatic; tx.GetTxid() functions correctly within the wallet). + self.add_new_utxos(tx, 1) def assert_utxos_from_mixdepth(self, utxos, expected): """Asserts that the list of UTXOs (as returned from UTXO selection diff --git a/jmdaemon/test/test_message_channel.py b/jmdaemon/test/test_message_channel.py index 949e36e..9ca3891 100644 --- a/jmdaemon/test/test_message_channel.py +++ b/jmdaemon/test/test_message_channel.py @@ -15,6 +15,7 @@ import base64 import struct import traceback import threading +import binascii import jmbitcoin as bitcoin from dummy_mc import DummyMessageChannel @@ -22,10 +23,11 @@ from dummy_mc import DummyMessageChannel jlog = get_log() def make_valid_nick(i=0): - nick_priv = hashlib.sha256(struct.pack(b'B', i)*16).hexdigest() + '01' - nick_pubkey = bitcoin.privtopub(nick_priv) - nick_pkh_raw = hashlib.sha256(nick_pubkey.encode('ascii')).digest()[:NICK_HASH_LENGTH] - nick_pkh = bitcoin.b58encode(nick_pkh_raw) + nick_priv = hashlib.sha256(struct.pack(b'B', i)*16).digest() + b"\x01" + nick_pubkey = bitcoin.privkey_to_pubkey(nick_priv) + nick_pkh_raw = hashlib.sha256(binascii.hexlify( + nick_pubkey)).digest()[:NICK_HASH_LENGTH] + nick_pkh = bitcoin.base58.encode(nick_pkh_raw) #right pad to maximum possible; b58 is not fixed length. #Use 'O' as one of the 4 not included chars in base58. nick_pkh += 'O' * (NICK_MAX_ENCODED - len(nick_pkh)) diff --git a/scripts/add-utxo.py b/scripts/add-utxo.py index e564df6..ea6de44 100755 --- a/scripts/add-utxo.py +++ b/scripts/add-utxo.py @@ -17,7 +17,7 @@ import jmbitcoin as btc from jmclient import load_program_config, jm_single, get_p2pk_vbyte,\ open_wallet, WalletService, add_external_commitments, update_commitments,\ PoDLE, get_podle_commitments, get_utxo_info, validate_utxo_data, quit,\ - get_wallet_path, add_base_options + get_wallet_path, add_base_options, BTCEngine, BTC_P2SH_P2WPKH from jmbase.support import EXIT_SUCCESS, EXIT_FAILURE, EXIT_ARGERROR, jmprint @@ -32,9 +32,10 @@ def add_ext_commitments(utxo_datas): This calls the underlying 'raw' code based on the class PoDLE, not the library 'generate_podle' which intelligently searches and updates commitments. """ - #Convert priv to hex - hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte()) - podle = PoDLE(u, hexpriv) + #Convert priv from wif; require P2SH-P2WPKH keys + rawpriv, keytype = BTCEngine.wif_to_privkey(priv) + assert keytype == BTC_P2SH_P2WPKH + podle = PoDLE(u, rawpriv) r = podle.generate_podle(i) return (r['P'], r['P2'], r['sig'], r['e'], r['commit']) diff --git a/scripts/convert_old_wallet.py b/scripts/convert_old_wallet.py index 4a23cc9..1896547 100755 --- a/scripts/convert_old_wallet.py +++ b/scripts/convert_old_wallet.py @@ -7,10 +7,9 @@ from binascii import hexlify, unhexlify from collections import defaultdict from pyaes import AESModeOfOperationCBC, Decrypter from jmbase import JM_APP_NAME -from jmclient import Storage, load_program_config +from jmclient import Storage, load_program_config, BTCEngine from jmclient.wallet_utils import get_password, get_wallet_cls,\ cli_get_wallet_passphrase_check, get_wallet_path -from jmbitcoin import wif_compressed_privkey class ConvertException(Exception): @@ -99,7 +98,7 @@ def new_wallet_from_data(data, file_name): for md in data['imported']: for privkey in data['imported'][md]: privkey += b'\x01' - wif = wif_compressed_privkey(hexlify(privkey).decode('ascii')) + wif = BTCEngine.privkey_to_wif(privkey) wallet.import_private_key(md, wif) wallet.save() diff --git a/scripts/joinmarket-qt.py b/scripts/joinmarket-qt.py index d10bdc8..12eee3a 100755 --- a/scripts/joinmarket-qt.py +++ b/scripts/joinmarket-qt.py @@ -75,7 +75,7 @@ from jmclient import load_program_config, get_network, update_persist_config,\ wallet_generate_recover_bip39, wallet_display, get_utxos_enabled_disabled,\ NO_ROUNDING, get_max_cj_fee_values, get_default_max_absolute_fee, \ get_default_max_relative_fee, RetryableStorageError, add_base_options, \ - FidelityBondMixin + BTCEngine, BTC_P2SH_P2WPKH, FidelityBondMixin from qtsupport import ScheduleWizard, TumbleRestartWizard, config_tips,\ config_types, QtHandler, XStream, Buttons, OkButton, CancelButton,\ PasswordDialog, MyTreeWidget, JMQtMessageBox, BLUE_FG,\ @@ -1530,14 +1530,18 @@ class JMMainWindow(QMainWindow): done = False def privkeys_thread(): + # To explain this (given setting was already done in + # load_program_config), see: + # https://github.com/Simplexum/python-bitcointx/blob/9f1fa67a5445f8c187ef31015a4008bc5a048eea/bitcointx/__init__.py#L242-L243 + # note, we ignore the return value as we only want to apply + # the chainparams setting logic: + get_blockchain_interface_instance(jm_single().config) for addr in addresses: time.sleep(0.1) if done: break priv = self.wallet_service.get_key_from_addr(addr) - private_keys[addr] = btc.wif_compressed_privkey( - priv, - vbyte=get_p2pk_vbyte()) + private_keys[addr] = BTCEngine.privkey_to_wif(priv) self.computing_privkeys_signal.emit() self.show_privkeys_signal.emit() @@ -1569,10 +1573,13 @@ class JMMainWindow(QMainWindow): privkeys_fn + '.json'), "wb") as f: for addr, pk in private_keys.items(): #sanity check - if not addr == btc.pubkey_to_p2sh_p2wpkh_address( - btc.privkey_to_pubkey( - btc.from_wif_privkey(pk, vbyte=get_p2pk_vbyte()) - ), get_p2sh_vbyte()): + rawpriv, keytype = BTCEngine.wif_to_privkey(pk) + if not keytype == BTC_P2SH_P2WPKH: + JMQtMessageBox(None, "Failed to create privkey export, " + "should be keytype p2sh-p2wpkh but is not.", + mbtype='crit') + return + if not addr == self.wallet_service._ENGINE.privkey_to_address(rawpriv): JMQtMessageBox(None, "Failed to create privkey export -" +\ " critical error in key parsing.", mbtype='crit') @@ -2009,7 +2016,7 @@ if isinstance(jm_single().bc_interface, RegtestBitcoinCoreInterface): jm_single().bc_interface.simulating = True jm_single().maker_timeout_sec = 15 #trigger start with a fake tx - jm_single().bc_interface.pushtx("00"*20) + jm_single().bc_interface.pushtx(b"\x00"*20) #prepare for logging for dname in ['logs', 'wallets', 'cmtdata']: diff --git a/scripts/sendtomany.py b/scripts/sendtomany.py index 23941db..33118a0 100755 --- a/scripts/sendtomany.py +++ b/scripts/sendtomany.py @@ -8,10 +8,10 @@ for other reasons). from pprint import pformat from optparse import OptionParser import jmbitcoin as btc -from jmbase import get_log, jmprint +from jmbase import get_log, jmprint, bintohex, utxostr_to_utxo from jmclient import load_program_config, estimate_tx_fee, jm_single,\ get_p2pk_vbyte, validate_address, get_utxo_info, add_base_options,\ - validate_utxo_data, quit + validate_utxo_data, quit, BTCEngine, BTC_P2SH_P2WPKH, BTC_P2PKH log = get_log() @@ -39,10 +39,12 @@ def sign(utxo, priv, destaddrs, segwit=True): log.info("Using fee: " + str(fee)) for i, addr in enumerate(destaddrs): outs.append({'address': addr, 'value': share}) - unsigned_tx = btc.mktx(ins, outs) + tx = btc.mktx(ins, outs) amtforsign = amt if segwit else None - return btc.sign(unsigned_tx, 0, btc.from_wif_privkey( - priv, vbyte=get_p2pk_vbyte()), amount=amtforsign) + rawpriv, _ = BTCEngine.wif_to_privkey(priv) + success, msg = btc.sign(tx, 0, rawpriv, amount=amtforsign) + assert success, msg + return tx def main(): parser = OptionParser( @@ -106,17 +108,20 @@ def main(): for d in destaddrs: if not validate_address(d): quit(parser, "Address was not valid; wrong network?: " + d) - txsigned = sign(u, priv, destaddrs, segwit = not options.nonsegwit) + success, utxo = utxostr_to_utxo(u) + if not success: + quit(parser, "Failed to load utxo from string: " + utxo) + txsigned = sign(utxo, priv, destaddrs, segwit = not options.nonsegwit) if not txsigned: log.info("Transaction signing operation failed, see debug messages for details.") return - log.debug("Got signed transaction:\n" + txsigned) + log.info("Got signed transaction:\n" + bintohex(txsigned.serialize())) log.debug("Deserialized:") - log.debug(pformat(btc.deserialize(txsigned))) + log.debug(pformat(str(txsigned))) if input('Would you like to push to the network? (y/n):')[0] != 'y': log.info("You chose not to broadcast the transaction, quitting.") return - jm_single().bc_interface.pushtx(txsigned) + jm_single().bc_interface.pushtx(txsigned.serialize()) if __name__ == "__main__": main() diff --git a/test/test_segwit.py b/test/test_segwit.py index a222e5f..b5cc7a4 100644 --- a/test/test_segwit.py +++ b/test/test_segwit.py @@ -7,7 +7,7 @@ from common import make_wallets from pprint import pformat import jmbitcoin as btc import pytest -from jmbase import get_log +from jmbase import get_log, hextobin, bintohex from jmclient import load_test_config, jm_single, LegacyWallet log = get_log() @@ -19,9 +19,9 @@ def test_segwit_valid_txs(setup_segwit): for j in valid_txs: if len(j) < 2: continue - deserialized_tx = btc.deserialize(str(j[1])) + deserialized_tx = btc.CMutableTransaction.deserialize(hextobin(j[1])) print(pformat(deserialized_tx)) - assert btc.serialize(deserialized_tx) == str(j[1]) + assert deserialized_tx.serialize() == hextobin(j[1]) #TODO use bcinterface to decoderawtransaction #and compare the json values @@ -63,8 +63,8 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, sw_wallet_service = make_wallets(1, [[len(segwit_ins), 0, 0, 0, 0]], segwit_amt)[0]['wallet'] sw_wallet_service.sync_wallet(fast=True) - nsw_utxos = nsw_wallet_service.get_utxos_by_mixdepth(hexfmt=False)[MIXDEPTH] - sw_utxos = sw_wallet_service.get_utxos_by_mixdepth(hexfmt=False)[MIXDEPTH] + nsw_utxos = nsw_wallet_service.get_utxos_by_mixdepth()[MIXDEPTH] + sw_utxos = sw_wallet_service.get_utxos_by_mixdepth()[MIXDEPTH] assert len(o_ins) <= len(nsw_utxos), "sync failed" assert len(segwit_ins) <= len(sw_utxos), "sync failed" @@ -92,8 +92,8 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, # FIXME: encoding mess, mktx should accept binary input formats tx_ins = [] - for i, (txid, data) in sorted(all_ins.items(), key=lambda x: x[0]): - tx_ins.append('{}:{}'.format(binascii.hexlify(txid[0]).decode('ascii'), txid[1])) + for i, (txin, data) in sorted(all_ins.items(), key=lambda x: x[0]): + tx_ins.append(txin) # create outputs FEE = 50000 @@ -104,11 +104,11 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, change_amt = total_amt_in_sat - amount - FEE tx_outs = [ - {'script': binascii.hexlify(cj_script).decode('ascii'), + {'address': nsw_wallet_service.script_to_addr(cj_script), 'value': amount}, - {'script': binascii.hexlify(change_script).decode('ascii'), + {'address': nsw_wallet_service.script_to_addr(change_script), 'value': change_amt}] - tx = btc.deserialize(btc.mktx(tx_ins, tx_outs)) + tx = btc.mktx(tx_ins, tx_outs) # import new addresses to bitcoind jm_single().bc_interface.import_addresses( @@ -120,18 +120,20 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, for nsw_in_index in o_ins: inp = nsw_ins[nsw_in_index][1] scripts[nsw_in_index] = (inp['script'], inp['value']) - tx = nsw_wallet_service.sign_tx(tx, scripts) + success, msg = nsw_wallet_service.sign_tx(tx, scripts) + assert success, msg scripts = {} for sw_in_index in segwit_ins: inp = sw_ins[sw_in_index][1] scripts[sw_in_index] = (inp['script'], inp['value']) - tx = sw_wallet_service.sign_tx(tx, scripts) + success, msg = sw_wallet_service.sign_tx(tx, scripts) + assert success, msg print(tx) # push and verify - txid = jm_single().bc_interface.pushtx(btc.serialize(tx)) + txid = jm_single().bc_interface.pushtx(tx.serialize()) assert txid balances = jm_single().bc_interface.get_received_by_addr(