python-bitcointx backend for jmbitcoin.

Replaces core transaction, address, serialization
and sign functionality for Bitcoin with
python-bitcointx backend.

Removes bech32 and btscript
modules from jmbitcoin. Removes all string,
hex, binary conversion routines. A generic
hex/binary conversion now is added to jmbase.
Removes all transaction serialization and
deserialization routines. Removes the now
irrelevant test modules.

Remaining functions in jmbitcoin remove any parsing of
hex format, requiring callers to use binary only.

One additional test added, testing the remaining
function in secp256k1_transaction.py: the signing
of transactions. Deserialized form is now
bitcointx.CMutableTransaction.

For jmbase, in addition to the above, generic conversions
for utxos to and from strings is added, and a dynamic conversion
for AMP messages to binary-only. Within the code, utxos are
now only in (binarytxid, int) form, except where converted
for communcation.

Tthe largest part of the changes are
the modifications to jmbitcoin calls in jmclient;
as well as different encapsulation with CMutableTransaction,
there is also a removal of some but not all hex parsing;
it remains for rpc calls to Core and for AMP message
parsing. Backwards compatibility must be ensured so some
joinmarket protocol messages still use hex, and it is
also preserved in persistence of PoDLE data.
As part of this, some significant simplification of
certain legacy functions within the wallet has been done.

jmdaemon is entirely unaltered (save for one test which
simulates jmclient code).

jmbase/jmbase/__init__.py

@@ -2,7 +2,10 @@
 from .support import (get_log, chunks, debug_silence, jmprint,
                       joinmarket_alert, core_alert, get_password,
                       set_logging_level, set_logging_color,
-                      lookup_appdata_folder,
-                      JM_WALLET_NAME_PREFIX, JM_APP_NAME)
+                      lookup_appdata_folder, bintohex, bintolehex,
+                      hextobin, lehextobin, utxostr_to_utxo,
+                      utxo_to_utxostr, EXIT_ARGERROR, EXIT_FAILURE,
+                      EXIT_SUCCESS, hexbin, dictchanger, listchanger,
+                      cv, JM_WALLET_NAME_PREFIX, JM_APP_NAME)
 from .commands import *
 

jmbase/jmbase/support.py

@@ -1,8 +1,9 @@
 
 import logging, sys
+import binascii
 from getpass import getpass
 from os import path, environ
-
+from functools import wraps
 # JoinMarket version
 JM_CORE_VERSION = '0.7.0dev'
 
@@ -80,6 +81,71 @@ handler = JoinMarketStreamHandler()
 handler.setFormatter(logFormatter)
 log.addHandler(handler)
 
+# hex/binary conversion routines used by dependent packages
+def hextobin(h):
+    """Convert a hex string to bytes"""
+    return binascii.unhexlify(h.encode('utf8'))
+
+
+def bintohex(b):
+    """Convert bytes to a hex string"""
+    return binascii.hexlify(b).decode('utf8')
+
+
+def lehextobin(h):
+    """Convert a little-endian hex string to bytes
+
+    Lets you write uint256's and uint160's the way the Satoshi codebase shows
+    them.
+    """
+    return binascii.unhexlify(h.encode('utf8'))[::-1]
+
+
+def bintolehex(b):
+    """Convert bytes to a little-endian hex string
+
+    Lets you show uint256's and uint160's the way the Satoshi codebase shows
+    them.
+    """
+    return binascii.hexlify(b[::-1]).decode('utf8')
+
+def utxostr_to_utxo(x):
+    if not isinstance(x, str):
+        return (False, "not a string")
+    y = x.split(":")
+    if len(y) != 2:
+        return (False,
+                "string is not two items separated by :")
+    try:
+        n = int(y[1])
+    except:
+        return (False, "utxo index was not an integer.")
+    if n < 0:
+        return (False, "utxo index must not be negative.")
+    if len(y[0]) != 64:
+        return (False, "txid is not 64 hex characters.")
+    try:
+        txid = binascii.unhexlify(y[0])
+    except:
+        return (False, "txid is not hex.")
+    return (True, (txid, n))
+
+def utxo_to_utxostr(u):
+    if not isinstance(u, tuple):
+        return (False, "utxo is not a tuple.")
+    if not len(u) == 2:
+        return (False, "utxo should have two elements.")
+    if not isinstance(u[0], bytes):
+        return (False, "txid should be bytes.")
+    if not isinstance(u[1], int):
+        return (False, "index should be int.")
+    if u[1] < 0:
+        return (False, "index must be a positive integer.")
+    if not len(u[0]) == 32:
+        return (False, "txid must be 32 bytes.")
+    txid = binascii.hexlify(u[0]).decode("ascii")
+    return (True, txid + ":" + str(u[1]))
+
 def jmprint(msg, level="info"):
     """ Provides the ability to print messages
     with consistent formatting, outside the logging system
@@ -150,3 +216,67 @@ def lookup_appdata_folder(appname):
 def print_jm_version(option, opt_str, value, parser):
     print("JoinMarket " + JM_CORE_VERSION)
     sys.exit(EXIT_SUCCESS)
+
+# helper functions for conversions of format between over-the-wire JM
+# and internal. See details in hexbin() docstring.
+
+def cv(x):
+    success, utxo = utxostr_to_utxo(x)
+    if success:
+        return utxo
+    else:
+        try:
+            b = hextobin(x)
+            return b
+        except:
+            return x
+
+def listchanger(l):
+    rlist = []
+    for x in l:
+        if isinstance(x, list):
+            rlist.append(listchanger(x))
+        elif isinstance(x, dict):
+            rlist.append(dictchanger(x))
+        else:
+            rlist.append(cv(x))
+    return rlist
+
+def dictchanger(d):
+    rdict = {}
+    for k, v in d.items():
+        if isinstance(v, dict):
+            rdict[cv(k)] = dictchanger(v)
+        elif isinstance(v, list):
+            rdict[cv(k)] = listchanger(v)
+        else:
+            rdict[cv(k)] = cv(v)
+    return rdict
+
+def hexbin(func):
+    """ Decorator for functions of taker and maker receiving over
+    the wire AMP arguments that may be in hex or hextxid:n format
+    and converting all to binary.
+    Functions to which this decorator applies should have all arguments
+    be one of:
+    - hex string (keys), converted here to binary
+    - lists of keys or txid:n strings (converted here to binary, or
+      (txidbytes, n))
+    - lists of lists or dicts, to which these rules apply recursively.
+    - any other string (unchanged)
+    - dicts with keys as per above; values are altered recursively according
+      to the rules above.
+    """
+    @wraps(func)
+    def func_wrapper(inst, *args, **kwargs):
+        newargs = []
+        for arg in args:
+            if isinstance(arg, (list, tuple)):
+                newargs.append(listchanger(arg))
+            elif isinstance(arg, dict):
+                newargs.append(dictchanger(arg))
+            else:
+                newargs.append(cv(arg))
+        return func(inst, *newargs, **kwargs)
+
+    return func_wrapper

jmbitcoin/jmbitcoin/__init__.py

@@ -2,8 +2,14 @@ import coincurve as secp256k1
 from jmbitcoin.secp256k1_main import *
 from jmbitcoin.secp256k1_transaction import *
 from jmbitcoin.secp256k1_deterministic import *
-from jmbitcoin.btscript import *
-from jmbitcoin.bech32 import *
 from jmbitcoin.amount import *
 from jmbitcoin.bip21 import *
+from bitcointx import select_chain_params
+from bitcointx.core import (x, b2x, b2lx, lx, COutPoint, CTxOut, CTxIn,
+                            CTxInWitness, CTxWitness, CMutableTransaction,
+                            Hash160, coins_to_satoshi, satoshi_to_coins)
+from bitcointx.core.script import (CScript, OP_0, SignatureHash, SIGHASH_ALL,
+                                   SIGVERSION_WITNESS_V0, CScriptWitness)
+from bitcointx.wallet import (CBitcoinSecret, P2WPKHBitcoinAddress, CCoinAddress,
+                              P2SHCoinAddress)
 

jmbitcoin/jmbitcoin/bech32.py

@@ -1,122 +0,0 @@
-# Copyright (c) 2017 Pieter Wuille
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-
-"""Reference implementation for Bech32 and segwit addresses."""
-
-CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l"
-
-
-def bech32_polymod(values):
-    """Internal function that computes the Bech32 checksum."""
-    generator = [0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3]
-    chk = 1
-    for value in values:
-        top = chk >> 25
-        chk = (chk & 0x1ffffff) << 5 ^ value
-        for i in range(5):
-            chk ^= generator[i] if ((top >> i) & 1) else 0
-    return chk
-
-
-def bech32_hrp_expand(hrp):
-    """Expand the HRP into values for checksum computation."""
-    return [ord(x) >> 5 for x in hrp] + [0] + [ord(x) & 31 for x in hrp]
-
-
-def bech32_verify_checksum(hrp, data):
-    """Verify a checksum given HRP and converted data characters."""
-    return bech32_polymod(bech32_hrp_expand(hrp) + data) == 1
-
-
-def bech32_create_checksum(hrp, data):
-    """Compute the checksum values given HRP and data."""
-    values = bech32_hrp_expand(hrp) + data
-    polymod = bech32_polymod(values + [0, 0, 0, 0, 0, 0]) ^ 1
-    return [(polymod >> 5 * (5 - i)) & 31 for i in range(6)]
-
-
-def bech32_encode(hrp, data):
-    """Compute a Bech32 string given HRP and data values."""
-    combined = data + bech32_create_checksum(hrp, data)
-    return hrp + '1' + ''.join([CHARSET[d] for d in combined])
-
-
-def bech32_decode(bech):
-    """Validate a Bech32 string, and determine HRP and data."""
-    if ((any(ord(x) < 33 or ord(x) > 126 for x in bech)) or
-            (bech.lower() != bech and bech.upper() != bech)):
-        return (None, None)
-    bech = bech.lower()
-    pos = bech.rfind('1')
-    if pos < 1 or pos + 7 > len(bech) or len(bech) > 90:
-        return (None, None)
-    if not all(x in CHARSET for x in bech[pos+1:]):
-        return (None, None)
-    hrp = bech[:pos]
-    data = [CHARSET.find(x) for x in bech[pos+1:]]
-    if not bech32_verify_checksum(hrp, data):
-        return (None, None)
-    return (hrp, data[:-6])
-
-
-def convertbits(data, frombits, tobits, pad=True):
-    """General power-of-2 base conversion."""
-    acc = 0
-    bits = 0
-    ret = []
-    maxv = (1 << tobits) - 1
-    max_acc = (1 << (frombits + tobits - 1)) - 1
-    for value in data:
-        if value < 0 or (value >> frombits):
-            return None
-        acc = ((acc << frombits) | value) & max_acc
-        bits += frombits
-        while bits >= tobits:
-            bits -= tobits
-            ret.append((acc >> bits) & maxv)
-    if pad:
-        if bits:
-            ret.append((acc << (tobits - bits)) & maxv)
-    elif bits >= frombits or ((acc << (tobits - bits)) & maxv):
-        return None
-    return ret
-
-
-def bech32addr_decode(hrp, addr):
-    """Decode a segwit address."""
-    hrpgot, data = bech32_decode(addr)
-    if hrpgot != hrp:
-        return (None, None)
-    decoded = convertbits(data[1:], 5, 8, False)
-    if decoded is None or len(decoded) < 2 or len(decoded) > 40:
-        return (None, None)
-    if data[0] > 16:
-        return (None, None)
-    if data[0] == 0 and len(decoded) != 20 and len(decoded) != 32:
-        return (None, None)
-    return (data[0], decoded)
-
-
-def bech32addr_encode(hrp, witver, witprog):
-    """Encode a segwit address."""
-    ret = bech32_encode(hrp, [witver] + convertbits(witprog, 8, 5))
-    if bech32addr_decode(hrp, ret) == (None, None):
-        return None
-    return ret

jmbitcoin/jmbitcoin/btscript.py

@@ -1,142 +0,0 @@
-#OP codes; disabled commented.
-
-# push value
-OP_0 = 0x00
-OP_FALSE = OP_0
-OP_PUSHDATA1 = 0x4c
-OP_PUSHDATA2 = 0x4d
-OP_PUSHDATA4 = 0x4e
-OP_1NEGATE = 0x4f
-OP_RESERVED = 0x50
-OP_1 = 0x51
-OP_TRUE = OP_1
-OP_2 = 0x52
-OP_3 = 0x53
-OP_4 = 0x54
-OP_5 = 0x55
-OP_6 = 0x56
-OP_7 = 0x57
-OP_8 = 0x58
-OP_9 = 0x59
-OP_10 = 0x5a
-OP_11 = 0x5b
-OP_12 = 0x5c
-OP_13 = 0x5d
-OP_14 = 0x5e
-OP_15 = 0x5f
-OP_16 = 0x60
-
-# control
-OP_NOP = 0x61
-OP_VER = 0x62
-OP_IF = 0x63
-OP_NOTIF = 0x64
-#OP_VERIF = 0x65
-#OP_VERNOTIF = 0x66
-OP_ELSE = 0x67
-OP_ENDIF = 0x68
-OP_VERIFY = 0x69
-OP_RETURN = 0x6a
-
-# stack ops
-OP_TOALTSTACK = 0x6b
-OP_FROMALTSTACK = 0x6c
-OP_2DROP = 0x6d
-OP_2DUP = 0x6e
-OP_3DUP = 0x6f
-OP_2OVER = 0x70
-OP_2ROT = 0x71
-OP_2SWAP = 0x72
-OP_IFDUP = 0x73
-OP_DEPTH = 0x74
-OP_DROP = 0x75
-OP_DUP = 0x76
-OP_NIP = 0x77
-OP_OVER = 0x78
-OP_PICK = 0x79
-OP_ROLL = 0x7a
-OP_ROT = 0x7b
-OP_SWAP = 0x7c
-OP_TUCK = 0x7d
-
-# splice ops
-#OP_CAT = 0x7e
-#OP_SUBSTR = 0x7f
-#OP_LEFT = 0x80
-#OP_RIGHT = 0x81
-OP_SIZE = 0x82
-
-# bit logic
-#OP_INVERT = 0x83
-#OP_AND = 0x84
-#OP_OR = 0x85
-#OP_XOR = 0x86
-OP_EQUAL = 0x87
-OP_EQUALVERIFY = 0x88
-OP_RESERVED1 = 0x89
-OP_RESERVED2 = 0x8a
-
-# numeric
-OP_1ADD = 0x8b
-OP_1SUB = 0x8c
-#OP_2MUL = 0x8d
-#OP_2DIV = 0x8e
-OP_NEGATE = 0x8f
-OP_ABS = 0x90
-OP_NOT = 0x91
-OP_0NOTEQUAL = 0x92
-
-OP_ADD = 0x93
-OP_SUB = 0x94
-#OP_MUL = 0x95
-#OP_DIV = 0x96
-#OP_MOD = 0x97
-#OP_LSHIFT = 0x98
-#OP_RSHIFT = 0x99
-
-OP_BOOLAND = 0x9a
-OP_BOOLOR = 0x9b
-OP_NUMEQUAL = 0x9c
-OP_NUMEQUALVERIFY = 0x9d
-OP_NUMNOTEQUAL = 0x9e
-OP_LESSTHAN = 0x9f
-OP_GREATERTHAN = 0xa0
-OP_LESSTHANOREQUAL = 0xa1
-OP_GREATERTHANOREQUAL = 0xa2
-OP_MIN = 0xa3
-OP_MAX = 0xa4
-
-OP_WITHIN = 0xa5
-
-# crypto
-OP_RIPEMD160 = 0xa6
-OP_SHA1 = 0xa7
-OP_SHA256 = 0xa8
-OP_HASH160 = 0xa9
-OP_HASH256 = 0xaa
-OP_CODESEPARATOR = 0xab
-OP_CHECKSIG = 0xac
-OP_CHECKSIGVERIFY = 0xad
-OP_CHECKMULTISIG = 0xae
-OP_CHECKMULTISIGVERIFY = 0xaf
-
-# expansion
-OP_NOP1 = 0xb0
-OP_NOP2 = 0xb1
-OP_CHECKLOCKTIMEVERIFY = OP_NOP2
-OP_NOP3 = 0xb2
-OP_NOP4 = 0xb3
-OP_NOP5 = 0xb4
-OP_NOP6 = 0xb5
-OP_NOP7 = 0xb6
-OP_NOP8 = 0xb7
-OP_NOP9 = 0xb8
-OP_NOP10 = 0xb9
-
-# template matching params
-OP_SMALLINTEGER = 0xfa
-OP_PUBKEYS = 0xfb
-OP_PUBKEYHASH = 0xfd
-OP_PUBKEY = 0xfe
-
-OP_INVALIDOPCODE = 0xff

jmbitcoin/jmbitcoin/secp256k1_deterministic.py

@@ -2,6 +2,8 @@ from jmbitcoin.secp256k1_main import *
 import hmac
 import hashlib
 import struct
+from bitcointx.core import Hash160, Hash
+from bitcointx import base58
 
 # Below code ASSUMES binary inputs and compressed pubkeys
 MAINNET_PRIVATE = b'\x04\x88\xAD\xE4'
@@ -11,6 +13,8 @@ TESTNET_PUBLIC = b'\x04\x35\x87\xCF'
 PRIVATE = [MAINNET_PRIVATE, TESTNET_PRIVATE]
 PUBLIC = [MAINNET_PUBLIC, TESTNET_PUBLIC]
 
+privtopub = privkey_to_pubkey
+
 # BIP32 child key derivation
 
 def raw_bip32_ckd(rawtuple, i):
@@ -19,25 +23,25 @@ def raw_bip32_ckd(rawtuple, i):
 
     if vbytes in PRIVATE:
         priv = key
-        pub = privtopub(key, False)
+        pub = privtopub(key)
     else:
         pub = key
 
     if i >= 2**31:
         if vbytes in PUBLIC:
             raise Exception("Can't do private derivation on public key!")
-        I = hmac.new(chaincode, b'\x00' + priv[:32] + encode(i, 256, 4),
+        I = hmac.new(chaincode, b'\x00' + priv[:32] + struct.pack(b'>L', i),
                      hashlib.sha512).digest()
     else:
-        I = hmac.new(chaincode, pub + encode(i, 256, 4),
+        I = hmac.new(chaincode, pub + struct.pack(b'>L', i),
                      hashlib.sha512).digest()
 
     if vbytes in PRIVATE:
-        newkey = add_privkeys(I[:32] + b'\x01', priv, False)
-        fingerprint = bin_hash160(privtopub(key, False))[:4]
+        newkey = add_privkeys(I[:32] + b'\x01', priv)
+        fingerprint = Hash160(privtopub(key))[:4]
     if vbytes in PUBLIC:
-        newkey = add_pubkeys([privtopub(I[:32] + b'\x01', False), key], False)
-        fingerprint = bin_hash160(key)[:4]
+        newkey = add_pubkeys([privtopub(I[:32] + b'\x01'), key])
+        fingerprint = Hash160(key)[:4]
 
     return (vbytes, depth + 1, fingerprint, i, I[32:], newkey)
 
@@ -47,18 +51,17 @@ def bip32_serialize(rawtuple):
         i = struct.pack(b'>L', i)
     chaincode = chaincode
     keydata = b'\x00' + key[:-1] if vbytes in PRIVATE else key
-    bindata = vbytes + from_int_to_byte(
-        depth % 256) + fingerprint + i + chaincode + keydata
-    return b58encode(bindata + bin_dbl_sha256(bindata)[:4])
+    bindata = vbytes + struct.pack(b'B',depth % 256) + fingerprint + i + chaincode + keydata
+    return base58.encode(bindata + Hash(bindata)[:4])
 
 def bip32_deserialize(data):
-    dbin = b58decode(data)
-    if bin_dbl_sha256(dbin[:-4])[:4] != dbin[-4:]:
+    dbin = base58.decode(data)
+    if Hash(dbin[:-4])[:4] != dbin[-4:]:
         raise Exception("Invalid checksum")
     vbytes = dbin[0:4]
     depth = dbin[4]
     fingerprint = dbin[5:9]
-    i = decode(dbin[9:13], 256)
+    i = struct.unpack(b'>L',dbin[9:13])[0]
     chaincode = dbin[13:45]
     key = dbin[46:78] + b'\x01' if vbytes in PRIVATE else dbin[45:78]
     return (vbytes, depth, fingerprint, i, chaincode, key)
@@ -68,7 +71,7 @@ def raw_bip32_privtopub(rawtuple):
     if vbytes in PUBLIC:
         return rawtuple
     newvbytes = MAINNET_PUBLIC if vbytes == MAINNET_PRIVATE else TESTNET_PUBLIC
-    return (newvbytes, depth, fingerprint, i, chaincode, privtopub(key, False))
+    return (newvbytes, depth, fingerprint, i, chaincode, privtopub(key))
 
 def bip32_privtopub(data):
     return bip32_serialize(raw_bip32_privtopub(bip32_deserialize(data)))
@@ -77,13 +80,12 @@ def bip32_ckd(data, i):
     return bip32_serialize(raw_bip32_ckd(bip32_deserialize(data), i))
 
 def bip32_master_key(seed, vbytes=MAINNET_PRIVATE):
-    I = hmac.new(
-        from_string_to_bytes("Bitcoin seed"), seed, hashlib.sha512).digest()
+    I = hmac.new("Bitcoin seed".encode("utf-8"), seed, hashlib.sha512).digest()
     return bip32_serialize((vbytes, 0, b'\x00' * 4, 0, I[32:], I[:32] + b'\x01'
                            ))
 
 def bip32_extract_key(data):
-    return safe_hexlify(bip32_deserialize(data)[-1])
+    return bip32_deserialize(data)[-1]
 
 def bip32_descend(*args):
     if len(args) == 2:

jmbitcoin/jmbitcoin/secp256k1_main.py

@@ -7,213 +7,16 @@ import base64
 import struct
 import coincurve as secp256k1
 
+from bitcointx import base58
+from bitcointx.core import Hash
+from bitcointx.signmessage import BitcoinMessage
+
 #Required only for PoDLE calculation:
 N = 115792089237316195423570985008687907852837564279074904382605163141518161494337
 
 BTC_P2PK_VBYTE = {"mainnet": b'\x00', "testnet": b'\x6f', "regtest": 100}
 BTC_P2SH_VBYTE = {"mainnet": b'\x05', "testnet": b'\xc4'}
 
-#Standard prefix for Bitcoin message signing.
-BITCOIN_MESSAGE_MAGIC = b'\x18' + b'Bitcoin Signed Message:\n'
-
-string_types = (str)
-string_or_bytes_types = (str, bytes)
-int_types = (int, float)
-
-# Base switching
-code_strings = {
-    2: '01',
-    10: '0123456789',
-    16: '0123456789abcdef',
-    32: 'abcdefghijklmnopqrstuvwxyz234567',
-    58: '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz',
-    256: ''.join([chr(x) for x in range(256)])
-}
-
-def lpad(msg, symbol, length):
-    if len(msg) >= length:
-        return msg
-    return symbol * (length - len(msg)) + msg
-
-def get_code_string(base):
-    if base in code_strings:
-        return code_strings[base]
-    else:
-        raise ValueError("Invalid base!")
-
-def bin_to_b58check(inp, magicbyte=b'\x00'):
-    if not isinstance(magicbyte, int):
-        magicbyte = struct.unpack(b'B', magicbyte)[0]
-    assert(0 <= magicbyte <= 0xff)
-    if magicbyte == 0:
-        inp_fmtd = struct.pack(b'B', magicbyte) + inp
-    while magicbyte > 0:
-        inp_fmtd = struct.pack(b'B', magicbyte % 256) + inp
-        magicbyte //= 256
-    checksum = bin_dbl_sha256(inp_fmtd)[:4]
-    return b58encode(inp_fmtd + checksum)
-
-def safe_from_hex(s):
-    return binascii.unhexlify(s)
-
-def from_int_to_byte(a):
-    return struct.pack(b'B', a)
-
-def from_byte_to_int(a):
-    return struct.unpack(b'B', a)[0]
-
-def from_string_to_bytes(a):
-    return a if isinstance(a, bytes) else bytes(a, 'utf-8')
-
-def safe_hexlify(a):
-    return binascii.hexlify(a).decode('ascii')
-
-class SerializationError(Exception):
-    """Base class for serialization errors"""
-
-
-class SerializationTruncationError(SerializationError):
-    """Serialized data was truncated
-    Thrown by deserialize() and stream_deserialize()
-    """
-
-def ser_read(f, n):
-    """Read from a stream safely
-    Raises SerializationError and SerializationTruncationError appropriately.
-    Use this instead of f.read() in your classes stream_(de)serialization()
-    functions.
-    """
-    MAX_SIZE = 0x02000000
-    if n > MAX_SIZE:
-        raise SerializationError('Asked to read 0x%x bytes; MAX_SIZE exceeded' % n)
-    r = f.read(n)
-    if len(r) < n:
-        raise SerializationTruncationError('Asked to read %i bytes, but only got %i' % (n, len(r)))
-    return r
-
-B58_DIGITS = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
-
-class Base58Error(Exception):
-    pass
-
-class InvalidBase58Error(Base58Error):
-    """Raised on generic invalid base58 data, such as bad characters.
-    Checksum failures raise Base58ChecksumError specifically.
-    """
-    pass
-
-def b58encode(b):
-    """Encode bytes to a base58-encoded string"""
-
-    # Convert big-endian bytes to integer
-    n = int('0x0' + binascii.hexlify(b).decode('ascii'), 16)
-
-    # Divide that integer into bas58
-    res = []
-    while n > 0:
-        n, r = divmod(n, 58)
-        res.append(B58_DIGITS[r])
-    res = ''.join(res[::-1])
-
-    # Encode leading zeros as base58 zeros
-    czero = b'\x00'
-    if sys.version_info >= (3,0):
-        # In Python3 indexing a bytes returns numbers, not characters.
-        czero = 0
-    pad = 0
-    for c in b:
-        if c == czero:
-            pad += 1
-        else:
-            break
-    return B58_DIGITS[0] * pad + res
-
-def b58decode(s):
-    """Decode a base58-encoding string, returning bytes"""
-    if not s:
-        return b''
-
-    # Convert the string to an integer
-    n = 0
-    for c in s:
-        n *= 58
-        if c not in B58_DIGITS:
-            raise InvalidBase58Error('Character %r is not a valid base58 character' % c)
-        digit = B58_DIGITS.index(c)
-        n += digit
-
-    # Convert the integer to bytes
-    h = '%x' % n
-    if len(h) % 2:
-        h = '0' + h
-    res = bytes(binascii.unhexlify(h.encode('utf8')))
-
-    # Add padding back.
-    pad = 0
-    for c in s[:-1]:
-        if c == B58_DIGITS[0]: pad += 1
-        else: break
-    return b'\x00' * pad + res
-
-def uint256encode(s):
-    """Convert bytes to uint256"""
-    r = 0
-    t = struct.unpack(b"<IIIIIIII", s[:32])
-    for i in range(8):
-        r += t[i] << (i * 32)
-    return r
-
-def uint256decode(u):
-    r = b""
-    for i in range(8):
-        r += struct.pack(b'<I', u >> (i * 32) & 0xffffffff)
-    return r
-
-def encode(val, base, minlen=0):
-    base, minlen = int(base), int(minlen)
-    code_string = get_code_string(base)
-    result_bytes = bytes()
-    while val > 0:
-        curcode = code_string[val % base]
-        result_bytes = bytes([ord(curcode)]) + result_bytes
-        val //= base
-
-    pad_size = minlen - len(result_bytes)
-
-    padding_element = b'\x00' if base == 256 else b'1' \
-        if base == 58 else b'0'
-    if (pad_size > 0):
-        result_bytes = padding_element*pad_size + result_bytes
-
-    result_string = ''.join([chr(y) for y in result_bytes])
-    result = result_bytes if base == 256 else result_string
-
-    return result
-
-def decode(string, base):
-    if base == 256 and isinstance(string, str):
-        string = bytes(bytearray.fromhex(string))
-    base = int(base)
-    code_string = get_code_string(base)
-    result = 0
-    if base == 256:
-        def extract(d, cs):
-            if isinstance(d, int):
-                return d
-            else:
-                return struct.unpack(b'B', d)[0]
-    else:
-        def extract(d, cs):
-            return cs.find(d if isinstance(d, str) else chr(d))
-
-    if base == 16:
-        string = string.lower()
-    while len(string) > 0:
-        result *= base
-        result += extract(string[0], code_string)
-        string = string[1:]
-    return result
-
 """PoDLE related primitives
 """
 def getG(compressed=True):
@@ -237,165 +40,6 @@ def podle_PrivateKey(priv):
     """
     return secp256k1.PrivateKey(priv)
 
-
-def privkey_to_address(priv, from_hex=True, magicbyte=0):
-    return pubkey_to_address(privkey_to_pubkey(priv, from_hex), magicbyte)
-
-privtoaddr = privkey_to_address
-
-# Hashes
-def bin_hash160(string):
-    intermed = hashlib.sha256(string).digest()
-    return hashlib.new('ripemd160', intermed).digest()
-
-def hash160(string):
-    if not isinstance(string, bytes):
-        string = string.encode('utf-8')
-    return safe_hexlify(bin_hash160(string))
-
-def bin_sha256(string):
-    binary_data = string if isinstance(string, bytes) else bytes(string,
-                                                                 'utf-8')
-    return hashlib.sha256(binary_data).digest()
-
-def sha256(string):
-    return safe_hexlify(bin_sha256(string))
-
-def bin_dbl_sha256(bytes_to_hash):
-    if not isinstance(bytes_to_hash, bytes):
-        bytes_to_hash = bytes_to_hash.encode('utf-8')
-    return hashlib.sha256(hashlib.sha256(bytes_to_hash).digest()).digest()
-
-def dbl_sha256(string):
-    return hashlib.sha256(hashlib.sha256(string).digest()).hexdigest()
-
-def hash_to_int(x):
-    if len(x) in [40, 64]:
-        return decode(x, 16)
-    return decode(x, 256)
-
-def num_to_var_int(x):
-    if not isinstance(x, int):
-        if len(x) == 0:
-            return b'\x00'
-        x = struct.unpack(b'B', x)[0]
-    if x < 253: return from_int_to_byte(x)
-    elif x < 65536: return from_int_to_byte(253) + struct.pack(b'<H', x)
-    elif x < 4294967296: return from_int_to_byte(254) + struct.pack(b'<I', x)
-    else: return from_int_to_byte(255) + struct.pack(b'<Q', x)
-
-def message_sig_hash(message):
-    """Used for construction of signatures of
-    messages, intended to be compatible with Bitcoin Core.
-    """
-    padded = BITCOIN_MESSAGE_MAGIC + num_to_var_int(len(
-        message)) + from_string_to_bytes(message)
-    return bin_dbl_sha256(padded)
-
-# Encodings
-def b58check_to_bin(inp):
-    data = b58decode(inp)
-    assert bin_dbl_sha256(data[:-4])[:4] == data[-4:]
-    return data[1:-4]
-
-def get_version_byte(inp):
-    data = b58decode(inp)
-    assert bin_dbl_sha256(data[:-4])[:4] == data[-4:]
-    return data[:1]
-
-def hex_to_b58check(inp, magicbyte=b'\x00'):
-    return bin_to_b58check(binascii.unhexlify(inp), magicbyte)
-
-def b58check_to_hex(inp):
-    return binascii.hexlify(b58check_to_bin(inp)).decode('ascii')
-
-def pubkey_to_address(pubkey, magicbyte=0):
-    if len(pubkey) in [66, 130]:
-        return bin_to_b58check(
-            bin_hash160(binascii.unhexlify(pubkey)), magicbyte)
-    return bin_to_b58check(bin_hash160(pubkey), magicbyte)
-
-pubtoaddr = pubkey_to_address
-
-def wif_compressed_privkey(priv, vbyte=b'\x00'):
-    """Convert privkey in hex compressed to WIF compressed
-    """
-    if not isinstance(vbyte, int):
-        vbyte = struct.unpack(b'B', vbyte)[0]
-    if len(priv) != 66:
-        raise Exception("Wrong length of compressed private key")
-    if priv[-2:] != '01':
-        raise Exception("Private key has wrong compression byte")
-    return bin_to_b58check(binascii.unhexlify(priv), 128 + int(vbyte))
-
-
-def from_wif_privkey(wif_priv, compressed=True, vbyte=b'\x00'):
-    """Convert WIF compressed privkey to hex compressed.
-    Caller specifies the network version byte (0 for mainnet, 0x6f
-    for testnet) that the key should correspond to; if there is
-    a mismatch an error is thrown. WIF encoding uses 128+ this number.
-    """
-    if isinstance(vbyte, int):
-        vbyte = struct.pack(b'B', vbyte)
-    bin_key = b58check_to_bin(wif_priv)
-    claimed_version_byte = get_version_byte(wif_priv)
-    if not from_int_to_byte(128+from_byte_to_int(vbyte)) == claimed_version_byte:
-        raise Exception(
-            "WIF key version byte is wrong network (mainnet/testnet?)")
-    if compressed and not len(bin_key) == 33:
-        raise Exception("Compressed private key is not 33 bytes")
-    if compressed and not bin_key[-1:] == b'\x01':
-        raise Exception("Private key has incorrect compression byte")
-    return safe_hexlify(bin_key)
-
-def ecdsa_sign(msg, priv, formsg=False, usehex=True):
-    hashed_msg = message_sig_hash(msg)
-    if usehex:
-        #arguments to raw sign must be consistently hex or bin
-        hashed_msg = binascii.hexlify(hashed_msg).decode('ascii')
-    sig = ecdsa_raw_sign(hashed_msg, priv, usehex, rawmsg=True, formsg=formsg)
-    #note those functions only handles binary, not hex
-    if usehex:
-        sig = binascii.unhexlify(sig)
-    return base64.b64encode(sig).decode('ascii')
-
-def ecdsa_verify(msg, sig, pub, usehex=True):
-    hashed_msg = message_sig_hash(msg)
-    sig = base64.b64decode(sig)
-    if usehex:
-        #arguments to raw_verify must be consistently hex or bin
-        hashed_msg = binascii.hexlify(hashed_msg).decode('ascii')
-        sig = binascii.hexlify(sig).decode('ascii')
-    return ecdsa_raw_verify(hashed_msg, pub, sig, usehex, rawmsg=True)
-
-#Use secp256k1 to handle all EC and ECDSA operations.
-#Data types: only hex and binary.
-#Compressed and uncompressed private and public keys.
-def hexbin(func):
-    '''To enable each function to 'speak' either hex or binary,
-    requires that the decorated function's final positional argument
-    is a boolean flag, True for hex and False for binary.
-    '''
-
-    def func_wrapper(*args, **kwargs):
-        if args[-1]:
-            newargs = []
-            for arg in args[:-1]:
-                if isinstance(arg, (list, tuple)):
-                    newargs += [[binascii.unhexlify(x) for x in arg]]
-                else:
-                    newargs += [binascii.unhexlify(arg)]
-            newargs += [False]
-            returnval = func(*newargs, **kwargs)
-            if isinstance(returnval, bool):
-                return returnval
-            else:
-                return binascii.hexlify(returnval).decode('ascii')
-        else:
-            return func(*args, **kwargs)
-
-    return func_wrapper
-
 def read_privkey(priv):
     if len(priv) == 33:
         if priv[-1:] == b'\x01':
@@ -408,32 +52,54 @@ def read_privkey(priv):
         raise Exception("Invalid private key")
     return (compressed, priv[:32])
 
-@hexbin
-def privkey_to_pubkey_inner(priv, usehex):
+def privkey_to_pubkey(priv):
     '''Take 32/33 byte raw private key as input.
     If 32 bytes, return compressed (33 byte) raw public key.
     If 33 bytes, read the final byte as compression flag,
     and return compressed/uncompressed public key as appropriate.'''
     compressed, priv = read_privkey(priv)
     #secp256k1 checks for validity of key value.
-    if sys.version_info >= (3,0):
-        newpriv = secp256k1.PrivateKey(secret=native_bytes(priv))
-    else:
-        newpriv = secp256k1.PrivateKey(secret=bytes_to_native_str(priv))
+    newpriv = secp256k1.PrivateKey(secret=native_bytes(priv))
     return newpriv.public_key.format(compressed)
 
-def privkey_to_pubkey(priv, usehex=True):
-    '''To avoid changing the interface from the legacy system,
-    allow an *optional* hex argument here (called differently from
-    maker/taker code to how it's called in bip32 code), then
-    pass to the standard hexbin decorator under the hood.
-    '''
-    return privkey_to_pubkey_inner(priv, usehex)
+# b58check wrapper functions around bitcointx.base58 functions:
+# (avoids complexity of key management structure)
+
+def bin_to_b58check(inp, magicbyte=b'\x00'):
+    """ The magic byte (prefix byte) should be passed either
+    as a single byte or an integer. What is returned is a string
+    in base58 encoding, with the prefix and the checksum.
+    """
+    if not isinstance(magicbyte, int):
+        magicbyte = struct.unpack(b'B', magicbyte)[0]
+    assert(0 <= magicbyte <= 0xff)
+    if magicbyte == 0:
+        inp_fmtd = struct.pack(b'B', magicbyte) + inp
+    while magicbyte > 0:
+        inp_fmtd = struct.pack(b'B', magicbyte % 256) + inp
+        magicbyte //= 256
+    checksum = Hash(inp_fmtd)[:4]
+    return base58.encode(inp_fmtd + checksum)
+
+def b58check_to_bin(s):
+    data = base58.decode(s)
+    assert Hash(data[:-4])[:4] == data[-4:]
+    return struct.pack(b"B", data[0]), data[1:-4]
 
-privtopub = privkey_to_pubkey
+def get_version_byte(s):
+    return b58check_to_bin(s)[0]
 
-@hexbin
-def is_valid_pubkey(pubkey, usehex, require_compressed=False):
+def ecdsa_sign(msg, priv, formsg=False):
+    hashed_msg = BitcoinMessage(msg).GetHash()
+    sig = ecdsa_raw_sign(hashed_msg, priv, rawmsg=True, formsg=formsg)
+    return base64.b64encode(sig).decode('ascii')
+
+def ecdsa_verify(msg, sig, pub):
+    hashed_msg = BitcoinMessage(msg).GetHash()
+    sig = base64.b64decode(sig)
+    return ecdsa_raw_verify(hashed_msg, pub, sig, rawmsg=True)
+
+def is_valid_pubkey(pubkey, require_compressed=False):
     """ Returns True if the serialized pubkey is a valid secp256k1
     pubkey serialization or False if not; returns False for an
     uncompressed encoding if require_compressed is True.
@@ -458,8 +124,8 @@ def is_valid_pubkey(pubkey, usehex, require_compressed=False):
         return False
     return True
 
-@hexbin
-def multiply(s, pub, usehex, rawpub=True, return_serialized=True):
+
+def multiply(s, pub, return_serialized=True):
     '''Input binary compressed pubkey P(33 bytes)
     and scalar s(32 bytes), return s*P.
     The return value is a binary compressed public key,
@@ -470,24 +136,19 @@ def multiply(s, pub, usehex, rawpub=True, return_serialized=True):
     '''
     newpub = secp256k1.PublicKey(pub)
     #see note to "tweak_mul" function in podle.py
-    if sys.version_info >= (3,0):
-        res = newpub.multiply(native_bytes(s))
-    else:
-        res = newpub.multiply(bytes_to_native_str(s))
+    res = newpub.multiply(native_bytes(s))
     if not return_serialized:
         return res
     return res.format()
 
-@hexbin
-def add_pubkeys(pubkeys, usehex):
+def add_pubkeys(pubkeys):
     '''Input a list of binary compressed pubkeys
     and return their sum as a binary compressed pubkey.'''
     pubkey_list = [secp256k1.PublicKey(x) for x in pubkeys]
     r = secp256k1.PublicKey.combine_keys(pubkey_list)
     return r.format()
 
-@hexbin
-def add_privkeys(priv1, priv2, usehex):
+def add_privkeys(priv1, priv2):
     '''Add privkey 1 to privkey 2.
     Input keys must be in binary either compressed or not.
     Returned key will have the same compression state.
@@ -504,58 +165,34 @@ def add_privkeys(priv1, priv2, usehex):
         res += b'\x01'
     return res
 
-@hexbin
+
 def ecdsa_raw_sign(msg,
                    priv,
-                   usehex,
-                   rawpriv=True,
                    rawmsg=False,
-                   usenonce=None,
                    formsg=False):
     '''Take the binary message msg and sign it with the private key
     priv.
-    By default priv is just a 32 byte string, if rawpriv is false
-    it is assumed to be hex encoded (note only works if usehex=False).
     If rawmsg is True, no sha256 hash is applied to msg before signing.
     In this case, msg must be a precalculated hash (256 bit).
     If rawmsg is False, the secp256k1 lib will hash the message as part
     of the ECDSA-SHA256 signing algo.
-    If usenonce is not None, its value is passed to the secp256k1 library
-    sign() function as the ndata value, which is then used in conjunction
-    with a custom nonce generating function, such that the nonce used in the ECDSA
-    sign algorithm is exactly that value (ndata there, usenonce here). 32 bytes.
     Return value: the calculated signature.'''
     if rawmsg and len(msg) != 32:
         raise Exception("Invalid hash input to ECDSA raw sign.")
-    if rawpriv:
-        compressed, p = read_privkey(priv)
-        newpriv = secp256k1.PrivateKey(p)
-    else:
-        newpriv = secp256k1.PrivateKey.from_hex(priv)
+
+    compressed, p = read_privkey(priv)
+    newpriv = secp256k1.PrivateKey(p)
     if formsg:
         sig = newpriv.sign_recoverable(msg)
         return sig
-    #Donations, thus custom nonce, currently disabled, hence not covered.
-    elif usenonce: #pragma: no cover
-        raise NotImplementedError
-        #if len(usenonce) != 32:
-        #    raise ValueError("Invalid nonce passed to ecdsa_sign: " + str(
-        #        usenonce))
-        #nf = ffi.addressof(_noncefunc.lib, "nonce_function_rand")
-        #ndata = ffi.new("char [32]", usenonce)
-        #usenonce = (nf, ndata)
-        #sig = newpriv.ecdsa_sign(msg, raw=rawmsg, custom_nonce=usenonce)
     else:
-        #partial fix for secp256k1-transient not including customnonce;
-        #partial because donations will crash on windows in the "if".
         if rawmsg:
             sig = newpriv.sign(msg, hasher=None)
         else:
             sig = newpriv.sign(msg)
     return sig
 
-@hexbin
-def ecdsa_raw_verify(msg, pub, sig, usehex, rawmsg=False):
+def ecdsa_raw_verify(msg, pub, sig, rawmsg=False):
     '''Take the binary message msg and binary signature sig,
     and verify it against the pubkey pub.
     If rawmsg is True, no sha256 hash is applied to msg before verifying.
@@ -575,56 +212,6 @@ def ecdsa_raw_verify(msg, pub, sig, usehex, rawmsg=False):
             retval = newpub.verify(sig, msg, hasher=None)
         else:
             retval = newpub.verify(sig, msg)
-    except:
+    except Exception as e:
         return False
     return retval
-
-def estimate_tx_size(ins, outs, txtype='p2pkh'):
-    '''Estimate transaction size.
-    The txtype field as detailed below is used to distinguish
-    the type, but there is at least one source of meaningful roughness:
-    we assume the output types are the same as the input (to be fair,
-    outputs only contribute a little to the overall total). This combined
-    with a few bytes variation in signature sizes means we will expect,
-    say, 10% inaccuracy here.
-
-    Assuming p2pkh:
-    out: 8+1+3+2+20=34, in: 1+32+4+1+1+~73+1+1+33=147,
-    ver:4,seq:4, +2 (len in,out)
-    total ~= 34*len_out + 147*len_in + 10 (sig sizes vary slightly)
-    Assuming p2sh M of N multisig:
-    "ins" must contain M, N so ins= (numins, M, N) (crude assuming all same)
-    74*M + 34*N + 45 per input, so total ins ~ len_ins * (45+74M+34N)
-    so total ~ 34*len_out + (45+74M+34N)*len_in + 10
-    Assuming p2sh-p2wpkh:
-    witness are roughly 3+~73+33 for each input
-    (txid, vin, 4+20 for witness program encoded as scriptsig, 4 for sequence)
-    non-witness input fields are roughly 32+4+4+20+4=64, so total becomes
-    n_in * 64 + 4(ver) + 4(locktime) + n_out*34
-    Assuming p2wpkh native:
-    witness as previous case
-    non-witness loses the 24 witnessprogram, replaced with 1 zero,
-    in the scriptSig, so becomes:
-    n_in * 41 + 4(ver) + 4(locktime) +2 (len in, out) + n_out*34
-    '''
-    if txtype == 'p2pkh':
-        return 10 + ins * 147 + 34 * outs
-    elif txtype == 'p2sh-p2wpkh':
-        #return the estimate for the witness and non-witness
-        #portions of the transaction, assuming that all the inputs
-        #are of segwit type p2sh-p2wpkh
-        # Note as of Jan19: this misses 2 bytes (trivial) for len in, out
-        # and also overestimates output size by 2 bytes.
-        witness_estimate = ins*109
-        non_witness_estimate = 4 + 4 + outs*34 + ins*64
-        return (witness_estimate, non_witness_estimate)
-    elif txtype == 'p2wpkh':
-        witness_estimate = ins*109
-        non_witness_estimate = 4 + 4 + 2 + outs*31 + ins*41
-        return (witness_estimate, non_witness_estimate)
-    elif txtype == 'p2shMofN':
-        ins, M, N = ins
-        return 10 + (45 + 74*M + 34*N) * ins + 34 * outs
-    else:
-        raise NotImplementedError("Transaction size estimation not" +
-                                  "yet implemented for type: " + txtype)

jmbitcoin/setup.py

@@ -9,6 +9,6 @@ setup(name='joinmarketbitcoin',
       author_email='',
       license='GPL',
       packages=['jmbitcoin'],
-      install_requires=['future', 'coincurve', 'urldecode'],
+      install_requires=['future', 'coincurve', 'python-bitcointx', 'urldecode'],
       python_requires='>=3.5',
       zip_safe=False)

jmbitcoin/test/test_addresses.py

@@ -1,56 +0,0 @@
-import jmbitcoin as btc
-import json
-import pytest
-import os
-testdir = os.path.dirname(os.path.realpath(__file__))
-
-def validate_address(addr, nettype):
-    """A mock of jmclient.validate_address
-    """
-    BTC_P2PK_VBYTE = {"mainnet": b'\x00', "testnet": b'\x6f'}
-    BTC_P2SH_VBYTE = {"mainnet": b'\x05', "testnet": b'\xc4'}    
-    try:
-        ver = btc.get_version_byte(addr)
-    except AssertionError as e:
-        return False, 'Checksum wrong. Typo in address?'
-    except Exception as e:
-        return False, "Invalid bitcoin address"
-    if ver not in [BTC_P2PK_VBYTE[nettype], BTC_P2SH_VBYTE[nettype]]:
-        return False, 'Wrong address version. Testnet/mainnet confused?'
-    if len(btc.b58check_to_bin(addr)) != 20:
-        return False, "Address has correct checksum but wrong length."
-    return True, 'address validated'
-
-@pytest.mark.parametrize(
-    "net",
-    [
-        # 1
-        ("mainnet"),
-        # 2
-        ("testnet")
-    ])
-def test_b58_invalid_addresses(net):
-    #none of these are valid as any kind of key or address
-    with open(os.path.join(testdir,"base58_keys_invalid.json"), "r") as f:
-        json_data = f.read()
-    invalid_key_list = json.loads(json_data)
-    for k in invalid_key_list:
-        bad_key = k[0]
-        res, message = validate_address(bad_key, nettype=net)
-        assert res == False, "Incorrectly validated address: " + bad_key + " with message: " + message
-
-def test_b58_valid_addresses():
-    with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f:
-        json_data = f.read()
-    valid_keys_list = json.loads(json_data)
-    for a in valid_keys_list:
-        addr, pubkey, prop_dict = a
-        if not prop_dict["isPrivkey"]:
-            if prop_dict["isTestnet"]:
-                net = "testnet"
-            else:
-                net = "mainnet"
-            #if using pytest -s ; sanity check to see what's actually being tested
-            res, message = validate_address(addr, net)
-            assert res == True, "Incorrectly failed to validate address: " + addr + " with message: " + message
-

jmbitcoin/test/test_bech32.py

@@ -1,131 +0,0 @@
-#!/usr/bin/python
-
-# Copyright (c) 2017 Pieter Wuille
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-
-
-"""Reference tests for segwit adresses"""
-
-import binascii
-import unittest
-import jmbitcoin as btc
-
-VALID_CHECKSUM = [
-    "A12UEL5L",
-    "an83characterlonghumanreadablepartthatcontainsthenumber1andtheexcludedcharactersbio1tt5tgs",
-    "abcdef1qpzry9x8gf2tvdw0s3jn54khce6mua7lmqqqxw",
-    "11qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqc8247j",
-    "split1checkupstagehandshakeupstreamerranterredcaperred2y9e3w",
-]
-
-INVALID_CHECKSUM = [
-    " 1nwldj5",
-    "\x7F" + "1axkwrx",
-    "an84characterslonghumanreadablepartthatcontainsthenumber1andtheexcludedcharactersbio1569pvx",
-    "pzry9x0s0muk",
-    "1pzry9x0s0muk",
-    "x1b4n0q5v",
-    "li1dgmt3",
-    "de1lg7wt\xff",
-]
-
-VALID_ADDRESS = [
-    ["BC1QW508D6QEJXTDG4Y5R3ZARVARY0C5XW7KV8F3T4", "0014751e76e8199196d454941c45d1b3a323f1433bd6"],
-    ["tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sl5k7",
-     "00201863143c14c5166804bd19203356da136c985678cd4d27a1b8c6329604903262"],
-    ["bc1pw508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7k7grplx",
-     "5128751e76e8199196d454941c45d1b3a323f1433bd6751e76e8199196d454941c45d1b3a323f1433bd6"],
-    ["BC1SW50QA3JX3S", "6002751e"],
-    ["bc1zw508d6qejxtdg4y5r3zarvaryvg6kdaj", "5210751e76e8199196d454941c45d1b3a323"],
-    ["tb1qqqqqp399et2xygdj5xreqhjjvcmzhxw4aywxecjdzew6hylgvsesrxh6hy",
-     "0020000000c4a5cad46221b2a187905e5266362b99d5e91c6ce24d165dab93e86433"],
-]
-
-INVALID_ADDRESS = [
-    "tc1qw508d6qejxtdg4y5r3zarvary0c5xw7kg3g4ty",
-    "bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t5",
-    "BC13W508D6QEJXTDG4Y5R3ZARVARY0C5XW7KN40WF2",
-    "bc1rw5uspcuh",
-    "bc10w508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7kw5rljs90",
-    "BC1QR508D6QEJXTDG4Y5R3ZARVARYV98GJ9P",
-    "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sL5k7",
-    "bc1zw508d6qejxtdg4y5r3zarvaryvqyzf3du",
-    "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3pjxtptv",
-    "bc1gmk9yu",
-
-]
-
-INVALID_ADDRESS_ENC = [
-    ("BC", 0, 20),
-    ("bc", 0, 21),
-    ("bc", 17, 32),
-    ("bc", 1, 1),
-    ("bc", 16, 41),
-]
-
-class TestSegwitAddress(unittest.TestCase):
-    """Unit test class for segwit addressess."""
-
-    def test_valid_checksum(self):
-        """Test checksum creation and validation."""
-        for test in VALID_CHECKSUM:
-            hrp, _ = btc.bech32_decode(test)
-            self.assertIsNotNone(hrp)
-            pos = test.rfind('1')
-            test = test[:pos+1] + chr(ord(test[pos + 1]) ^ 1) + test[pos+2:]
-            hrp, _ = btc.bech32_decode(test)
-            self.assertIsNone(hrp)
-
-    def test_invalid_checksum(self):
-        """Test validation of invalid checksums."""
-        for test in INVALID_CHECKSUM:
-            hrp, _ = btc.bech32_decode(test)
-            self.assertIsNone(hrp)
-
-    def test_valid_address(self):
-        """Test whether valid addresses decode to the correct output."""
-        for (address, hexscript) in VALID_ADDRESS:
-            hrp = "bc"
-            witver, witprog = btc.bech32addr_decode(hrp, address)
-            if witver is None:
-                hrp = "tb"
-                witver, witprog = btc.bech32addr_decode(hrp, address)
-            self.assertIsNotNone(witver)
-            scriptpubkey = btc.segwit_scriptpubkey(witver, witprog)
-            self.assertEqual(scriptpubkey, binascii.unhexlify(hexscript))
-            addr = btc.bech32addr_encode(hrp, witver, witprog)
-            self.assertEqual(address.lower(), addr)
-
-    def test_invalid_address(self):
-        """Test whether invalid addresses fail to decode."""
-        for test in INVALID_ADDRESS:
-            witver, _ = btc.bech32addr_decode("bc", test)
-            self.assertIsNone(witver)
-            witver, _ = btc.bech32addr_decode("tb", test)
-            self.assertIsNone(witver)
-
-    def test_invalid_address_enc(self):
-        """Test whether address encoding fails on invalid input."""
-        for hrp, version, length in INVALID_ADDRESS_ENC:
-            code = btc.bech32addr_encode(hrp, version, [0] * length)
-            self.assertIsNone(code)
-
-if __name__ == "__main__":
-    unittest.main()

jmbitcoin/test/test_bip32.py

@@ -103,9 +103,9 @@ def test_ckd_pubkeys():
 def test_bip32_descend():
     master = btc.bip32_master_key(b'\x07'*32)
     end_key = btc.bip32_descend(master, [2, 3, 10000])
-    assert end_key=="6856ef965940a1a7b1311dc041050ac0013e326c7ff4e2c677a7694b4f0405c901"
+    assert end_key==binascii.unhexlify("6856ef965940a1a7b1311dc041050ac0013e326c7ff4e2c677a7694b4f0405c901")
     end_key = btc.bip32_descend(master, 2, 5, 4, 5)
-    assert end_key=="d2d816b6485103c0d7ff95482788f0e8e73fa11817079e006d47979d8196c4b101"
+    assert end_key==binascii.unhexlify("d2d816b6485103c0d7ff95482788f0e8e73fa11817079e006d47979d8196c4b101")
 
     
 

jmbitcoin/test/test_btc_formatting.py

@@ -1,57 +0,0 @@
-#! /usr/bin/env python
-'''Test bitcoin module data handling'''
-
-import jmbitcoin as btc
-import pytest
-import binascii
-
-#used in p2sh addresses
-def test_hash160():
-    assert '0e3397b4abc7a382b3ea2365883c3c7ca5f07600' == \
-           btc.hash160('The quick brown fox jumps over the lazy dog')
-
-def test_bad_code_string():
-    for i in [1,9,257,-3,"256"]:
-        with pytest.raises(ValueError) as e_info:
-            btc.get_code_string(i)
-
-#Tests of compactsize encoding, see:
-#https://bitcoin.org/en/developer-reference#compactsize-unsigned-integers
-#note that little endian is used.
-@pytest.mark.parametrize(
-    "num, compactsize",
-    [
-        (252, "fc"),
-        (253, "fdfd00"),
-        (254, "fdfe00"),
-        (515, "fd0302"),
-        (65535, "fdffff"),
-        (65536, "fe00000100"),
-        (65537, "fe01000100"),
-        (4294967295, "feffffffff"),
-        (4294967296, "ff0000000001000000"),
-        
-    ])
-def test_compact_size(num, compactsize):
-    assert btc.num_to_var_int(num) == binascii.unhexlify(compactsize)
-
-@pytest.mark.parametrize("frm, to", [
-    (("16001405481b7f1d90c5a167a15b00e8af76eb6984ea59"),
-     ["001405481b7f1d90c5a167a15b00e8af76eb6984ea59"]),
-    (("483045022100ad0dda327945e581a5effd83d75d76a9f07c3128f4dc6d25a54"
-      "e5ad5dd629bd00220487a992959bd540dbc335c655e6485ebfb394129eb48038f"
-      "0a2d319782f7cb690121039319452b6abafb5fcf06096196d0c141b8bd18a3de7"
-      "e9b9352da800d671ccd84"),
-     ["3045022100ad0dda327945e581a5effd83d75d76a9f07c3128f4dc6d25a54e5ad5dd629bd00220487a992959bd540dbc335c655e6485ebfb394129eb48038f0a2d319782f7cb6901",
-      "039319452b6abafb5fcf06096196d0c141b8bd18a3de7e9b9352da800d671ccd84"]),
-    (("51"), [1]),
-    (("00"), [None]),
-    (("510000"), [1, None, None]),
-    (("636505aaaaaaaaaa53"), [99, 101, "aaaaaaaaaa", 3]),
-    (("51" + "4d0101" + "aa"*257), [1, "aa"*257]),
-    (("4e" + "03000100" + "aa"*65539), ["aa"*65539]),
-])
-def test_deserialize_script(frm, to):
-    #print(len(btc.deserialize_script(frm)[0]))
-    assert btc.deserialize_script(frm) == to
-    assert btc.serialize_script(to) == frm

jmbitcoin/test/test_ecc_signing.py

@@ -12,25 +12,23 @@ vectors = None
 
 def test_valid_sigs(setup_ecc):
     for v in vectors['vectors']:
-        msg = v['msg']
-        sig = v['sig']
-        priv = v['privkey']
-        assert sig == btc.ecdsa_raw_sign(msg, priv, True, rawmsg=True)+'01'
-        #check that the signature verifies against the key(pair)
-        pubkey = btc.privtopub(priv)
-        assert btc.ecdsa_raw_verify(msg, pubkey, sig[:-2], True, rawmsg=True)
-        #check that it fails to verify against corrupted signatures
+        msg, sig, priv = (binascii.unhexlify(
+            v[a]) for a in ["msg", "sig", "privkey"])
+        assert sig == btc.ecdsa_raw_sign(msg, priv, rawmsg=True)+ b'\x01'
+        # check that the signature verifies against the key(pair)
+        pubkey = btc.privkey_to_pubkey(priv)
+        assert btc.ecdsa_raw_verify(msg, pubkey, sig[:-1], rawmsg=True)
+        # check that it fails to verify against corrupted signatures
         for i in [0,1,2,4,7,25,55]:
-            #corrupt one byte
-            binsig = binascii.unhexlify(sig)
-            checksig = binascii.hexlify(binsig[:i] + btc.from_string_to_bytes(chr(
-                (ord(binsig[i:i+1])+1) %256)) + binsig[i+1:-1]).decode('ascii')
+            # corrupt one byte
+            checksig = sig[:i] + chr(
+                (ord(sig[i:i+1])+1) %256).encode() + sig[i+1:-1]
             
-            #this kind of corruption will sometimes lead to an assert
-            #failure (if the DER format is corrupted) and sometimes lead
-            #to a signature verification failure.
+            # this kind of corruption will sometimes lead to an assert
+            # failure (if the DER format is corrupted) and sometimes lead
+            # to a signature verification failure.
             try:
-                res = btc.ecdsa_raw_verify(msg, pubkey, checksig, True, rawmsg=True)
+                res = btc.ecdsa_raw_verify(msg, pubkey, checksig, rawmsg=True)
             except:
                 continue
             assert res==False

jmbitcoin/test/test_main.py

@@ -1,61 +0,0 @@
-#! /usr/bin/env python
-'''Testing mostly exceptional cases in secp256k1_main.
-   Some of these may represent code that should be removed, TODO.'''
-
-import jmbitcoin as btc
-import binascii
-import pytest
-import os
-testdir = os.path.dirname(os.path.realpath(__file__))
-
-def test_hex2b58check():
-    assert btc.hex_to_b58check("aa"*32) == "12JAT9y2EcnV6DPUGikLJYjWwk5UmUEFXRiQVmTbfSLbL3njFzp"
-
-def test_bindblsha():
-    assert btc.bin_dbl_sha256("abc") == binascii.unhexlify(
-        "4f8b42c22dd3729b519ba6f68d2da7cc5b2d606d05daed5ad5128cc03e6c6358")
-
-def test_lpad():
-    assert btc.lpad("aaaa", "b", 5) == "baaaa"
-    assert btc.lpad("aaaa", "b", 4) == "aaaa"
-    assert btc.lpad("aaaa", "b", 3) == "aaaa"
-    
-def test_safe_from_hex():
-    assert btc.safe_from_hex('ff0100') == b'\xff\x01\x00'
-
-def test_hash2int():
-    assert btc.hash_to_int("aa"*32) == \
-    77194726158210796949047323339125271902179989777093709359638389338608753093290
-
-@btc.hexbin
-def dummyforwrap(a, b, c, d="foo", e="bar"):
-    newa = a+b"\x01"
-    x, y = b
-    newb = [x+b"\x02", y+b"\x03"]
-    if d == "foo":
-        return newb[1]
-    else:
-        return newb[0]
-
-def test_hexbin():
-    assert dummyforwrap("aa", ["bb", "cc"], True) == "cc03"
-    assert dummyforwrap("aa", ["bb", "cc"], True, d="baz") == "bb02"
-    assert dummyforwrap(b"\xaa", [b"\xbb", b"\xcc"], False) == b"\xcc\x03"
-
-def test_add_privkeys():
-    with pytest.raises(Exception) as e_info:
-        btc.add_privkeys("aa"*32, "bb"*32+"01", True)
-
-def test_ecdsa_raw_sign():
-    msg = "aa"*31
-    with pytest.raises(Exception) as e_info:
-        btc.ecdsa_raw_sign(msg, None, None, rawmsg=True)
-    assert e_info.match("Invalid hash input")
-    #build non-raw priv object as input
-    privraw = "aa"*32
-    msghash = b"\xbb"*32
-    sig = binascii.hexlify(btc.ecdsa_raw_sign(msghash, privraw, False, rawpriv=False, rawmsg=True)).decode('ascii')
-    assert sig == "3045022100b81960b4969b423199dea555f562a66b7f49dea5836a0168361f1a5f8a3c8298022003eea7d7ee4462e3e9d6d59220f950564caeb77f7b1cdb42af3c83b013ff3b2f"
-
-
-    

jmbitcoin/test/test_tx_serialize.py

@@ -1,234 +0,0 @@
-import jmbitcoin as btc
-import pytest
-import json
-import os
-testdir = os.path.dirname(os.path.realpath(__file__))
-
-#TODO: fold these examples into the tx_valid.json file
-@pytest.mark.parametrize(
-    "tx_type, tx_id, tx_hex",
-    [("simple-tx",
-      "f31916a1d398a4ec18d56a311c942bb6db934cee6aa8ac30af0b30aad9efb841",
-      "0100000001c74265f31fc5e24895fdc83f7157cc40045235f3a71ae326a219de9de873" +
-      "0d8b010000006a473044022076055917470b7ec4f4bb008096266cf816ebb089ad983e" +
-      "6a0f63340ba0e6a6cb022059ec938b996a75db10504e46830e13d399f28191b9832bd5" +
-      "f61df097b9e0d47801210291941334a00959af4aa5757abf81d2a7d1aca8adb3431c67" +
-      "e89419271ba71cb4feffffff023cdeeb03000000001976a914a2426748f14eba44b3f6" +
-      "abba3e8bce216ea233f388acf4ebf303000000001976a914bfa366464a464005ba0df8" +
-      "6024a6c3ed859f03ac88ac33280600"),
-     ("3partycoinjoin",
-      "d91278e125673f5b9201456b0c36efac3b2b6700fdd04fc2227352a63f941170",
-      "010000000cedf1756d1d679de268a533c2d009f1d06ba8918908369c5e3b818678603a" +
-      "4bcb030000006b483045022100a775815dcfaf706ee9b6e6015f162d20d227eb44cf88" +
-      "93fb3ac95f910058fbd50220500f1f3de75b3bdf7f6a0949528f836c694a5bd1effedb" +
-      "b02156bcb292a78e5d012102a726879b9d663467ab71726ff5469c8b0a4213a93643f6" +
-      "ed739fdbb1e81ca5b9ffffffff3e0ebec9b42f30657ef187bd263c57849541014700cc" +
-      "5480268dc63ce6997552070000006b4830450221008ec354dcb5d661a60651e988356a" +
-      "215ccaaa688be1e78861b3b89f35e0c1bb7b02203d2117791b74ad2aa594e4dc806d03" +
-      "ec2a1241c4955b013dc365b6eacc4ee6c9012102632c8b13d9b94f2226c28295b52686" +
-      "4bbece7ea283a90ca001e498e2a08dbebcffffffff841e4ed900a6592687d7684ec0dd" +
-      "0893d7a3b86df07bb2626a1f51ae74ee0ba6010000006a473044022017f96effaf9812" +
-      "0745323859e1b52699baf0993675d028882e42310deac6921102200daaab3008e9bce1" +
-      "2a3e136a7e0472a12da31121237ac382a6a64e4ab5ed2e5201210365a1491d9c10f866" +
-      "0a02f47a54e5c0285f186a9f5b4cb75405548cef3427c4eeffffffff7e131188b66001" +
-      "c0f5a4b45bd3503d2ce17f40e43e3803a7f30c20a8214abc95080000006a4730440220" +
-      "6c21a02e4bfcce3f9a5775ce7b9033bebb354b15167dec1a1b3f0c7255633234022071" +
-      "994111c455b1ddd4cd093d0d4f2ecbbf1283a0a1f982962127768e3c4e251301210394" +
-      "1f5e1834e8b1c1503f2d20b3f3e06db2b80de47f7f31b4cb39f75525753822ffffffff" +
-      "955544430628ea06493caa541d59374d7708dbef0e59a6653ac025f341c888c8040000" +
-      "006a473044022022c2afb23a7d401cdfff735ba390fac8dd2cbde2dcce4c3a8f988dfd" +
-      "a08b956c02204ba4ca6dc72993da6bf3d020530f01d7fc931ef28c78c2fa9b70657e50" +
-      "8873c801210240dc515cf540c43575615f18670d3f511f6f47b719650be60183ad9f61" +
-      "f805cdffffffffedf1756d1d679de268a533c2d009f1d06ba8918908369c5e3b818678" +
-      "603a4bcb010000006b483045022100ee3b322aab4c77debc6ebf48967f7677c03cd5fb" +
-      "df775885691f1162cca1f04f0220031a8c4017618562f4a2f4f0ece98bf13063c9e03b" +
-      "720b2eb3ead0ef024bee1f012103ce920bc0f19adf76e0f8b36eb62873d3cba991ee3e" +
-      "6504e0dbf114a0d5865827ffffffffa8b4085018a3e8848531eecf3164ec2b89ed5ed2" +
-      "172e22a19e3343f29faa3c03050000006a47304402202ff247fb0ec1eb9fa24903957d" +
-      "8973eb9a3a8e2dc4d6e6dbbb366c7563b49e680220301a0df3f1dbff1f939f14b3bf78" +
-      "687d8a2da26dafcb310ade85efd9cd37bd050121038cfc01875ccaacd4863515259e52" +
-      "b96188e83ca30358bce4111c4021bfc66343fffffffff5e7b4d41b53a1fce37b349d02" +
-      "33aed80be2b5da0ff45c59014ed5e4ff09b0a5030000006a473044022015035665d467" +
-      "19031a1e09fffb92484365a38f8c51171046bedc0eea1385cb7402200a296a8315f9a6" +
-      "3fe9a7020d4fa3be20a2c0587f98ae0d8d0560985d9cfb0748012103d9537b8ccb7206" +
-      "383edff317fcb1e4188036a18a84a6727104dd9b4d00638037ffffffffedf1756d1d67" +
-      "9de268a533c2d009f1d06ba8918908369c5e3b818678603a4bcb000000006a47304402" +
-      "200742b54bb0d8e62dbb87eaa00823152ff46106824d791bcb538e47b9f52e9df90220" +
-      "705de19cdc96c9dcb9f8dfff070e780c129524a56f369dc3633f69b1472c41d2012102" +
-      "90428148e18a34190de8a0b9a7ab7e9353feac7bd7bb57dde4acb2a03d1dfb94ffffff" +
-      "ff841e4ed900a6592687d7684ec0dd0893d7a3b86df07bb2626a1f51ae74ee0ba60c00" +
-      "00006a4730440220719b2baf0422e98d7462d120c3406e7e3657cceb8fabe84ddece55" +
-      "d34affb4620220168f87c8bcd57f903104b5effc0feffe3bd776dce6c09bfda8a3a36c" +
-      "95788ec701210263c387f7b141cf649a5296096976b6b622cd44a611c0e485724f9432" +
-      "9cf22b79ffffffff3e0ebec9b42f30657ef187bd263c57849541014700cc5480268dc6" +
-      "3ce6997552010000006a47304402206e62849a47494e9294bd7e47d87b9d32a6b69097" +
-      "067184b551350b7b34291d46022017ca461f6bcceb97432e9890700a68a4df4f4c5cfd" +
-      "272926af00a05dfdb3c07b0121024decad000f3aabfdac5ea8f132e457f0056b1f2cab" +
-      "0c946884d2ff0afbdd68b8ffffffff7e131188b66001c0f5a4b45bd3503d2ce17f40e4" +
-      "3e3803a7f30c20a8214abc95090000006b483045022100fa9c274892db835066eeac6c" +
-      "8f25d9507b43c93f4557b16560745e627387d7780220461025024a97816ff0567dcdca" +
-      "4702b5238894ee7028d04af48a7eb11bd3520301210391c94422cd25ee416178827513" +
-      "365ee49e4b2f17cdf0b07aed04353c10e34f6dffffffff06008c8647000000001976a9" +
-      "14e924c7d01f0201df5b1465aed50012c4dffa0ff988ac89ac7800000000001976a914" +
-      "3dc5ae7471acbba55107b651ce20177b3b44721388ac9f258509000000001976a91400" +
-      "656d12c4b48359f8637bcf96f9f02ceb2fad1c88ac4de9ec04000000001976a914f7de" +
-      "454e8b402cbde90f4a325de69236040f144088ac008c8647000000001976a914b7d9f8" +
-      "29f1bd6919728f91f8a10ce0ca87bde1aa88ac008c8647000000001976a914b877256a" +
-      "588a315e30b2c5aec38ff70cea71dd0c88ac00000000"),
-     ("4partycoinjoin",
-      "55eac9d4a4159d4ba355122c6c18f85293c19ae358306a3773ec3a5d053e2f1b",
-      "01000000108048bbbc26c394d45b514835d998ae2679c31cecd7623041372703f469aa" +
-      "ff8e040000006b4830450220293480c5975676ddbb8b66fc6e4bc53668ae4cc59d0da5" +
-      "377e520f3f360ad15e022100fe6aaf3d890570d66728210afb13e221f0405bbbf43262" +
-      "16371459877531a2d80121033392a42f2a8e4e93a5d7b50b4d26eabda42e04e110e77e" +
-      "22b514982a832dde92ffffffff587af588bbe3f911f8234a858cdc62a1c04c12cf09f5" +
-      "95f92a7292fd3f54d17e080000006b483045022100c47a07c3dd537e1e11216dcf3440" +
-      "619ec80a88d25f91f8b9de74b348947ba47902205e5a335e2c75b202b75ff8a50a2560" +
-      "f49542fe4e96598be6fe77b00bc7c33484012103e21d83d80666a020c3e4657f7155ac" +
-      "4572edf9fc79d59b5b69e27f23ff5f8df5ffffffff8048bbbc26c394d45b514835d998" +
-      "ae2679c31cecd7623041372703f469aaff8e0b0000006b48304502202a5403bfed1826" +
-      "c0d5b4143229518d6c0dc6b0881ad04e26039cef4981aa9487022100cecad7bd8fd476" +
-      "2257f83080bf2f56249bb36554137e9af9c1588f8f2ed170a9012103fd52e4f1e64cdd" +
-      "003528cf5f388ad455d001ea691bd02125d694117a1d481bbcffffffff8048bbbc26c3" +
-      "94d45b514835d998ae2679c31cecd7623041372703f469aaff8e110000006b48304502" +
-      "204e024b69b4a78ea9daa63ad1d8dba5e8729fbb61749dacc49be20053db8c02710221" +
-      "00c24555b43e6e838fd7cb797cbf96c278531af29493aaf719bdfe3354941d0b320121" +
-      "02a322a78564a976271153b0e289d631e5c6a217542d540c94262c40a83e48281affff" +
-      "ffff8048bbbc26c394d45b514835d998ae2679c31cecd7623041372703f469aaff8e05" +
-      "0000006b483045022100d3c7d326e233b2238576aecf53d5c3dfb88a3b41d815ff2cfd" +
-      "c3e2c5664aa52102204f0dc1cd110e4317ab435533a4440bbdabee75f360551bb46206" +
-      "79043058288801210310950ae84cfb02e24960c7227e40dedbc8804397a19e277233b1" +
-      "f3ecc797fdc2ffffffff9dd9d7a38ab7f29f17c4f32ae5be1426f80ff58c158fc7c74a" +
-      "afb9d53659cd05050000006b483045022100cc46ea21d4a1622cf7507ab9c96863b5fa" +
-      "3744c8c4b34a74c7bfdb92af41f40702200def9598dfcae25abaac047969f231c565c0" +
-      "6df6f7cb8f3c588e390049b0217801210202d1c96a5b459ca4fd430afc4d1b0e468a2a" +
-      "ba84c48ab5f5ad906ec09f55cfb3ffffffff84dc724e49f71d2a82cc46d781b1f47a21" +
-      "18c4056ab6dad2a83f03c7ed4a08c6040000006b483045022100ee5db129b8264b8a12" +
-      "e85fbf29566ad7edb84704214ced4dbea3e628149f08af02201149ab3c4e0aa6fa48ec" +
-      "4816ecefda6105a6df5838e9a6dbe235d156898d3b40012102da5240065576c7567c3e" +
-      "cc8dc09f3fda64a0c6b89c3974a7506bdeb7fb440704ffffffff68e787d07a48d3ba00" +
-      "d8160311ff0533684dd66c7ddba8f7d74af3862d98bd8a020000006b483045022100a8" +
-      "0edec0b7a6946fbcb877f95c48e22e26710f3f7a7ec9e5dd0e7622832fca3002204a8f" +
-      "ecc644f674cfa8c5ba0682318f5b6cd8cc5f0c15b70e5e423e3e0888e6550121032708" +
-      "198b4c046db76f9e0ff7350d05243950738b39c431f55d85d5ec886a1389ffffffff21" +
-      "57aecaef0b44114b1035d6630a436ee3a6ecbeebfca48b5ce8fa6dbc5e100700000000" +
-      "6b483045022100df2085a721c026328ea6dec9307c45d5991d51f696c15896ed14d40c" +
-      "dc69a3d302205a598476fa84a37c5eb70bab35d4a414fce63bc8bedec7883a724ea1f7" +
-      "d4b98401210271f768f5b16806df11ee81619c1371864707de25b27df94bd4516b6680" +
-      "2f0709ffffffff7cabc836092178f7325571627777476d1796e446ac8039dd706754b6" +
-      "824ee723040000006a473044022036eedb5259b74f8a99cba9f4866fafce94b5b7ebf7" +
-      "d248e973bdba27ac1161b202203d58951892c1680fafc77dd6ee1a9c27754018b96030" +
-      "a51c8891c1f5273c077f01210271f768f5b16806df11ee81619c1371864707de25b27d" +
-      "f94bd4516b66802f0709ffffffff84dc724e49f71d2a82cc46d781b1f47a2118c4056a" +
-      "b6dad2a83f03c7ed4a08c6000000006a473044022014a918294f28157288ab1e66e22b" +
-      "fe5e5cb92dde076e133b5d1d807bfd252048022020f8ef9d0c094b476b31f177fb083c" +
-      "b6e65f5167a086fd25bebe5012e46ae8b601210271f768f5b16806df11ee81619c1371" +
-      "864707de25b27df94bd4516b66802f0709ffffffff4c8670675a981c70935cbd0a0de4" +
-      "98c9b28c41784d590f1b9e59930e64dd53f5020000006a473044022067b08694963116" +
-      "f7d53522c2327b46150084b7b9cf5b279daa1461492dffd7010220453b1fd98cdc26fe" +
-      "d9385ec578da1dd11d1ca7367ebfd5586f4ddb913feabb6c01210397db984e478086ca" +
-      "f0bf90181c3c597905a52416e152bff007dffd8edf06e2deffffffff8048bbbc26c394" +
-      "d45b514835d998ae2679c31cecd7623041372703f469aaff8e070000006c4930460221" +
-      "0094e5123e70426ff6732bebbcaaac2d871b873880d77c71a97507a074fe1046000221" +
-      "00c67b7d7663817507dd03f00fd2b690d33daa645a13b6966ec1c76e5a8df0b0a40121" +
-      "035a9e5cdd37824ad390d73abf2c7bc902de6dd7e4ab92fcf4e84506747e8c6cdaffff" +
-      "ffff8048bbbc26c394d45b514835d998ae2679c31cecd7623041372703f469aaff8e01" +
-      "0000006b483045022100cfab7cb7b645e201515b756439ba943fecb77e68c76c482de5" +
-      "225bf7a80ca94a022061600e67eaf33b59c5863de617bdb79f63e69b310710cb2b7f4f" +
-      "63d9d64fb3bd012102dabd952ef6ebfe396dfd66436089f5df731dd8300ecb6d88eefe" +
-      "458bc73a4c8fffffffff11565b2030be339919eaa5d1c1b47f7d265485c2e230fe44d9" +
-      "5ce41f3f6d5bdc030000006b483045022021d350b215077c4b1aee83c747a0cabbe148" +
-      "2fc4ccc97657f07d481df3f89d45022100b16d629e35e857506af6e8dd74373f370f4b" +
-      "965f332a488878dc983f554378c40121022cde2859fb62f3b671a887fcce054626cdb2" +
-      "8ed4d20d0dfcf8e0b090e2d7fe4effffffff8048bbbc26c394d45b514835d998ae2679" +
-      "c31cecd7623041372703f469aaff8e0c0000006a47304402203adf6153783989ca7ec1" +
-      "fa12785e5abae5c6bd8a4e26bb0c2ae0cbc4635af981022056faa34a713bede42d0efe" +
-      "af84c35506f47d487bed59fba2de53167fdad64c570121025d3368bbb24f5980fc6081" +
-      "d2a1e0e04675d11751a53d13d896cff7b5940ac205ffffffff1cb00400000000000019" +
-      "76a9146418ea7a9cbcfbd9dfb4f552c00b834072ac833a88ac40420f00000000001976" +
-      "a91492f8428eae5228083b588b330f9dfc1cccd300fc88ac40420f00000000001976a9" +
-      "14871e08ec7dd94ef87fd7722ee703007badb7020d88ace8030000000000001976a914" +
-      "2ced0ea9992d7e5492b9053437702c83ea0750c088ac40420f00000000001976a91449" +
-      "464457f02609ba1d54153eaba614ab84d593cb88ac877fc30f000000001976a9141c20" +
-      "8af75037715bdca6c42f29e104af62204e4788ac540b0000000000001976a914cfccd5" +
-      "185a3c4ced7514fa584eee6ebd895c984888ac73037000000000001976a9149f0ddeb2" +
-      "473d013452bce71973d7515c0b4cd5cf88ac40420f00000000001976a9142cb4e342c4" +
-      "398a08397a8cedfd270e432e977da488ac09c2e005000000001976a914ca9ec96314a5" +
-      "f4842e3fe33a0cf6d047a4398d0a88ac40420f00000000001976a914cd0124b0969f16" +
-      "4132301032a0f99de5c8d8471e88ace9320400000000001976a9145c5f87677c7b50bc" +
-      "4fe88bdec9f2221a0ca3c35088ac40420f00000000001976a914fa40beb4e557e14d5e" +
-      "461c7aa66d674ea6b7ce5988ac40420f00000000001976a914564bdac0f31da4f35ce5" +
-      "3c9aaef7e796ed7acdcd88ace33c2b00000000001976a9142ef77132983b843b07396b" +
-      "731d213e3a60b4c6e488ac40420f00000000001976a9144aa2c6a81b4ad8c9414b1039" +
-      "d5b2c51d3416233888ac40420f00000000001976a914713667aa801d3d7cbf9c3d1667" +
-      "626131a9937a6288ac40420f00000000001976a9142fa24ea7272cbb3ee06beeb274b3" +
-      "486c3d83f60188ac40420f00000000001976a9147c2e0ee7484234b0a20fab40855dff" +
-      "93c3b195da88ac40420f00000000001976a914123d15de9b64aea49a9cf81d781fdfad" +
-      "00287acf88ac40420f00000000001976a91463f456082de5dbd63c7ab9cd511813d836" +
-      "a2a7cd88ac306a2e00000000001976a9144103b28ac18a4ad0ce075fe323ee3d38a95c" +
-      "4d2188acc0541d00000000001976a91484d1b7dadb4638895fc5eb79145dca39daea86" +
-      "ca88acb80b0000000000001976a9146baff4a7a65316aed38c032c6eded2e48111fb62" +
-      "88ac34080000000000001976a914a5e54586ea1626fc90359f55fa4a291c99cffae488" +
-      "accdc30100000000001976a914d547e4e5b7eb382f9309622bdf0271836083936988ac" +
-      "40420f00000000001976a914a0b17374b3085de303202b29439957ccd71ba13088acde" +
-      "af2600000000001976a914af4d27ced320ae73068d0e2bb2c3abfe6fbe2c6988ac0000" +
-      "0000"),
-     ("p2sh-in-to-p2sh-out",
-      "355a6090bfdb77d1d7fd7c67ba6a711b9f493344e372928fe79ca0206b34796b",
-      "0100000001b5bbd7adb1f9f6f599d8e547f7f789ba9b63da9f036e8b739477bb8295fd" +
-      "4a2600000000fdfd0000473044022062a3ca46f976d4719d94fa67c9360c65b0946e3a" +
-      "7877f7f2c4cc37b8c77dede302203af47e6c6ccd804726295eb99f90dc198c86414d85" +
-      "7e21bc617f26903d380b0901483045022100af39dc96cf32aed7165a7644dcdfa939cb" +
-      "7e6af8236126644be937ff8a10b3a10220204b03b1a6b8acc7748e3c76ee968f9e9997" +
-      "a81c6e944910d2dc626133aa82bf014c695221021de4261f30b149c3f14f93b1a06a5e" +
-      "96aa24780c789343f80c341043d46d700a21036daab402e66c56470eda26d981f80a1b" +
-      "8f224f21fdbe64329b0481b1b37d8a0b2103c72347824e9099dbdfad5a2dac56e5d056" +
-      "8c9c100caa7490f62f2731c23af27353aeffffffff028a7faf00000000001976a91472" +
-      "2a53477336bd4960bed86be7fc47b8ece07ba688acbda438010000000017a9148a6425" +
-      "ccc0cb64097c29579a40e733819d4e07c58700000000"),
-     ("op-return-output",
-      "5be05925b8ef04f326b19e2c57f9ce1f3e2024aa992dd0d744f7942e92a200a5",
-      "0100000001067c0bd822cdcfabf0978bdede3a7b395a5c175219fd704b1ee47f137858" +
-      "6362000000006b483045022100dffb89e46c734b6429b6322bce9b6343c98d0d9eed12" +
-      "059ac2b9dfcffaa5577702204350bc86aea9a7e3b2868a643d27c5a36bd3105c8a2a2c" +
-      "5daf6e8441c83c870f012102a1260627f8845765759454a2ee47603312bbd7fdad0fa0" +
-      "0f3ad0cea0c5602d2cffffffff03e9690700000000001976a91497d34e7a0c8082f180" +
-      "546040030afb925c0f2dd888ac0000000000000000166a146f6d6e6900000000000000" +
-      "0300000000000186a0aa0a0000000000001976a91488d924f51033b74a895863a5fb57" +
-      "fd545529df7d88ac00000000")])
-def test_serialization_roundtrip(tx_type, tx_id, tx_hex):
-    assert tx_hex == btc.serialize(btc.deserialize(tx_hex))
-
-@pytest.mark.parametrize(
-    "ins, outs, txtype, valid",
-    [
-        (4, 3, "p2pkh", True),
-        (4, 3, "p2sh", False),
-    ])
-def test_estimate_tx_size(ins, outs, txtype, valid):
-    #TODO: this function should throw on invalid number of ins or outs
-    if valid:
-        assert btc.estimate_tx_size(ins, outs, txtype)== 10 + 147*ins + 34*outs
-    else:
-        with pytest.raises(NotImplementedError) as e_info:
-            btc.estimate_tx_size(ins, outs, txtype)
-
-        
-def test_serialization_roundtrip2():
-    #Data extracted from:
-    #https://github.com/bitcoin/bitcoin/blob/master/src/test/data/tx_valid.json
-    #These are a variety of rather strange edge case transactions, which are
-    #still valid.
-    #Note that of course this is only a serialization, not validity test, so
-    #only currently of very limited significance
-    with open(os.path.join(testdir,"tx_valid.json"), "r") as f:
-        json_data = f.read()
-    valid_txs = json.loads(json_data)
-    for j in valid_txs:
-        #ignore comment entries
-        if len(j) < 2:
-            continue
-        print(j)
-        deserialized = btc.deserialize(str(j[0]))
-        print(deserialized)
-        assert j[0] == btc.serialize(deserialized)

jmbitcoin/test/test_tx_signing.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+
+import sys
+import pytest
+
+import binascii
+import hashlib
+import jmbitcoin as btc
+
+@pytest.mark.parametrize(
+    "addrtype",
+    [("p2wpkh"),
+     ("p2sh-p2wpkh"),
+     ("p2pkh"),
+    ])
+def test_sign_standard_txs(addrtype):
+    # liberally copied from python-bitcoinlib tests,
+    # in particular see:
+    # https://github.com/petertodd/python-bitcoinlib/pull/227
+
+    # Create the (in)famous correct brainwallet secret key.
+    priv = hashlib.sha256(b'correct horse battery staple').digest() + b"\x01"
+    pub = btc.privkey_to_pubkey(priv)
+    
+    # Create an address from that private key.
+    # (note that the input utxo is fake so we are really only creating
+    # a destination here).
+    scriptPubKey = btc.CScript([btc.OP_0, btc.Hash160(pub)])
+    address = btc.P2WPKHBitcoinAddress.from_scriptPubKey(scriptPubKey)
+    
+    # Create a dummy outpoint; use same 32 bytes for convenience
+    txid = priv[:32]
+    vout = 2
+    amount = btc.coins_to_satoshi(float('0.12345'))
+    
+    # Calculate an amount for the upcoming new UTXO. Set a high fee to bypass
+    # bitcoind minfee setting.
+    amount_less_fee = int(amount - btc.coins_to_satoshi(0.01))
+    
+    # Create a destination to send the coins.
+    destination_address = address
+    target_scriptPubKey = scriptPubKey
+    
+    # Create the unsigned transaction.
+    txin = btc.CTxIn(btc.COutPoint(txid[::-1], vout))
+    txout = btc.CTxOut(amount_less_fee, target_scriptPubKey)
+    tx = btc.CMutableTransaction([txin], [txout])
+    
+    # Calculate the signature hash for the transaction. This is then signed by the
+    # private key that controls the UTXO being spent here at this txin_index.
+    if addrtype == "p2wpkh":
+        sig, msg = btc.sign(tx, 0, priv, amount=amount, native=True)
+    elif addrtype == "p2sh-p2wpkh":
+        sig, msg = btc.sign(tx, 0, priv, amount=amount, native=False)
+    elif addrtype == "p2pkh":
+        sig, msg = btc.sign(tx, 0, priv)
+    else:
+        assert False
+    if not sig:
+        print(msg)
+        raise
+    print("created signature: ", binascii.hexlify(sig))
+    print("serialized transaction: {}".format(btc.b2x(tx.serialize())))
+
+def test_mk_shuffled_tx():
+    # prepare two addresses for the outputs
+    pub = btc.privkey_to_pubkey(btc.Hash(b"priv") + b"\x01")
+    scriptPubKey = btc.CScript([btc.OP_0, btc.Hash160(pub)])
+    addr1 = btc.P2WPKHBitcoinAddress.from_scriptPubKey(scriptPubKey)
+    scriptPubKey_p2sh = scriptPubKey.to_p2sh_scriptPubKey()
+    addr2 = btc.CCoinAddress.from_scriptPubKey(scriptPubKey_p2sh)
+
+    ins = [(btc.Hash(b"blah"), 7), (btc.Hash(b"foo"), 15)]
+    # note the casts str() ; most calls to mktx will have addresses fed
+    # as strings, so this is enforced for simplicity.
+    outs = [{"address": str(addr1), "value": btc.coins_to_satoshi(float("0.1"))},
+            {"address": str(addr2), "value": btc.coins_to_satoshi(float("45981.23331234"))}]
+    tx = btc.make_shuffled_tx(ins, outs, version=2, locktime=500000)
+
+def test_bip143_tv():
+    # p2sh-p2wpkh case:
+    rawtx_hex = "0100000001db6b1b20aa0fd7b23880be2ecbd4a98130974cf4748fb66092ac4d3ceb1a54770100000000feffffff02b8b4eb0b000000001976a914a457b684d7f0d539a46a45bbc043f35b59d0d96388ac0008af2f000000001976a914fd270b1ee6abcaea97fea7ad0402e8bd8ad6d77c88ac92040000"
+    inp_spk_hex = "a9144733f37cf4db86fbc2efed2500b4f4e49f31202387"
+    value = 10
+    redeemScript = "001479091972186c449eb1ded22b78e40d009bdf0089"
+    privkey_hex = "eb696a065ef48a2192da5b28b694f87544b30fae8327c4510137a922f32c6dcf01"
+    pubkey_hex = "03ad1d8e89212f0b92c74d23bb710c00662ad1470198ac48c43f7d6f93a2a26873"
+    tx = btc.CMutableTransaction.deserialize(btc.x(rawtx_hex))
+    btc.sign(tx, 0, btc.x(privkey_hex), amount=btc.coins_to_satoshi(10), native=False)
+    expectedsignedtx = "01000000000101db6b1b20aa0fd7b23880be2ecbd4a98130974cf4748fb66092ac4d3ceb1a5477010000001716001479091972186c449eb1ded22b78e40d009bdf0089feffffff02b8b4eb0b000000001976a914a457b684d7f0d539a46a45bbc043f35b59d0d96388ac0008af2f000000001976a914fd270b1ee6abcaea97fea7ad0402e8bd8ad6d77c88ac02473044022047ac8e878352d3ebbde1c94ce3a10d057c24175747116f8288e5d794d12d482f0220217f36a485cae903c713331d877c1f64677e3622ad4010726870540656fe9dcb012103ad1d8e89212f0b92c74d23bb710c00662ad1470198ac48c43f7d6f93a2a2687392040000"
+    assert btc.b2x(tx.serialize()) == expectedsignedtx

jmclient/jmclient/__init__.py

@@ -18,7 +18,8 @@ from .wallet import (Mnemonic, estimate_tx_fee, WalletError, BaseWallet, ImportW
                      UTXOManager, WALLET_IMPLEMENTATIONS, compute_tx_locktime)
 from .storage import (Argon2Hash, Storage, StorageError, RetryableStorageError,
                       StoragePasswordError, VolatileStorage)
-from .cryptoengine import BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, EngineError
+from .cryptoengine import (BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, EngineError,
+                           TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH)
 from .configure import (load_test_config,
     load_program_config, get_p2pk_vbyte, jm_single, get_network, update_persist_config,
     validate_address, is_burn_destination, get_irc_mchannels,

jmclient/jmclient/blockchaininterface.py

@@ -6,7 +6,7 @@ import time
 from decimal import Decimal
 import binascii
 from twisted.internet import reactor, task
-
+from jmbase import bintohex, hextobin
 import jmbitcoin as btc
 
 from jmclient.jsonrpc import JsonRpcConnectionError, JsonRpcError
@@ -297,9 +297,8 @@ class BitcoinCoreInterface(BlockchainInterface):
         if not "hex" in rpcretval:
             log.info("Malformed gettransaction output")
             return None
-        #str cast for unicode
-        hexval = str(rpcretval["hex"])
-        return btc.deserialize(hexval)
+        return btc.CMutableTransaction.deserialize(
+            hextobin(rpcretval["hex"]))
 
     def list_transactions(self, num, skip=0):
         """ Return a list of the last `num` transactions seen
@@ -309,20 +308,22 @@ class BitcoinCoreInterface(BlockchainInterface):
         return self.rpc("listtransactions", ["*", num, skip, True])
 
     def get_transaction(self, txid):
-        """ Returns a serialized transaction for txid txid,
+        """ Argument txid is passed in binary.
+        Returns a serialized transaction for txid txid,
         in hex as returned by Bitcoin Core rpc, or None
         if no transaction can be retrieved. Works also for
         watch-only wallets.
         """
+        htxid = bintohex(txid)
         #changed syntax in 0.14.0; allow both syntaxes
         try:
-            res = self.rpc("gettransaction", [txid, True])
-        except:
+            res = self.rpc("gettransaction", [htxid, True])
+        except Exception as e:
             try:
-                res = self.rpc("gettransaction", [txid, 1])
+                res = self.rpc("gettransaction", [htxid, 1])
             except JsonRpcError as e:
                 #This should never happen (gettransaction is a wallet rpc).
-                log.warn("Failed gettransaction call; JsonRpcError")
+                log.warn("Failed gettransaction call; JsonRpcError: " + repr(e))
                 return None
             except Exception as e:
                 log.warn("Failed gettransaction call; unexpected error:")
@@ -333,7 +334,11 @@ class BitcoinCoreInterface(BlockchainInterface):
             return None
         return res
 
-    def pushtx(self, txhex):
+    def pushtx(self, txbin):
+        """ Given a binary serialized valid bitcoin transaction,
+        broadcasts it to the network.
+        """
+        txhex = bintohex(txbin)
         try:
             txid = self.rpc('sendrawtransaction', [txhex])
         except JsonRpcConnectionError as e:
@@ -345,9 +350,9 @@ class BitcoinCoreInterface(BlockchainInterface):
         return True
 
     def query_utxo_set(self, txout, includeconf=False, includeunconf=False):
-        """If txout is either (a) a single string in hex encoded txid:n form,
+        """If txout is either (a) a single utxo in (txidbin, n) form,
         or a list of the same, returns, as a list for each txout item,
-        the result of gettxout from the bitcoind rpc for those utxs;
+        the result of gettxout from the bitcoind rpc for those utxos;
         if any utxo is invalid, None is returned.
         includeconf: if this is True, the current number of confirmations
         of the prescribed utxo is included in the returned result dict.
@@ -360,16 +365,18 @@ class BitcoinCoreInterface(BlockchainInterface):
             txout = [txout]
         result = []
         for txo in txout:
-            if len(txo) < 66:
+            txo_hex = bintohex(txo[0])
+            if len(txo_hex) != 64:
+                log.warn("Invalid utxo format, ignoring: {}".format(txo))
                 result.append(None)
                 continue
             try:
-                txo_idx = int(txo[65:])
+                txo_idx = int(txo[1])
             except ValueError:
                 log.warn("Invalid utxo format, ignoring: {}".format(txo))
                 result.append(None)
                 continue
-            ret = self.rpc('gettxout', [txo[:64], txo_idx, includeunconf])
+            ret = self.rpc('gettxout', [txo_hex, txo_idx, includeunconf])
             if ret is None:
                 result.append(None)
             else:
@@ -380,7 +387,7 @@ class BitcoinCoreInterface(BlockchainInterface):
                 result_dict = {'value': int(Decimal(str(ret['value'])) *
                                             Decimal('1e8')),
                                'address': address,
-                               'script': ret['scriptPubKey']['hex']}
+                               'script': hextobin(ret['scriptPubKey']['hex'])}
                 if includeconf:
                     result_dict['confirms'] = int(ret['confirmations'])
                 result.append(result_dict)

jmclient/jmclient/client_protocol.py

@@ -9,16 +9,17 @@ try:
 except ImportError:
     pass
 from jmbase import commands
-
+import binascii
 import json
 import hashlib
 import os
 import sys
-from jmbase import get_log
+from jmbase import (get_log, EXIT_FAILURE, hextobin, bintohex,
+                    utxo_to_utxostr, dictchanger)
 from jmclient import (jm_single, get_irc_mchannels,
                       RegtestBitcoinCoreInterface)
 import jmbitcoin as btc
-from jmbase.support import EXIT_FAILURE
+
 
 jlog = get_log()
 
@@ -30,7 +31,8 @@ class JMClientProtocol(amp.AMP):
             self.client = client
             self.factory = factory
             if not nick_priv:
-                self.nick_priv = hashlib.sha256(os.urandom(16)).hexdigest() + '01'
+                self.nick_priv = hashlib.sha256(
+                    os.urandom(16)).digest() + b"\x01"
             else:
                 self.nick_priv = nick_priv
 
@@ -59,10 +61,18 @@ class JMClientProtocol(amp.AMP):
         self.clientStart()
 
     def set_nick(self):
-        self.nick_pubkey = btc.privtopub(self.nick_priv)
-        self.nick_pkh_raw = btc.bin_sha256(self.nick_pubkey)[
-                    :self.nick_hashlen]
-        self.nick_pkh = btc.b58encode(self.nick_pkh_raw)
+        """ Algorithm: take pubkey and hex-serialized it;
+        then SHA2(hexpub) but truncate output to nick_hashlen.
+        Then encode to a base58 string (no check).
+        Then prepend J and version char (e.g. '5').
+        Finally append padding to nick_maxencoded (+2).
+        """
+        self.nick_pubkey = btc.privkey_to_pubkey(self.nick_priv)
+        # note we use binascii hexlify directly here because input
+        # to hashing must be encoded.
+        self.nick_pkh_raw = hashlib.sha256(binascii.hexlify(
+            self.nick_pubkey)).digest()[:self.nick_hashlen]
+        self.nick_pkh = btc.base58.encode(self.nick_pkh_raw)
         #right pad to maximum possible; b58 is not fixed length.
         #Use 'O' as one of the 4 not included chars in base58.
         self.nick_pkh += 'O' * (self.nick_maxencoded - len(self.nick_pkh))
@@ -93,7 +103,7 @@ class JMClientProtocol(amp.AMP):
     @commands.JMRequestMsgSig.responder
     def on_JM_REQUEST_MSGSIG(self, nick, cmd, msg, msg_to_be_signed, hostid):
         sig = btc.ecdsa_sign(str(msg_to_be_signed), self.nick_priv)
-        msg_to_return = str(msg) + " " + self.nick_pubkey + " " + sig
+        msg_to_return = str(msg) + " " + bintohex(self.nick_pubkey) + " " + sig
         d = self.callRemote(commands.JMMsgSignature,
                             nick=nick,
                             cmd=cmd,
@@ -105,20 +115,21 @@ class JMClientProtocol(amp.AMP):
     @commands.JMRequestMsgSigVerify.responder
     def on_JM_REQUEST_MSGSIG_VERIFY(self, msg, fullmsg, sig, pubkey, nick,
                                     hashlen, max_encoded, hostid):
+        pubkey_bin = hextobin(pubkey)
         verif_result = True
-        if not btc.ecdsa_verify(str(msg), sig, pubkey):
+        if not btc.ecdsa_verify(str(msg), sig, pubkey_bin):
             # workaround for hostid, which sometimes is lowercase-only for some IRC connections
-            if not btc.ecdsa_verify(str(msg[:-len(hostid)] + hostid.lower()), sig, pubkey):
+            if not btc.ecdsa_verify(str(msg[:-len(hostid)] + hostid.lower()), sig, pubkey_bin):
                 jlog.debug("nick signature verification failed, ignoring: " + str(nick))
                 verif_result = False
         #check that nick matches hash of pubkey
-        nick_pkh_raw = btc.bin_sha256(pubkey)[:hashlen]
+        nick_pkh_raw = hashlib.sha256(pubkey.encode("ascii")).digest()[:hashlen]
         nick_stripped = nick[2:2 + max_encoded]
         #strip right padding
         nick_unpadded = ''.join([x for x in nick_stripped if x != 'O'])
-        if not nick_unpadded == btc.b58encode(nick_pkh_raw):
+        if not nick_unpadded == btc.base58.encode(nick_pkh_raw):
             jlog.debug("Nick hash check failed, expected: " + str(nick_unpadded)
-                       + ", got: " + str(btc.b58encode(nick_pkh_raw)))
+                       + ", got: " + str(btc.base58.encode(nick_pkh_raw)))
             verif_result = False
         d = self.callRemote(commands.JMMsgSignatureVerify,
                             verif_result=verif_result,
@@ -234,10 +245,18 @@ class JMMakerClientProtocol(JMClientProtocol):
             jlog.info("Maker refuses to continue on receiving auth.")
         else:
             utxos, auth_pub, cj_addr, change_addr, btc_sig = retval[1:]
+            # json does not allow non-string keys:
+            utxos_strkeyed = {}
+            for k in utxos:
+                success, u = utxo_to_utxostr(k)
+                assert success
+                utxos_strkeyed[u] = {"value": utxos[k]["value"],
+                                     "address": utxos[k]["address"]}
+            auth_pub_hex = bintohex(auth_pub)
             d = self.callRemote(commands.JMIOAuth,
                                 nick=nick,
-                                utxolist=json.dumps(utxos),
-                                pubkey=auth_pub,
+                                utxolist=json.dumps(utxos_strkeyed),
+                                pubkey=auth_pub_hex,
                                 cjaddr=cj_addr,
                                 changeaddr=change_addr,
                                 pubkeysig=btc_sig)
@@ -258,14 +277,14 @@ class JMMakerClientProtocol(JMClientProtocol):
         else:
             sigs = retval[1]
             self.finalized_offers[nick] = offer
-            tx = btc.deserialize(txhex)
+            tx = btc.CMutableTransaction.deserialize(hextobin(txhex))
             self.finalized_offers[nick]["txd"] = tx
-            txid = btc.txhash(btc.serialize(tx))
+            txid = tx.GetTxid()[::-1]
             # we index the callback by the out-set of the transaction,
             # because the txid is not known until all scriptSigs collected
             # (hence this is required for Makers, but not Takers).
             # For more info see WalletService.transaction_monitor():
-            txinfo = tuple((x["script"], x["value"]) for x in tx["outs"])
+            txinfo = tuple((x.scriptPubKey, x.nValue) for x in tx.vout)
             self.client.wallet_service.register_callbacks([self.unconfirm_callback],
                                               txinfo, "unconfirmed")
             self.client.wallet_service.register_callbacks([self.confirm_callback],
@@ -285,8 +304,8 @@ class JMMakerClientProtocol(JMClientProtocol):
 
     def tx_match(self, txd):
         for k,v in iteritems(self.finalized_offers):
-            #Tx considered defined by its output set
-            if v["txd"]["outs"] == txd["outs"]:
+            # Tx considered defined by its output set
+            if v["txd"].vout == txd.vout:
                 offerinfo = v
                 break
         else:
@@ -302,7 +321,7 @@ class JMMakerClientProtocol(JMClientProtocol):
                                                                txid)
         self.client.modify_orders(to_cancel, to_announce)
 
-        txinfo = tuple((x["script"], x["value"]) for x in txd["outs"])
+        txinfo = tuple((x.scriptPubKey, x.nValue) for x in txd.vout)
         confirm_timeout_sec = float(jm_single().config.get(
             "TIMEOUT", "confirm_timeout_hours")) * 3600
         task.deferLater(reactor, confirm_timeout_sec,

jmclient/jmclient/commitment_utils.py

@@ -2,8 +2,10 @@
 import sys
 import jmbitcoin as btc
 from jmbase import jmprint
-from jmclient import jm_single, get_p2pk_vbyte, get_p2sh_vbyte
-from jmbase.support import EXIT_FAILURE
+from jmclient import (jm_single, get_p2pk_vbyte, get_p2sh_vbyte,
+                      BTCEngine, TYPE_P2PKH, TYPE_P2SH_P2WPKH,
+                      BTC_P2PKH, BTC_P2SH_P2WPKH)
+from jmbase.support import EXIT_FAILURE, utxostr_to_utxo
 
 
 def quit(parser, errmsg): #pragma: no cover
@@ -18,17 +20,16 @@ def get_utxo_info(upriv):
         u, priv = upriv.split(',')
         u = u.strip()
         priv = priv.strip()
-        txid, n = u.split(':')
-        assert len(txid)==64
-        assert len(n) in range(1, 4)
-        n = int(n)
-        assert n in range(256)
+        success, utxo = utxostr_to_utxo(u)
+        assert success, utxo
     except:
         #not sending data to stdout in case privkey info
         jmprint("Failed to parse utxo information for utxo", "error")
         raise
     try:
-        hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte())
+        # see note below for why keytype is ignored, and note that
+        # this calls read_privkey to validate.
+        raw, _ = BTCEngine.wif_to_privkey(priv)
     except:
         jmprint("failed to parse privkey, make sure it's WIF compressed format.", "error")
         raise
@@ -40,16 +41,20 @@ def validate_utxo_data(utxo_datas, retrieve=False, segwit=False):
     then use the blockchain instance to look up
     the utxo and check that its address field matches.
     If retrieve is True, return the set of utxos and their values.
+    If segwit is true, assumes a p2sh wrapped p2wpkh, i.e.
+    native segwit is NOT currently supported here. If segwit
+    is false, p2pkh is assumed.
     """
     results = []
     for u, priv in utxo_datas:
         jmprint('validating this utxo: ' + str(u), "info")
-        hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte())
-        if segwit:
-            addr = btc.pubkey_to_p2sh_p2wpkh_address(
-                btc.privkey_to_pubkey(hexpriv), get_p2sh_vbyte())
-        else:
-            addr = btc.privkey_to_address(hexpriv, magicbyte=get_p2pk_vbyte())
+        # as noted in `ImportWalletMixin` code comments, there is not
+        # yet a functional auto-detection of key type from WIF, so the
+        # second argument is ignored; we assume p2sh-p2wpkh if segwit,
+        # else we assume p2pkh.
+        engine = BTC_P2SH_P2WPKH if segwit else BTC_P2PKH
+        rawpriv, _ = BTCEngine.wif_to_privkey(priv)
+        addr = engine.privkey_to_address(rawpriv)
         jmprint('claimed address: ' + addr, "info")
         res = jm_single().bc_interface.query_utxo_set([u])
         if len(res) != 1 or None in res:

jmclient/jmclient/configure.py

@@ -377,33 +377,18 @@ def get_p2pk_vbyte():
 
 def validate_address(addr):
     try:
-        assert len(addr) > 2
-        if addr[:2].lower() in ['bc', 'tb']:
-            # Regtest special case
-            if addr[:4] == 'bcrt':
-                if btc.bech32addr_decode('bcrt', addr)[1]:
-                    return True, 'address validated'
-                return False, 'Invalid bech32 regtest address'
-            #Else, enforce testnet/mainnet per config
-            if get_network() == "testnet":
-                hrpreq = 'tb'
-            else:
-                hrpreq = 'bc'
-            if btc.bech32addr_decode(hrpreq, addr)[1]:
-                return True, 'address validated'
-            return False, 'Invalid bech32 address'
-        #Not bech32; assume b58 from here
-        ver = btc.get_version_byte(addr)
-    except AssertionError:
-        return False, 'Checksum wrong. Typo in address?'
-    except Exception:
-        return False, "Invalid bitcoin address"
-    if ver != get_p2pk_vbyte() and ver != get_p2sh_vbyte():
-        return False, 'Wrong address version. Testnet/mainnet confused?'
-    if len(btc.b58check_to_bin(addr)) != 20:
-        return False, "Address has correct checksum but wrong length."
-    return True, 'address validated'
-
+        # automatically respects the network
+        # as set in btc.select_chain_params(...)
+        x = btc.CCoinAddress(addr)
+    except Exception as e:
+        return False, repr(e)
+    # additional check necessary because python-bitcointx
+    # does not check hash length on p2sh construction.
+    try:
+        x.to_scriptPubKey()
+    except Exception as e:
+        return False, repr(e)
+    return True, "address validated"
 
 _BURN_DESTINATION = "BURN"
 
@@ -565,6 +550,7 @@ def get_blockchain_interface_instance(_config):
     source = _config.get("BLOCKCHAIN", "blockchain_source")
     network = get_network()
     testnet = network == 'testnet'
+
     if source in ('bitcoin-rpc', 'regtest', 'bitcoin-rpc-no-history'):
         rpc_host = _config.get("BLOCKCHAIN", "rpc_host")
         rpc_port = _config.get("BLOCKCHAIN", "rpc_port")
@@ -574,10 +560,20 @@ def get_blockchain_interface_instance(_config):
             rpc_wallet_file)
         if source == 'bitcoin-rpc': #pragma: no cover
             bc_interface = BitcoinCoreInterface(rpc, network)
+            if testnet:
+                btc.select_chain_params("bitcoin/testnet")
+            else:
+                btc.select_chain_params("bitcoin")
         elif source == 'regtest':
             bc_interface = RegtestBitcoinCoreInterface(rpc)
+            btc.select_chain_params("bitcoin/regtest")
         elif source == "bitcoin-rpc-no-history":
             bc_interface = BitcoinCoreNoHistoryInterface(rpc, network)
+            if testnet or network == "regtest":
+                # TODO will not work for bech32 regtest addresses:
+                btc.select_chain_params("bitcoin/testnet")
+            else:
+                btc.select_chain_params("bitcoin")
         else:
             assert 0
     elif source == 'electrum':

jmclient/jmclient/cryptoengine.py

@@ -19,20 +19,30 @@ NET_MAP = {'mainnet': NET_MAINNET, 'testnet': NET_TESTNET}
 WIF_PREFIX_MAP = {'mainnet': b'\x80', 'testnet': b'\xef'}
 BIP44_COIN_MAP = {'mainnet': 2**31, 'testnet': 2**31 + 1}
 
-def detect_script_type(script):
-    if script.startswith(btc.P2PKH_PRE) and script.endswith(btc.P2PKH_POST) and\
-       len(script) == 0x14 + len(btc.P2PKH_PRE) + len(btc.P2PKH_POST):
+def detect_script_type(script_str):
+    """ Given a scriptPubKey, decide which engine
+    to use, one of: p2pkh, p2sh-p2wpkh, p2wpkh.
+    Note that for the p2sh case, we are assuming the nature
+    of the redeem script (p2wpkh wrapped) because that is what
+    we support; but we can't know for sure, from the sPK only.
+    Raises EngineError if the type cannot be detected, so
+    callers MUST handle this exception to avoid crashes.
+    """
+    script = btc.CScript(script_str)
+    if not script.is_valid():
+        raise EngineError("Unknown script type for script '{}'"
+                          .format(hexlify(script_str)))
+    if script.is_p2pkh():
         return TYPE_P2PKH
-    elif (script.startswith(btc.P2SH_P2WPKH_PRE) and
-          script.endswith(btc.P2SH_P2WPKH_POST) and
-          len(script) == 0x14 + len(btc.P2SH_P2WPKH_PRE) + len(
-              btc.P2SH_P2WPKH_POST)):
+    elif script.is_p2sh():
+        # see note above.
+        # note that is_witness_v0_nested_keyhash does not apply,
+        # since that picks up scriptSigs not scriptPubKeys.
         return TYPE_P2SH_P2WPKH
-    elif script.startswith(btc.P2WPKH_PRE) and\
-            len(script) == 0x14 + len(btc.P2WPKH_PRE):
+    elif script.is_witness_v0_keyhash():
         return TYPE_P2WPKH
     raise EngineError("Unknown script type for script '{}'"
-                      .format(hexlify(script)))
+                      .format(hexlify(script_str)))
 
 class classproperty(object):
     """
@@ -97,28 +107,38 @@ class BTCEngine(object):
 
     @staticmethod
     def privkey_to_pubkey(privkey):
-        return btc.privkey_to_pubkey(privkey, False)
+        return btc.privkey_to_pubkey(privkey)
 
     @staticmethod
     def address_to_script(addr):
-        return unhexlify(btc.address_to_script(addr))
+        return btc.CCoinAddress(addr).to_scriptPubKey()
 
     @classmethod
     def wif_to_privkey(cls, wif):
-        raw = btc.b58check_to_bin(wif)
+        raw = btc.b58check_to_bin(wif)[1]
+        # see note to `privkey_to_wif`; same applies here.
+        # We only handle valid private keys, not any byte string.
+        btc.read_privkey(raw)
+
         vbyte = struct.unpack('B', btc.get_version_byte(wif))[0]
 
-        if (struct.unpack('B', btc.BTC_P2PK_VBYTE[get_network()])[0] + struct.unpack('B', cls.WIF_PREFIX)[0]) & 0xff == vbyte:
+        if (struct.unpack('B', btc.BTC_P2PK_VBYTE[get_network()])[0] + \
+            struct.unpack('B', cls.WIF_PREFIX)[0]) & 0xff == vbyte:
             key_type = TYPE_P2PKH
-        elif (struct.unpack('B', btc.BTC_P2SH_VBYTE[get_network()])[0] + struct.unpack('B', cls.WIF_PREFIX)[0]) & 0xff == vbyte:
+        elif (struct.unpack('B', btc.BTC_P2SH_VBYTE[get_network()])[0] + \
+              struct.unpack('B', cls.WIF_PREFIX)[0]) & 0xff == vbyte:
             key_type = TYPE_P2SH_P2WPKH
         else:
             key_type = None
-
         return raw, key_type
 
     @classmethod
     def privkey_to_wif(cls, priv):
+        # refuse to WIF-ify something that we don't recognize
+        # as a private key; ignoring the return value of this
+        # function as we only want to raise whatever Exception
+        # it does:
+        btc.read_privkey(priv)
         return btc.bin_to_b58check(priv, cls.WIF_PREFIX)
 
     @classmethod
@@ -166,12 +186,12 @@ class BTCEngine(object):
     @classmethod
     def privkey_to_address(cls, privkey):
         script = cls.key_to_script(privkey)
-        return btc.script_to_address(script, cls.VBYTE)
+        return str(btc.CCoinAddress.from_scriptPubKey(script))
 
     @classmethod
     def pubkey_to_address(cls, pubkey):
         script = cls.pubkey_to_script(pubkey)
-        return btc.script_to_address(script, cls.VBYTE)
+        return str(btc.script_to_address(script, cls.VBYTE))
 
     @classmethod
     def pubkey_has_address(cls, pubkey, addr):
@@ -203,7 +223,12 @@ class BTCEngine(object):
 
     @classmethod
     def script_to_address(cls, script):
-        return btc.script_to_address(script, vbyte=cls.VBYTE)
+        """ a script passed in as binary converted to a
+        Bitcoin address of the appropriate type.
+        """
+        s = btc.CScript(script)
+        assert s.is_valid()
+        return str(btc.CCoinAddress.from_scriptPubKey(s))
 
 
 class BTC_P2PKH(BTCEngine):
@@ -213,6 +238,7 @@ class BTC_P2PKH(BTCEngine):
 
     @classmethod
     def pubkey_to_script(cls, pubkey):
+        # this call does not enforce compressed:
         return btc.pubkey_to_p2pkh_script(pubkey)
 
     @classmethod
@@ -222,7 +248,7 @@ class BTC_P2PKH(BTCEngine):
     @classmethod
     def sign_transaction(cls, tx, index, privkey, *args, **kwargs):
         hashcode = kwargs.get('hashcode') or btc.SIGHASH_ALL
-        return btc.sign(btc.serialize(tx), index, privkey,
+        return btc.sign(tx, index, privkey,
                         hashcode=hashcode, amount=None, native=False)
 
 
@@ -250,8 +276,9 @@ class BTC_P2SH_P2WPKH(BTCEngine):
     def sign_transaction(cls, tx, index, privkey, amount,
                          hashcode=btc.SIGHASH_ALL, **kwargs):
         assert amount is not None
-        return btc.sign(btc.serialize(tx), index, privkey,
+        a, b = btc.sign(tx, index, privkey,
                         hashcode=hashcode, amount=amount, native=False)
+        return a, b
 
 class BTC_P2WPKH(BTCEngine):
 
@@ -286,7 +313,7 @@ class BTC_P2WPKH(BTCEngine):
     def sign_transaction(cls, tx, index, privkey, amount,
                          hashcode=btc.SIGHASH_ALL, **kwargs):
         assert amount is not None
-        return btc.sign(btc.serialize(tx), index, privkey,
+        return btc.sign(tx, index, privkey,
                         hashcode=hashcode, amount=amount, native=True)
 
 class BTC_Timelocked_P2WSH(BTCEngine):

jmclient/jmclient/maker.py

@@ -5,15 +5,13 @@ import pprint
 import random
 import sys
 import abc
-from binascii import unhexlify
 
-
-from jmbitcoin import SerializationError, SerializationTruncationError
 import jmbitcoin as btc
+from jmbase import (bintohex, hextobin, hexbin,
+                    get_log, EXIT_SUCCESS, EXIT_FAILURE)
 from jmclient.wallet import estimate_tx_fee, compute_tx_locktime
 from jmclient.wallet_service import WalletService
 from jmclient.configure import jm_single
-from jmbase.support import get_log, EXIT_SUCCESS, EXIT_FAILURE
 from jmclient.support import calc_cj_fee, select_one_utxo
 from jmclient.podle import verify_podle, PoDLE, PoDLEError
 from twisted.internet import task, reactor
@@ -48,10 +46,15 @@ class Maker(object):
             sys.exit(EXIT_FAILURE)
         jlog.info('offerlist={}'.format(self.offerlist))
 
+    @hexbin
     def on_auth_received(self, nick, offer, commitment, cr, amount, kphex):
         """Receives data on proposed transaction offer from daemon, verifies
         commitment, returns necessary data to send ioauth message (utxos etc)
         """
+        # special case due to cjfee passed as string: it can accidentally parse
+        # as hex:
+        if not isinstance(offer["cjfee"], str):
+            offer["cjfee"] = bintohex(offer["cjfee"])
         #check the validity of the proof of discrete log equivalence
         tries = jm_single().config.getint("POLICY", "taker_utxo_retries")
         def reject(msg):
@@ -65,9 +68,8 @@ class Maker(object):
             reason = repr(e)
             return reject(reason)
 
-        if not verify_podle(str(cr_dict['P']), str(cr_dict['P2']), str(cr_dict['sig']),
-                                str(cr_dict['e']), str(commitment),
-                                index_range=range(tries)):
+        if not verify_podle(cr_dict['P'], cr_dict['P2'], cr_dict['sig'],
+                                cr_dict['e'], commitment, index_range=range(tries)):
             reason = "verify_podle failed"
             return reject(reason)
         #finally, check that the proffered utxo is real, old enough, large enough,
@@ -89,14 +91,14 @@ class Maker(object):
 
         try:
             if not self.wallet_service.pubkey_has_script(
-                    unhexlify(cr_dict['P']), unhexlify(res[0]['script'])):
+                    cr_dict['P'], res[0]['script']):
                 raise EngineError()
         except EngineError:
             reason = "Invalid podle pubkey: " + str(cr_dict['P'])
             return reject(reason)
 
         # authorisation of taker passed
-        #Find utxos for the transaction now:
+        # Find utxos for the transaction now:
         utxos, cj_addr, change_addr = self.oid_to_order(offer, amount)
         if not utxos:
             #could not find funds
@@ -109,21 +111,28 @@ class Maker(object):
         # Just choose the first utxo in self.utxos and retrieve key from wallet.
         auth_address = utxos[list(utxos.keys())[0]]['address']
         auth_key = self.wallet_service.get_key_from_addr(auth_address)
-        auth_pub = btc.privtopub(auth_key)
-        btc_sig = btc.ecdsa_sign(kphex, auth_key)
+        auth_pub = btc.privkey_to_pubkey(auth_key)
+        # kphex was auto-converted by @hexbin but we actually need to sign the
+        # hex version to comply with pre-existing JM protocol:
+        btc_sig = btc.ecdsa_sign(bintohex(kphex), auth_key)
         return (True, utxos, auth_pub, cj_addr, change_addr, btc_sig)
 
-    def on_tx_received(self, nick, txhex, offerinfo):
+    @hexbin
+    def on_tx_received(self, nick, tx_from_taker, offerinfo):
         """Called when the counterparty has sent an unsigned
         transaction. Sigs are created and returned if and only
         if the transaction passes verification checks (see
         verify_unsigned_tx()).
         """
+        # special case due to cjfee passed as string: it can accidentally parse
+        # as hex:
+        if not isinstance(offerinfo["offer"]["cjfee"], str):
+            offerinfo["offer"]["cjfee"] = bintohex(offerinfo["offer"]["cjfee"])
         try:
-            tx = btc.deserialize(txhex)
-        except (IndexError, SerializationError, SerializationTruncationError) as e:
+            tx = btc.CMutableTransaction.deserialize(tx_from_taker)
+        except Exception as e:
             return (False, 'malformed txhex. ' + repr(e))
-        jlog.info('obtained tx\n' + pprint.pformat(tx))
+        jlog.info('obtained tx\n' + bintohex(tx.serialize()))
         goodtx, errmsg = self.verify_unsigned_tx(tx, offerinfo)
         if not goodtx:
             jlog.info('not a good tx, reason=' + errmsg)
@@ -133,25 +142,26 @@ class Maker(object):
         utxos = offerinfo["utxos"]
 
         our_inputs = {}
-        for index, ins in enumerate(tx['ins']):
-            utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
+        for index, ins in enumerate(tx.vin):
+            utxo = (ins.prevout.hash[::-1], ins.prevout.n)
             if utxo not in utxos:
                 continue
             script = self.wallet_service.addr_to_script(utxos[utxo]['address'])
             amount = utxos[utxo]['value']
             our_inputs[index] = (script, amount)
 
-        txs = self.wallet_service.sign_tx(btc.deserialize(unhexlify(txhex)), our_inputs)
+        success, msg = self.wallet_service.sign_tx(tx, our_inputs)
+        assert success, msg
         for index in our_inputs:
-            sigmsg = unhexlify(txs['ins'][index]['script'])
-            if 'txinwitness' in txs['ins'][index]:
+            sigmsg = tx.vin[index].scriptSig
+            if tx.has_witness():
                 # Note that this flag only implies that the transaction
                 # *as a whole* is using segwit serialization; it doesn't
                 # imply that this specific input is segwit type (to be
                 # fully general, we allow that even our own wallet's
                 # inputs might be of mixed type). So, we catch the EngineError
                 # which is thrown by non-segwit types. This way the sigmsg
-                # will only contain the scriptSig field if the wallet object
+                # will only contain the scriptCode field if the wallet object
                 # decides it's necessary/appropriate for this specific input
                 # If it is segwit, we prepend the witness data since we want
                 # (sig, pub, witnessprogram=scriptSig - note we could, better,
@@ -160,16 +170,16 @@ class Maker(object):
                 # transaction scriptSig), else (non-sw) the !sig message remains
                 # unchanged as (sig, pub).
                 try:
-                    scriptSig = btc.pubkey_to_p2wpkh_script(txs['ins'][index]['txinwitness'][1])
-                    sigmsg = b''.join(btc.serialize_script_unit(
-                x) for x in txs['ins'][index]['txinwitness'] + [scriptSig])
-                except IndexError:
+                    sig, pub = [a for a in iter(tx.wit.vtxinwit[index].scriptWitness)]
+                    scriptCode = btc.pubkey_to_p2wpkh_script(pub)
+                    sigmsg = btc.CScript([sig]) + btc.CScript(pub) + scriptCode
+                except Exception as e:
                     #the sigmsg was already set before the segwit check
                     pass
             sigs.append(base64.b64encode(sigmsg).decode('ascii'))
         return (True, sigs)
 
-    def verify_unsigned_tx(self, txd, offerinfo):
+    def verify_unsigned_tx(self, tx, offerinfo):
         """This code is security-critical.
         Before signing the transaction the Maker must ensure
         that all details are as expected, and most importantly
@@ -177,14 +187,13 @@ class Maker(object):
         in total. The data is taken from the offerinfo dict and
         compared with the serialized txhex.
         """
-        tx_utxo_set = set(ins['outpoint']['hash'] + ':' + str(
-                ins['outpoint']['index']) for ins in txd['ins'])
+        tx_utxo_set = set((x.prevout.hash[::-1], x.prevout.n) for x in tx.vin)
 
         utxos = offerinfo["utxos"]
         cjaddr = offerinfo["cjaddr"]
-        cjaddr_script = btc.address_to_script(cjaddr)
+        cjaddr_script = btc.CCoinAddress(cjaddr).to_scriptPubKey()
         changeaddr = offerinfo["changeaddr"]
-        changeaddr_script = btc.address_to_script(changeaddr)
+        changeaddr_script = btc.CCoinAddress(changeaddr).to_scriptPubKey()
         #Note: this value is under the control of the Taker,
         #see comment below.
         amount = offerinfo["amount"]
@@ -216,14 +225,14 @@ class Maker(object):
         #exactly once with the required amts, in the output.
         times_seen_cj_addr = 0
         times_seen_change_addr = 0
-        for outs in txd['outs']:
-            if outs['script'] == cjaddr_script:
+        for outs in tx.vout:
+            if outs.scriptPubKey == cjaddr_script:
                 times_seen_cj_addr += 1
-                if outs['value'] != amount:
+                if outs.nValue != amount:
                     return (False, 'Wrong cj_amount. I expect ' + str(amount))
-            if outs['script'] == changeaddr_script:
+            if outs.scriptPubKey == changeaddr_script:
                 times_seen_change_addr += 1
-                if outs['value'] != expected_change_value:
+                if outs.nValue != expected_change_value:
                     return (False, 'wrong change, i expect ' + str(
                         expected_change_value))
         if times_seen_cj_addr != 1 or times_seen_change_addr != 1:
@@ -392,7 +401,8 @@ class P2EPMaker(Maker):
         # will not be reached except in testing
         self.on_tx_unconfirmed(txd, txid)
 
-    def on_tx_received(self, nick, txhex):
+    @hexbin
+    def on_tx_received(self, nick, txser):
         """ Called when the sender-counterparty has sent a transaction proposal.
         1. First we check for the expected destination and amount (this is
            sufficient to identify our cp, as this info was presumably passed
@@ -418,29 +428,30 @@ class P2EPMaker(Maker):
            we broadcast the non-coinjoin fallback tx instead.
         """
         try:
-            tx = btc.deserialize(txhex)
-        except (IndexError, SerializationError, SerializationTruncationError) as e:
+            tx = btc.CMutableTransaction.deserialize(txser)
+        except Exception as e:
             return (False, 'malformed txhex. ' + repr(e))
         self.user_info('obtained proposed fallback (non-coinjoin) ' +\
-                       'transaction from sender:\n' + pprint.pformat(tx))
+                       'transaction from sender:\n' + str(tx))
 
-        if len(tx["outs"]) != 2:
+        if len(tx.vout) != 2:
             return (False, "Transaction has more than 2 outputs; not supported.")
         dest_found = False
         destination_index = -1
         change_index = -1
         proposed_change_value = 0
-        for index, out in enumerate(tx["outs"]):
-            if out["script"] == btc.address_to_script(self.destination_addr):
+        for index, out in enumerate(tx.vout):
+            if out.scriptPubKey == btc.CCoinAddress(
+                self.destination_addr).to_scriptPubKey():
                 # we found the expected destination; is the amount correct?
-                if not out["value"] == self.receiving_amount:
+                if not out.nValue == self.receiving_amount:
                     return (False, "Wrong payout value in proposal from sender.")
                 dest_found = True
                 destination_index = index
             else:
                 change_found = True
-                proposed_change_out = out["script"]
-                proposed_change_value = out["value"]
+                proposed_change_out = out.scriptPubKey
+                proposed_change_value = out.nValue
                 change_index = index
 
         if not dest_found:
@@ -450,9 +461,8 @@ class P2EPMaker(Maker):
         # batch retrieval of utxo data
         utxo = {}
         ctr = 0
-        for index, ins in enumerate(tx['ins']):
-            utxo_for_checking = ins['outpoint']['hash'] + ':' + str(ins[
-                'outpoint']['index'])
+        for index, ins in enumerate(tx.vin):
+            utxo_for_checking = (ins.prevout.hash[::-1], ins.prevout.n)
             utxo[ctr] = [index, utxo_for_checking]
             ctr += 1
 
@@ -471,7 +481,7 @@ class P2EPMaker(Maker):
         btc_fee = total_sender_input - self.receiving_amount - proposed_change_value
         self.user_info("Network transaction fee of fallback tx is: " + str(
             btc_fee) + " satoshis.")
-        fee_est = estimate_tx_fee(len(tx['ins']), len(tx['outs']),
+        fee_est = estimate_tx_fee(len(tx.vin), len(tx.vout),
                                   txtype=self.wallet_service.get_txtype())
         fee_ok = False
         if btc_fee > 0.3 * fee_est and btc_fee < 3 * fee_est:
@@ -488,51 +498,28 @@ class P2EPMaker(Maker):
         # It has the advantage of (a) being simpler and (b) allowing for any
         # non standard coins.
         #
-        #res = jm_single().bc_interface.rpc('testmempoolaccept', [txhex])
+        #res = jm_single().bc_interface.rpc('testmempoolaccept', [txser])
         #print("Got this result from rpc call: ", res)
         #if not res["accepted"]:
         #    return (False, "Proposed transaction was rejected from mempool.")
 
-        # Manual verification of the transaction signatures. Passing this
-        # test does imply that the transaction is valid (unless there is
-        # a double spend during the process), but is restricted to standard
-        # types: p2pkh, p2wpkh, p2sh-p2wpkh only. Double spend is not counted
-        # as a risk as this is a payment.
+        # Manual verification of the transaction signatures.
+        # TODO handle native segwit properly
         for i, u in iteritems(utxo):
-            if "txinwitness" in tx["ins"][u[0]]:
-                ver_amt = utxo_data[i]["value"]
-                try:
-                    ver_sig, ver_pub = tx["ins"][u[0]]["txinwitness"]
-                except Exception as e:
-                    self.user_info("Segwit error: " + repr(e))
-                    return (False, "Segwit input not of expected type, "
-                            "either p2sh-p2wpkh or p2wpkh")
-                # note that the scriptCode is the same whether nested or not
-                # also note that the scriptCode has to be inferred if we are
-                # only given a transaction serialization.
-                scriptCode = "76a914" + btc.hash160(unhexlify(ver_pub)) + "88ac"
-            else:
-                scriptCode = None
-                ver_amt = None
-                scriptSig = btc.deserialize_script(tx["ins"][u[0]]["script"])
-                if len(scriptSig) != 2:
-                    return (False,
-                        "Proposed transaction contains unsupported input type")
-                ver_sig, ver_pub = scriptSig
-            if not btc.verify_tx_input(txhex, u[0],
-                                           utxo_data[i]['script'],
-                                           ver_sig, ver_pub,
-                                           scriptCode=scriptCode,
-                                           amount=ver_amt):
+            if not btc.verify_tx_input(tx, i,
+                                       tx.vin[i].scriptSig,
+                                       btc.CScript(utxo_data[i]["script"]),
+                                       amount=utxo_data[i]["value"],
+                                       witness=tx.wit.vtxinwit[i].scriptWitness):
                 return (False, "Proposed transaction is not correctly signed.")
 
         # At this point we are satisfied with the proposal. Record the fallback
         # in case the sender disappears and the payjoin tx doesn't happen:
         self.user_info("We'll use this serialized transaction to broadcast if your"
                   " counterparty fails to broadcast the payjoin version:")
-        self.user_info(txhex)
+        self.user_info(bintohex(txser))
         # Keep a local copy for broadcast fallback:
-        self.fallback_tx = txhex
+        self.fallback_tx = tx
 
         # Now we add our own inputs:
         # See the gist comment here:
@@ -558,7 +545,7 @@ class P2EPMaker(Maker):
             self.user_info("Choosing one coin at random")
             try:
                 my_utxos = self.wallet_service.select_utxos(
-                    self.mixdepth,  jm_single().DUST_THRESHOLD,
+                    self.mixdepth, jm_single().DUST_THRESHOLD,
                     select_fn=select_one_utxo)
             except:
                 return self.no_coins_fallback()
@@ -566,7 +553,7 @@ class P2EPMaker(Maker):
         else:
             # get an approximate required amount assuming 4 inputs, which is
             # fairly conservative (but guess by necessity).
-            fee_for_select = estimate_tx_fee(len(tx['ins']) + 4, 2,
+            fee_for_select = estimate_tx_fee(len(tx.vin) + 4, 2,
                                              txtype=self.wallet_service.get_txtype())
             approx_sum = max_sender_amt - self.receiving_amount + fee_for_select
             try:
@@ -592,7 +579,7 @@ class P2EPMaker(Maker):
         # adjust the output amount at the destination based on our contribution
         new_destination_amount = self.receiving_amount + my_total_in
         # estimate the required fee for the new version of the transaction
-        total_ins = len(tx["ins"]) + len(my_utxos.keys())
+        total_ins = len(tx.vin) + len(my_utxos.keys())
         est_fee = estimate_tx_fee(total_ins, 2, txtype=self.wallet_service.get_txtype())
         self.user_info("We estimated a fee of: " + str(est_fee))
         new_change_amount = total_sender_input + my_total_in - \
@@ -604,25 +591,27 @@ class P2EPMaker(Maker):
         new_outs = [{"address": self.destination_addr,
                      "value": new_destination_amount}]
         if new_change_amount >= jm_single().BITCOIN_DUST_THRESHOLD:
-            new_outs.append({"script": proposed_change_out,
-                     "value": new_change_amount})
+            new_outs.append({"address": str(btc.CCoinAddress.from_scriptPubKey(
+                proposed_change_out)), "value": new_change_amount})
         new_ins = [x[1] for x in utxo.values()]
         new_ins.extend(my_utxos.keys())
-        new_tx = btc.make_shuffled_tx(new_ins, new_outs, False, 2, compute_tx_locktime())
-        new_tx_deser = btc.deserialize(new_tx)
+        new_tx = btc.make_shuffled_tx(new_ins, new_outs, 2, compute_tx_locktime())
+
 
         # sign our inputs before transfer
         our_inputs = {}
-        for index, ins in enumerate(new_tx_deser['ins']):
-            utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
+        for index, ins in enumerate(new_tx.vin):
+            utxo = (ins.prevout.hash[::-1], ins.prevout.n)
             if utxo not in my_utxos:
                 continue
-            script = self.wallet_service.addr_to_script(my_utxos[utxo]['address'])
-            amount = my_utxos[utxo]['value']
+            script = my_utxos[utxo]["script"]
+            amount = my_utxos[utxo]["value"]
             our_inputs[index] = (script, amount)
 
-        txs = self.wallet_service.sign_tx(btc.deserialize(new_tx), our_inputs)
-        txinfo = tuple((x["script"], x["value"]) for x in txs["outs"])
+        success, msg = self.wallet_service.sign_tx(new_tx, our_inputs)
+        if not success:
+            return (False, "Failed to sign new transaction, error: " + msg)
+        txinfo = tuple((x.scriptPubKey, x.nValue) for x in new_tx.vout)
         self.wallet_service.register_callbacks([self.on_tx_unconfirmed], txinfo, "unconfirmed")
         self.wallet_service.register_callbacks([self.on_tx_confirmed], txinfo, "confirmed")
         # The blockchain interface just abandons monitoring if the transaction
@@ -630,7 +619,7 @@ class P2EPMaker(Maker):
         # action in this case, so we add an additional callback to the reactor:
         reactor.callLater(jm_single().config.getint("TIMEOUT",
                             "unconfirm_timeout_sec"), self.broadcast_fallback)
-        return (True, nick, btc.serialize(txs))
+        return (True, nick, bintohex(new_tx.serialize()))
 
     def no_coins_fallback(self):
         """ Broadcast, optionally, the fallback non-coinjoin transaction
@@ -646,8 +635,8 @@ class P2EPMaker(Maker):
 
     def broadcast_fallback(self):
         self.user_info("Broadcasting non-coinjoin fallback transaction.")
-        txid = btc.txhash(self.fallback_tx)
-        success = jm_single().bc_interface.pushtx(self.fallback_tx)
+        txid = self.fallback_tx.GetTxid()[::-1]
+        success = jm_single().bc_interface.pushtx(self.fallback_tx.serialize())
         if not success:
             self.user_info("ERROR: the fallback transaction did not broadcast. "
                       "The payment has NOT been made.")

jmclient/jmclient/output.py

@@ -61,7 +61,7 @@ def generate_podle_error_string(priv_utxo_pairs, to, ts, wallet_service, cjamoun
                "with 'python add-utxo.py --help'\n\n")
     errmsg += ("***\nFor reference, here are the utxos in your wallet:\n")
 
-    for md, utxos in wallet_service.get_utxos_by_mixdepth(hexfmt=False).items():
+    for md, utxos in wallet_service.get_utxos_by_mixdepth().items():
         if not utxos:
             continue
         errmsg += ("\nmixdepth {}:\n{}".format(

jmclient/jmclient/podle.py

@@ -7,11 +7,12 @@ import hashlib
 import json
 import binascii
 import struct
+from pprint import pformat
 from jmbase import jmprint
 from jmbitcoin import multiply, add_pubkeys, getG, podle_PublicKey,\
-    podle_PrivateKey, encode, decode, N, podle_PublicKey_class
-from jmbase.support import EXIT_FAILURE
-
+    podle_PrivateKey, N, podle_PublicKey_class
+from jmbase import (EXIT_FAILURE, utxostr_to_utxo,
+                    utxo_to_utxostr, hextobin, bintohex)
 
 PODLE_COMMIT_FILE = None
 
@@ -45,33 +46,31 @@ class PoDLE(object):
                  used=False):
         #This class allows storing of utxo in format "txid:n" only for
         #convenience of storage/access; it doesn't check or use the data.
-        #Arguments must be provided in hex.
+        #Arguments must be provided in binary not hex.
         self.u = u
         if not priv:
             if P:
-                #Construct a pubkey from raw hex
-                self.P = podle_PublicKey(binascii.unhexlify(P))
+                self.P = podle_PublicKey(P)
             else:
                 self.P = None
         else:
             if P:
                 raise PoDLEError("Pubkey should not be provided with privkey")
             #any other formatting abnormality will just throw in PrivateKey
-            if len(priv) == 66 and priv[-2:] == '01':
-                priv = priv[:-2]
-            self.priv = podle_PrivateKey(binascii.unhexlify(priv))
+            if len(priv) == 33 and priv[-1:] == b"\x01":
+                priv = priv[:-1]
+            self.priv = podle_PrivateKey(priv)
             self.P = self.priv.public_key
         if P2:
-            self.P2 = podle_PublicKey(binascii.unhexlify(P2))
+            self.P2 = podle_PublicKey(P2)
         else:
             self.P2 = None
-        #These sig values should be passed in hex.
         self.s = None
         self.e = None
         if s:
-            self.s = binascii.unhexlify(s)
+            self.s = s
         if e:
-            self.e = binascii.unhexlify(e)
+            self.e = e
         #Optionally maintain usage state (boolean)
         self.used = used
         #the H(P2) value
@@ -79,14 +78,13 @@ class PoDLE(object):
 
     def get_commitment(self):
         """Set the commitment to sha256(serialization of public key P2)
-        Return in hex to calling function
         """
         if not self.P2:
             raise PoDLEError("Cannot construct commitment, no P2 available")
         if not isinstance(self.P2, podle_PublicKey_class):
             raise PoDLEError("Cannot construct commitment, P2 is not a pubkey")
         self.commitment = hashlib.sha256(self.P2.format()).digest()
-        return binascii.hexlify(self.commitment).decode('ascii')
+        return self.commitment
 
     def generate_podle(self, index=0, k=None):
         """Given a raw private key, in hex format,
@@ -117,79 +115,85 @@ class PoDLE(object):
         and the associated signature data that can be used to open
         the commitment.
         """
+        self.i = index
         #TODO nonce could be rfc6979?
         if not k:
             k = os.urandom(32)
-        J = getNUMS(index)
+        J = getNUMS(self.i)
         KG = podle_PrivateKey(k).public_key
-        KJ = multiply(k, J.format(), False, return_serialized=False)
+        KJ = multiply(k, J.format(), return_serialized=False)
         self.P2 = getP2(self.priv, J)
         self.get_commitment()
         self.e = hashlib.sha256(b''.join([x.format(
         ) for x in [KG, KJ, self.P, self.P2]])).digest()
-        k_int = decode(k, 256)
-        priv_int = decode(self.priv.secret, 256)
-        e_int = decode(self.e, 256)
+        k_int, priv_int, e_int = (int.from_bytes(x,
+            byteorder="big") for x in [k, self.priv.secret, self.e])
         sig_int = (k_int + priv_int * e_int) % N
-        self.s = encode(sig_int, 256, minlen=32)
+        self.s = (sig_int).to_bytes(32, byteorder="big")
         return self.reveal()
 
     def reveal(self):
         """Encapsulate all the data representing the proof
-        in a dict for client functions. Data output in hex.
+        in a dict for client functions.
         """
         if not all([self.u, self.P, self.P2, self.s, self.e]):
             raise PoDLEError("Cannot generate proof, data is missing")
         if not self.commitment:
             self.get_commitment()
-        Phex, P2hex, shex, ehex, commit = [
-            binascii.hexlify(x).decode('ascii')
-            for x in [self.P.format(), self.P2.format(), self.s, self.e,
-                      self.commitment]
-        ]
-        return {'used': str(self.used),
+        return {'used': self.used,
                 'utxo': self.u,
-                'P': Phex,
-                'P2': P2hex,
-                'commit': commit,
-                'sig': shex,
-                'e': ehex}
+                'P': self.P.format(),
+                'P2': self.P2.format(),
+                'commit': self.commitment,
+                'sig': self.s,
+                'e': self.e}
 
     def serialize_revelation(self, separator='|'):
+        """ Outputs the over-the-wire format as used in
+        Joinmarket communication protocol.
+        """
         state_dict = self.reveal()
-        ser_list = []
-        for k in ['utxo', 'P', 'P2', 'sig', 'e']:
-            ser_list += [state_dict[k]]
+        success, utxo = utxo_to_utxostr(state_dict["utxo"])
+        assert success, "invalid utxo in PoDLE"
+        ser_list = [utxo]
+        ser_list += [bintohex(state_dict[x]) for x in ["P", "P2", "sig", "e"]]
         ser_string = separator.join(ser_list)
         return ser_string
 
     @classmethod
     def deserialize_revelation(cls, ser_rev, separator='|'):
+        """ Reads the over-the-wire format as used in
+        Joinmarket communication protocol.
+        """
         ser_list = ser_rev.split(separator)
         if len(ser_list) != 5:
             raise PoDLEError("Failed to deserialize, wrong format")
-        utxo, P, P2, s, e = ser_list
-        return {'utxo': utxo, 'P': P, 'P2': P2, 'sig': s, 'e': e}
+        utxostr, P, P2, s, e = ser_list
+        success, utxo = utxostr_to_utxo(utxostr)
+        assert success, "invalid utxo format in PoDLE."
+        return {'utxo': utxo, 'P': hextobin(P),
+                'P2': hextobin(P2), 'sig': hextobin(s), 'e': hextobin(e)}
 
     def verify(self, commitment, index_range):
         """For an object created without a private key,
         check that the opened commitment verifies for at least
-        one NUMS point as defined by the range in index_range
+        one NUMS point as defined by the range in index_range.
         """
         if not all([self.P, self.P2, self.s, self.e]):
             raise PoDLEError("Verify called without sufficient data")
         if not self.get_commitment() == commitment:
             return False
+
         for J in [getNUMS(i) for i in index_range]:
             sig_priv = podle_PrivateKey(self.s)
             sG = sig_priv.public_key
-            sJ = multiply(self.s, J.format(), False)
-            e_int = decode(self.e, 256)
-            minus_e = encode(-e_int % N, 256, minlen=32)
-            minus_e_P = multiply(minus_e, self.P.format(), False)
-            minus_e_P2 = multiply(minus_e, self.P2.format(), False)
-            KGser = add_pubkeys([sG.format(), minus_e_P], False)
-            KJser = add_pubkeys([sJ, minus_e_P2], False)
+            sJ = multiply(self.s, J.format())
+            e_int = int.from_bytes(self.e, byteorder="big")
+            minus_e = (-e_int % N).to_bytes(32, byteorder="big")
+            minus_e_P = multiply(minus_e, self.P.format())
+            minus_e_P2 = multiply(minus_e, self.P2.format())
+            KGser = add_pubkeys([sG.format(), minus_e_P])
+            KJser = add_pubkeys([sJ, minus_e_P2])
             #check 2: e =?= H(K_G || K_J || P || P2)
             e_check = hashlib.sha256(KGser + KJser + self.P.format() +
                                      self.P2.format()).digest()
@@ -198,6 +202,20 @@ class PoDLE(object):
         #commitment fails for any NUMS in the provided range
         return False
 
+    def __repr__(self):
+        """ Specified here to allow logging.
+        """
+        # note: will throw if not fully initalised
+        r = self.reveal()
+        success, utxo = utxo_to_utxostr(r["utxo"])
+        assert success, "invalid utxo in PoDLE."
+        return pformat({'used': r["used"],
+                'utxo': utxo,
+                'P': bintohex(r["P"]),
+                'P2': bintohex(r["P2"]),
+                'commit': bintohex(r["commit"]),
+                'sig': bintohex(r["sig"]),
+                'e': bintohex(r["e"])})
 
 def getNUMS(index=0):
     """Taking secp256k1's G as a seed,
@@ -244,7 +262,7 @@ def verify_all_NUMS(write=False):
     """
     nums_points = {}
     for i in range(256):
-        nums_points[i] = binascii.hexlify(getNUMS(i).format()).decode('ascii')
+        nums_points[i] = bintohex(getNUMS(i).format())
     if write:
         with open("nums_basepoints.txt", "wb") as f:
             from pprint import pformat
@@ -263,13 +281,82 @@ def getP2(priv, nums_pt):
     priv_raw = priv.secret
     return multiply(priv_raw,
                     nums_pt.format(),
-                    False,
                     return_serialized=False)
 
+# functions which interact with the external persistence of podle data:
+
+def switch_external_dict_format(ed, utxo_converter, hexbinconverter):
+    """External dict has structure:
+    {txid:N:{'P':pubkey, 'reveal':{1:{'P2':P2,'s':s,'e':e}, 2:{..},..}}}
+    This function switches between readable/writable in file (strings, hex)
+    and that used internally.
+    """
+    retval = {}
+    for u in ed:
+        success, u2 = utxo_converter(u)
+        assert success, "invalid utxo format in external dict parsing."
+        retval[u2] = {"P": hexbinconverter(ed[u]["P"])}
+        retval[u2]["reveal"] = {}
+        for i in ed[u]["reveal"]:
+            # hack: python json does not allow int dict keys, they must
+            # be str, so when reading from file, we convert back to int:
+            if not isinstance(i, int):
+                j = int(i)
+            else:
+                j = i
+            retval[u2]["reveal"][j] = {
+                "P2": hexbinconverter(ed[u]["reveal"][i]["P2"]),
+                "s": hexbinconverter(ed[u]["reveal"][i]["s"]),
+                "e": hexbinconverter(ed[u]["reveal"][i]["e"])}
+    return retval
+
+def external_dict_to_file(ed):
+    """ Converts internal format of dict to one writable/readable
+    in file.
+    """
+    return switch_external_dict_format(ed, utxo_to_utxostr, bintohex)
+
+def external_dict_from_file(ed):
+    """ Takes the external dict extracted through json deserialization
+    from a file and converts it to internal format:
+    {txid:N:{'P':pubkey, 'reveal':{1:{'P2':P2,'s':s,'e':e}, 2:{..},..}}}
+    """
+    return switch_external_dict_format(ed, utxostr_to_utxo, hextobin)
+
+def write_to_podle_file(used, external):
+    """ Update persisted commitment data in PODLE_COMMIT_FILE.
+    """
+    to_write = {}
+    to_write['used'] = [bintohex(x) for x in used]
+    externalfmt = external_dict_to_file(external)
+    to_write['external'] = externalfmt
+    with open(PODLE_COMMIT_FILE, "wb") as f:
+        f.write(json.dumps(to_write, indent=4).encode('utf-8'))
+
+def read_from_podle_file():
+    """ Returns used commitment list and external commitments dict
+    struct currently stored in PODLE_COMMIT_FILE.
+    """
+    if os.path.isfile(PODLE_COMMIT_FILE):
+        with open(PODLE_COMMIT_FILE, "rb") as f:
+            try:
+                c = json.loads(f.read().decode('utf-8'))
+            except ValueError: #pragma: no cover
+                #Exit conditions cannot be included in tests.
+                jmprint("the file: " + PODLE_COMMIT_FILE + " is not valid json.",
+                        "error")
+                sys.exit(EXIT_FAILURE)
+            if 'used' not in c.keys() or 'external' not in c.keys():
+                raise PoDLEError("Incorrectly formatted file: " + PODLE_COMMIT_FILE)
+
+        used = [hextobin(x) for x in c["used"]]
+        external = external_dict_from_file(c["external"])
+        return (used, external)
+    return ([], {})
 
 def get_podle_commitments():
     """Returns set of commitments used as a list:
-    [H(P2),..] (hex) and a dict of all existing external commitments.
+    [H(P2),..] and a dict of all existing external commitments.
     It is presumed that each H(P2) can
     be used only once (this may not literally be true, but represents
     good joinmarket "citizenship").
@@ -280,18 +367,11 @@ def get_podle_commitments():
     """
     if not os.path.isfile(PODLE_COMMIT_FILE):
         return ([], {})
-    with open(PODLE_COMMIT_FILE, "rb") as f:
-        c = json.loads(f.read().decode('utf-8'))
-    if 'used' not in c.keys() or 'external' not in c.keys():
-        raise PoDLEError("Incorrectly formatted file: " + PODLE_COMMIT_FILE)
-    return (c['used'], c['external'])
-
+    return read_from_podle_file()
 
 def add_external_commitments(ecs):
     """To allow external functions to add
-    PoDLE commitments that were calculated elsewhere;
-    the format of each entry in ecs must be:
-    {txid:N:{'P':pubkey, 'reveal':{1:{'P2':P2,'s':s,'e':e}, 2:{..},..}}}
+    PoDLE commitments that were calculated elsewhere.
     """
     update_commitments(external_to_add=ecs)
 
@@ -304,25 +384,7 @@ def update_commitments(commitment=None,
     whose key value is the utxo in external_to_remove,
     persist updated entries to disk.
     """
-    c = {}
-    if os.path.isfile(PODLE_COMMIT_FILE):
-        with open(PODLE_COMMIT_FILE, "rb") as f:
-            try:
-                c = json.loads(f.read().decode('utf-8'))
-            except ValueError: #pragma: no cover
-                #Exit conditions cannot be included in tests.
-                jmprint("the file: " + PODLE_COMMIT_FILE + " is not valid json.",
-                        "error")
-                sys.exit(EXIT_FAILURE)
-
-    if 'used' in c:
-        commitments = c['used']
-    else:
-        commitments = []
-    if 'external' in c:
-        external = c['external']
-    else:
-        external = {}
+    commitments, external = read_from_podle_file()
     if commitment:
         commitments.append(commitment)
         #remove repeats
@@ -334,11 +396,7 @@ def update_commitments(commitment=None,
         }
     if external_to_add:
         external.update(external_to_add)
-    to_write = {}
-    to_write['used'] = commitments
-    to_write['external'] = external
-    with open(PODLE_COMMIT_FILE, "wb") as f:
-        f.write(json.dumps(to_write, indent=4).encode('utf-8'))
+    write_to_podle_file(commitments, external)
 
 def get_podle_tries(utxo, priv=None, max_tries=1, external=False):
     used_commitments, external_commitments = get_podle_commitments()
@@ -349,9 +407,8 @@ def get_podle_tries(utxo, priv=None, max_tries=1, external=False):
             #use as many as were provided in the file, up to a max of max_tries
             m = min([len(ec['reveal'].keys()), max_tries])
             for i in reversed(range(m)):
-                key = str(i)
-                p = PoDLE(u=utxo,P=ec['P'],P2=ec['reveal'][key]['P2'],
-                          s=ec['reveal'][key]['s'], e=ec['reveal'][key]['e'])
+                p = PoDLE(u=utxo, P=ec["P"], P2=ec["reveal"][i]["P2"],
+                          s=ec["reveal"][i]["s"], e=ec["reveal"][i]["e"])
                 if p.get_commitment() in used_commitments:
                     return i+1
     else:
@@ -387,10 +444,10 @@ def generate_podle(priv_utxo_pairs, max_tries=1, allow_external=None, k=None):
         #which is still available.
         index = tries
         p = PoDLE(u=utxo, priv=priv)
-        c = p.generate_podle(index)
+        p.generate_podle(index)
         #persist for future checks
-        update_commitments(commitment=c['commit'])
-        return c
+        update_commitments(commitment=p.commitment)
+        return p
     if allow_external:
         for u in allow_external:
             tries = get_podle_tries(utxo=u, max_tries=max_tries, external=True)
@@ -400,19 +457,17 @@ def generate_podle(priv_utxo_pairs, max_tries=1, allow_external=None, k=None):
                 #remove this entry
                 update_commitments(external_to_remove=u)
                 continue
-            index = str(tries)
             ec = external_commitments[u]
-            p = PoDLE(u=u,P=ec['P'],P2=ec['reveal'][index]['P2'],
-                      s=ec['reveal'][index]['s'], e=ec['reveal'][index]['e'])
+            ecri = ec["reveal"][tries]
+            p = PoDLE(u=u, P=ec["P"], P2=ecri["P2"], s=ecri["s"], e=ecri["e"])
             update_commitments(commitment=p.get_commitment())
-            return p.reveal()
+            return p
     #Failed to find any non-used valid commitment:
     return None
 
 
 def verify_podle(Pser, P2ser, sig, e, commitment, index_range=range(10)):
     verifying_podle = PoDLE(P=Pser, P2=P2ser, s=sig, e=e)
-    #check 1: Hash(P2ser) =?= commitment
     if not verifying_podle.verify(commitment, index_range):
         return False
     return True

jmclient/jmclient/taker.py

@@ -5,12 +5,10 @@ import base64
 import pprint
 import random
 from twisted.internet import reactor, task
-from binascii import hexlify, unhexlify
 
-from jmbitcoin import SerializationError, SerializationTruncationError
 import jmbitcoin as btc
 from jmclient.configure import jm_single, validate_address
-from jmbase.support import get_log
+from jmbase import get_log, hextobin, bintohex, hexbin
 from jmclient.support import (calc_cj_fee, weighted_order_choose, choose_orders,
                               choose_sweep_orders)
 from jmclient.wallet import estimate_tx_fee, compute_tx_locktime
@@ -238,7 +236,6 @@ class Taker(object):
         #Initialization has been successful. We must set the nonrespondants
         #now to keep track of what changed when we receive the utxo data
         self.nonrespondants = list(self.orderbook.keys())
-
         return (True, self.cjamount, commitment, revelation, self.orderbook)
 
     def filter_orderbook(self, orderbook, sweep=False):
@@ -344,6 +341,7 @@ class Taker(object):
         self.utxos = {None: list(self.input_utxos.keys())}
         return True
 
+    @hexbin
     def receive_utxos(self, ioauth_data):
         """Triggered when the daemon returns utxo data from
         makers who responded; this is the completion of phase 1
@@ -354,6 +352,7 @@ class Taker(object):
 
         #Temporary list used to aggregate all ioauth data that must be removed
         rejected_counterparties = []
+
         #Need to authorize against the btc pubkey first.
         for nick, nickdata in iteritems(ioauth_data):
             utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata
@@ -376,10 +375,9 @@ class Taker(object):
         self.maker_utxo_data = {}
 
         for nick, nickdata in iteritems(ioauth_data):
-            utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata
+            utxo_list, auth_pub, cj_addr, change_addr, _, _ = nickdata
+            utxo_data = jm_single().bc_interface.query_utxo_set(utxo_list)
             self.utxos[nick] = utxo_list
-            utxo_data = jm_single().bc_interface.query_utxo_set(self.utxos[
-                nick])
             if None in utxo_data:
                 jlog.warn(('ERROR outputs unconfirmed or already spent. '
                            'utxo_data={}').format(pprint.pformat(utxo_data)))
@@ -391,13 +389,12 @@ class Taker(object):
             #Extract the address fields from the utxos
             #Construct the Bitcoin address for the auth_pub field
             #Ensure that at least one address from utxos corresponds.
-            auth_pub_bin = unhexlify(auth_pub)
             for inp in utxo_data:
                 try:
                     if self.wallet_service.pubkey_has_script(
-                            auth_pub_bin, unhexlify(inp['script'])):
+                            auth_pub, inp['script']):
                         break
-                except EngineError:
+                except EngineError as e:
                     pass
             else:
                 jlog.warn("ERROR maker's (" + nick + ")"
@@ -500,81 +497,85 @@ class Taker(object):
         else:
             self.outputs.append({'address': self.my_change_addr,
                                  'value': my_change_value})
-        self.utxo_tx = [dict([('output', u)])
-                        for u in sum(self.utxos.values(), [])]
+        self.utxo_tx = [u for u in sum(self.utxos.values(), [])]
         self.outputs.append({'address': self.coinjoin_address(),
                              'value': self.cjamount})
-        tx = btc.make_shuffled_tx(self.utxo_tx, self.outputs, False)
-        jlog.info('obtained tx\n' + pprint.pformat(btc.deserialize(tx)))
+        self.latest_tx = btc.make_shuffled_tx(self.utxo_tx, self.outputs)
+        jlog.info('obtained tx\n' + bintohex(self.latest_tx.serialize()))
 
-        self.latest_tx = btc.deserialize(tx)
-        for index, ins in enumerate(self.latest_tx['ins']):
-            utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
+        for index, ins in enumerate(self.latest_tx.vin):
+            utxo = (ins.prevout.hash[::-1], ins.prevout.n)
             if utxo not in self.input_utxos.keys():
                 continue
             # placeholders required
-            ins['script'] = 'deadbeef'
+            ins.scriptSig = btc.CScript.fromhex("deadbeef")
         self.taker_info_callback("INFO", "Built tx, sending to counterparties.")
-        return (True, list(self.maker_utxo_data.keys()), tx)
+        return (True, list(self.maker_utxo_data.keys()),
+                bintohex(self.latest_tx.serialize()))
 
+    @hexbin
     def auth_counterparty(self, btc_sig, auth_pub, maker_pk):
         """Validate the counterpartys claim to own the btc
         address/pubkey that will be used for coinjoining
         with an ecdsa verification.
         """
         try:
-            if not btc.ecdsa_verify(maker_pk, btc_sig, auth_pub):
+            # maker pubkey as message is in hex format:
+            if not btc.ecdsa_verify(bintohex(maker_pk), btc_sig, auth_pub):
                 jlog.debug('signature didnt match pubkey and message')
                 return False
         except Exception as e:
-            jlog.info("Failed ecdsa verify for maker pubkey: " + str(maker_pk))
+            jlog.info("Failed ecdsa verify for maker pubkey: " + bintohex(maker_pk))
             jlog.info("Exception was: " + repr(e))
             return False
         return True
 
     def on_sig(self, nick, sigb64):
         """Processes transaction signatures from counterparties.
-        Returns True if all signatures received correctly, else
-        returns False
+        If all signatures received correctly, returns the result
+        of self.self_sign_and_push() (i.e. we complete the signing
+        and broadcast); else returns False (thus returns False for
+        all but last signature).
         """
         if self.aborted:
             return False
         if nick not in self.nonrespondants:
             jlog.debug(('add_signature => nick={} '
                        'not in nonrespondants {}').format(nick, self.nonrespondants))
-            return
-        sig = hexlify(base64.b64decode(sigb64)).decode('ascii')
+            return False
+        sig = base64.b64decode(sigb64)
         inserted_sig = False
-        txhex = btc.serialize(self.latest_tx)
 
         # batch retrieval of utxo data
         utxo = {}
         ctr = 0
-        for index, ins in enumerate(self.latest_tx['ins']):
-            utxo_for_checking = ins['outpoint']['hash'] + ':' + str(ins[
-                'outpoint']['index'])
-             #'deadbeef' markers mean our own input scripts are not ''
-            if (ins['script'] != ''):
+        for index, ins in enumerate(self.latest_tx.vin):
+            utxo_for_checking = (ins.prevout.hash[::-1], ins.prevout.n)
+             # 'deadbeef' markers mean our own input scripts are not queried
+            if ins.scriptSig != b"":
                 continue
             utxo[ctr] = [index, utxo_for_checking]
             ctr += 1
         utxo_data = jm_single().bc_interface.query_utxo_set([x[
             1] for x in utxo.values()])
-
         # insert signatures
         for i, u in iteritems(utxo):
             if utxo_data[i] is None:
                 continue
-            #Check if the sender serialize_scripted the scriptCode
-            #item into the sig message; if so, also pick up the amount
-            #from the utxo data retrieved from the blockchain to verify
-            #the segwit-style signature. Note that this allows a mixed
-            #SW/non-SW transaction as each utxo is interpreted separately.
-            sig_deserialized = btc.deserialize_script(sig)
-            #verify_tx_input will not even parse the script if it has integers or None,
-            #so abort in case we were given a junk sig:
-            if not all([not isinstance(x, int) and x for x in sig_deserialized]):
-                jlog.warn("Junk signature: " + str(sig_deserialized) + \
+            # Check if the sender included the scriptCode in the sig message;
+            # if so, also pick up the amount from the utxo data retrieved
+            # from the blockchain to verify the segwit-style signature.
+            # Note that this allows a mixed SW/non-SW transaction as each utxo
+            # is interpreted separately.
+            try:
+                sig_deserialized = [a for a in iter(btc.CScript(sig))]
+            except Exception as e:
+                jlog.debug("Failed to parse junk sig message, ignoring.")
+                break
+            # abort in case we were given a junk sig (note this previously had
+            # to check to avoid crashes in verify_tx_input, no longer (Feb 2020)):
+            if not all([x for x in sig_deserialized]):
+                jlog.debug("Junk signature: " + str(sig_deserialized) + \
                           ", not attempting to verify")
                 break
             if len(sig_deserialized) == 2:
@@ -583,44 +584,48 @@ class Taker(object):
             elif len(sig_deserialized) == 3:
                 ver_sig, ver_pub, scriptCode =  sig_deserialized
             else:
-                jlog.debug("Invalid signature message - more than 3 items")
+                jlog.debug("Invalid signature message - not 2 or 3 items")
                 break
+
             ver_amt = utxo_data[i]['value'] if scriptCode else None
-            sig_good = btc.verify_tx_input(txhex, u[0], utxo_data[i]['script'],
-                            ver_sig, ver_pub, scriptCode=scriptCode, amount=ver_amt)
-
-            if ver_amt is not None and not sig_good:
-                # Special case to deal with legacy bots 0.5.0 or lower:
-                # the third field in the sigmessage was originally *not* the
-                # scriptCode, but the contents of tx['ins'][index]['script'],
-                # i.e. the witness program 0014... ; for this we can verify
-                # implicitly, as verify_tx_input used to, by reconstructing
-                # from the public key. For these cases, we can *assume* that
-                # the input is of type p2sh-p2wpkh; we call the jmbitcoin method
-                # directly, as we cannot assume that *our* wallet handles this.
-                scriptCode = hexlify(btc.pubkey_to_p2pkh_script(
-                    ver_pub, True)).decode('ascii')
-                sig_good = btc.verify_tx_input(txhex, u[0], utxo_data[i]['script'],
-                        ver_sig, ver_pub, scriptCode=scriptCode, amount=ver_amt)
+            witness = btc.CScriptWitness(
+                [ver_sig, ver_pub]) if scriptCode else None
+
+            # don't attempt to parse `pub` as pubkey unless it's valid.
+            if scriptCode:
+                try:
+                    s = btc.pubkey_to_p2wpkh_script(ver_pub)
+                except:
+                    jlog.debug("Junk signature message, invalid pubkey, ignoring.")
+                    break
+            scriptSig = btc.CScript([ver_sig, ver_pub]) if not scriptCode else btc.CScript([s])
+
+            # Pre-Feb 2020, we used the third field scriptCode differently in
+            # pre- and post-0.5.0; now the scriptCode is implicit (i.e. calculated
+            # by underlying library, so that exceptional case is covered.
+            sig_good = btc.verify_tx_input(self.latest_tx, u[0], scriptSig,
+                    btc.CScript(utxo_data[i]['script']), amount=ver_amt, witness=witness)
+
+            # verification for the native case is functionally identical but
+            # adds another flag; so we can allow it here:
+            if not sig_good:
+                sig_good = btc.verify_tx_input(self.latest_tx, u[0], scriptSig,
+                                    btc.CScript(utxo_data[i]['script']), amount=ver_amt,
+                                    witness=witness, native=True)
+                # if passes, below code executes, and we should change for native:
+                scriptSig = btc.CScript([b""])
 
             if sig_good:
                 jlog.debug('found good sig at index=%d' % (u[0]))
+
+                # Note that, due to the complexity of handling multisig or other
+                # arbitrary script (considering sending multiple signatures OTW),
+                # there is an assumption of p2sh-p2wpkh or p2wpkh, for the segwit
+                # case.
+                self.latest_tx.vin[u[0]].scriptSig = scriptSig
                 if ver_amt:
-                    # Note that, due to the complexity of handling multisig or other
-                    # arbitrary script (considering sending multiple signatures OTW),
-                    # there is an assumption of p2sh-p2wpkh or p2wpkh, for the segwit
-                    # case.
-                    self.latest_tx["ins"][u[0]]["txinwitness"] = [ver_sig, ver_pub]
-                    if btc.is_segwit_native_script(utxo_data[i]['script']):
-                        scriptSig = ""
-                    else:
-                        scriptSig = btc.serialize_script_unit(
-                            btc.pubkey_to_p2wpkh_script(ver_pub))
-                    self.latest_tx["ins"][u[0]]["script"] = scriptSig
-                else:
-                    # Non segwit (as per above comments) is limited only to single key,
-                    # p2pkh case.
-                    self.latest_tx["ins"][u[0]]["script"] = sig
+                    self.latest_tx.wit.vtxinwit[u[0]] = btc.CTxInWitness(
+                    btc.CScriptWitness(witness))
                 inserted_sig = True
 
                 # check if maker has sent everything possible
@@ -643,8 +648,8 @@ class Taker(object):
             # other guy sent a failed signature
 
         tx_signed = True
-        for ins in self.latest_tx['ins']:
-            if ins['script'] == '':
+        for ins in self.latest_tx.vin:
+            if ins.scriptSig == b"":
                 tx_signed = False
         if not tx_signed:
             return False
@@ -667,7 +672,7 @@ class Taker(object):
 
         def filter_by_coin_age_amt(utxos, age, amt):
             results = jm_single().bc_interface.query_utxo_set(utxos,
-                                                              includeconf=True)
+                                                    includeconf=True)
             newresults = []
             too_old = []
             too_small = []
@@ -683,7 +688,6 @@ class Taker(object):
                     too_small.append(utxos[i])
                 if valid_age and valid_amt:
                     newresults.append(utxos[i])
-
             return newresults, too_old, too_small
 
         def priv_utxo_pairs_from_utxos(utxos, age, amt):
@@ -695,7 +699,7 @@ class Taker(object):
                                                                    age, amt)
             new_utxos_dict = {k: v for k, v in utxos.items() if k in new_utxos}
             for k, v in iteritems(new_utxos_dict):
-                addr = v['address']
+                addr = self.wallet_service.script_to_addr(v["script"])
                 priv = self.wallet_service.get_key_from_addr(addr)
                 if priv:  #can be null from create-unsigned
                     priv_utxo_pairs.append((priv, k))
@@ -723,7 +727,7 @@ class Taker(object):
             #in the transaction, about to be consumed, rather than use
             #random utxos that will persist after. At this step we also
             #allow use of external utxos in the json file.
-            if any(self.wallet_service.get_utxos_by_mixdepth(hexfmt=False).values()):
+            if any(self.wallet_service.get_utxos_by_mixdepth().values()):
                 utxos = {}
                 for mdutxo in self.wallet_service.get_utxos_by_mixdepth().values():
                     utxos.update(mdutxo)
@@ -739,13 +743,9 @@ class Taker(object):
                 ext_valid = None
             podle_data = generate_podle(priv_utxo_pairs, tries, ext_valid)
         if podle_data:
-            jlog.debug("Generated PoDLE: " + pprint.pformat(podle_data))
-            revelation = PoDLE(u=podle_data['utxo'],
-                                   P=podle_data['P'],
-                                   P2=podle_data['P2'],
-                                   s=podle_data['sig'],
-                                   e=podle_data['e']).serialize_revelation()
-            return (commit_type_byte + podle_data["commit"], revelation,
+            jlog.debug("Generated PoDLE: " + repr(podle_data))
+            return (commit_type_byte + bintohex(podle_data.commitment),
+                    podle_data.serialize_revelation(),
                     "Commitment sourced OK")
         else:
             errmsgheader, errmsg = generate_podle_error_string(priv_utxo_pairs,
@@ -771,25 +771,26 @@ class Taker(object):
     def self_sign(self):
         # now sign it ourselves
         our_inputs = {}
-        for index, ins in enumerate(self.latest_tx['ins']):
-            utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
+        for index, ins in enumerate(self.latest_tx.vin):
+            utxo = (ins.prevout.hash[::-1], ins.prevout.n)
             if utxo not in self.input_utxos.keys():
                 continue
-            script = self.wallet_service.addr_to_script(self.input_utxos[utxo]['address'])
+            script = self.input_utxos[utxo]["script"]
             amount = self.input_utxos[utxo]['value']
             our_inputs[index] = (script, amount)
-        self.latest_tx = self.wallet_service.sign_tx(self.latest_tx, our_inputs)
+        success, msg = self.wallet_service.sign_tx(self.latest_tx, our_inputs)
+        if not success:
+            jlog.error("Failed to sign transaction: " + msg)
 
     def push(self):
-        tx = btc.serialize(self.latest_tx)
-        jlog.debug('\n' + tx)
-        self.txid = btc.txhash(tx)
+        jlog.debug('\n' + bintohex(self.latest_tx.serialize()))
+        self.txid = bintohex(self.latest_tx.GetTxid()[::-1])
         jlog.info('txid = ' + self.txid)
         #If we are sending to a bech32 address, in case of sweep, will
         #need to use that bech32 for address import, which requires
         #converting to script (Core does not allow import of bech32)
         if self.my_cj_addr.lower()[:2] in ['bc', 'tb']:
-            notify_addr = btc.address_to_script(self.my_cj_addr)
+            notify_addr = btc.CCoinAddress(self.my_cj_addr).to_scriptPubKey()
         else:
             notify_addr = self.my_cj_addr
         #add the callbacks *before* pushing to ensure triggering;
@@ -810,7 +811,7 @@ class Taker(object):
         tx_broadcast = jm_single().config.get('POLICY', 'tx_broadcast')
         nick_to_use = None
         if tx_broadcast == 'self':
-            pushed = jm_single().bc_interface.pushtx(tx)
+            pushed = jm_single().bc_interface.pushtx(self.latest_tx.serialize())
         elif tx_broadcast in ['random-peer', 'not-self']:
             n = len(self.maker_utxo_data)
             if tx_broadcast == 'random-peer':
@@ -818,14 +819,14 @@ class Taker(object):
             else:
                 i = random.randrange(n)
             if i == n:
-                pushed = jm_single().bc_interface.pushtx(tx)
+                pushed = jm_single().bc_interface.pushtx(self.latest_tx.serialize())
             else:
                 nick_to_use = list(self.maker_utxo_data.keys())[i]
                 pushed = True
         else:
             jlog.info("Only self, random-peer and not-self broadcast "
                       "methods supported. Reverting to self-broadcast.")
-            pushed = jm_single().bc_interface.pushtx(tx)
+            pushed = jm_single().bc_interface.pushtx(self.latest_tx.serialize())
         if not pushed:
             self.on_finished_callback(False, fromtx=True)
         else:
@@ -841,7 +842,7 @@ class Taker(object):
         # Takers process only in series, so this should not occur:
         assert self.latest_tx is not None
         # check if the transaction matches our created tx:
-        if txd['outs'] != self.latest_tx['outs']:
+        if txd.vout != self.latest_tx.vout:
             return False
         return True
 
@@ -928,12 +929,12 @@ class P2EPTaker(Taker):
         # For the p2ep taker, the variable 'my_cj_addr' is the destination:
         self.my_cj_addr = si[3]
         if isinstance(self.cjamount, float):
-            raise JMTakerError("P2EP coinjoin must use amount in satoshis")
+            raise JMTakerError("Payjoin must use amount in satoshis")
         if self.cjamount == 0:
             # Note that we don't allow sweep, currently, since the coin
             # choosing algo would not apply in that case (we'd have to rewrite
             # prepare_my_bitcoin_data for that case).
-            raise JMTakerError("P2EP coinjoin does not currently support sweep")
+            raise JMTakerError("Payjoin does not currently support sweep")
 
         # Next we prepare our coins with the inherited method
         # for this purpose; for this we must set the
@@ -1004,27 +1005,27 @@ class P2EPTaker(Taker):
         if self.my_change_addr is not None:
             self.outputs.append({'address': self.my_change_addr,
                                  'value': my_change_value})
-        # As for JM coinjoins, the `None` key is used for our own inputs
-        # to the transaction; this preparatory version contains only those.
-        tx = btc.make_shuffled_tx(self.utxos[None], self.outputs,
-                              False, 2, compute_tx_locktime())
-        jlog.info('Created proposed fallback tx\n' + pprint.pformat(
-            btc.deserialize(tx)))
+        # Oour own inputs to the transaction; this preparatory version
+        # contains only those.
+        tx = btc.make_shuffled_tx(self.input_utxos, self.outputs,
+                              version=2, locktime=compute_tx_locktime())
+        jlog.info('Created proposed fallback tx\n' + pprint.pformat(str(tx)))
         # We now sign as a courtesy, because if we disappear the recipient
         # can still claim his coins with this.
         # sign our inputs before transfer
         our_inputs = {}
-        dtx = btc.deserialize(tx)
-        for index, ins in enumerate(dtx['ins']):
-            utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
-            script = self.wallet_service.addr_to_script(self.input_utxos[utxo]['address'])
-            amount = self.input_utxos[utxo]['value']
-            our_inputs[index] = (script, amount)
-        self.signed_noncj_tx = btc.serialize(self.wallet_service.sign_tx(dtx, our_inputs))
+        for index, ins in enumerate(tx.vin):
+            utxo = (ins.prevout.hash[::-1], ins.prevout.n)
+            our_inputs[index] = (self.input_utxos[utxo]["script"],
+                                 self.input_utxos[utxo]['value'])
+        success, msg = self.wallet_service.sign_tx(tx, our_inputs)
+        if not success:
+            jlog.error("Failed to create backup transaction; error: " + msg)
         self.taker_info_callback("INFO", "Built tx proposal, sending to receiver.")
-        return (True, [self.p2ep_receiver_nick], self.signed_noncj_tx)
+        return (True, [self.p2ep_receiver_nick], bintohex(tx.serialize()))
 
-    def on_tx_received(self, nick, txhex):
+    @hexbin
+    def on_tx_received(self, nick, txser):
         """ Here the taker (payer) retrieves a version of the
         transaction from the maker (receiver) which should have
         the following properties:
@@ -1041,12 +1042,14 @@ class P2EPTaker(Taker):
         and then broadcast (TODO broadcast delay or don't broadcast).
         """
         try:
-            tx = btc.deserialize(txhex)
-        except (IndexError, SerializationError, SerializationTruncationError) as e:
+            tx = btc.CMutableTransaction.deserialize(txser)
+        except Exception as e:
             return (False, "malformed txhex. " + repr(e))
-        jlog.info("Obtained tx from receiver:\n" + pprint.pformat(tx))
-        cjaddr_script = btc.address_to_script(self.my_cj_addr)
-        changeaddr_script = btc.address_to_script(self.my_change_addr)
+        jlog.info("Obtained tx from receiver:\n" + pprint.pformat(str(tx)))
+        cjaddr_script = btc.CCoinAddress(
+            self.my_cj_addr).to_scriptPubKey()
+        changeaddr_script = btc.CCoinAddress(
+            self.my_change_addr).to_scriptPubKey()
 
         # We ensure that the coinjoin address and our expected change
         # address are still in the outputs, once (with the caveat that
@@ -1054,19 +1057,19 @@ class P2EPTaker(Taker):
         # of dust change, which we assess after).
         times_seen_cj_addr = 0
         times_seen_change_addr = 0
-        for outs in tx['outs']:
-            if outs['script'] == cjaddr_script:
+        for outs in tx.vout:
+            if outs.scriptPubKey == cjaddr_script:
                 times_seen_cj_addr += 1
-                new_cj_amount = outs['value']
+                new_cj_amount = outs.nValue
                 if new_cj_amount < self.cjamount:
                     # This is a violation of protocol;
                     # receiver must be providing extra bitcoin
                     # as input, so his receiving amount should have increased.
                     return (False,
                     'Wrong cj_amount. I expect at least' + str(self.cjamount))
-            if outs['script'] == changeaddr_script:
+            if outs.scriptPubKey == changeaddr_script:
                 times_seen_change_addr += 1
-                new_change_amount = outs['value']
+                new_change_amount = outs.nValue
         if times_seen_cj_addr != 1:
             fmt = ('cj addr not in tx outputs once, #cjaddr={}').format
             return (False, (fmt(times_seen_cj_addr)))
@@ -1077,8 +1080,7 @@ class P2EPTaker(Taker):
             new_change_amount = 0
 
         # Check that our inputs are present.
-        tx_utxo_set = set(ins['outpoint']['hash'] + ':' + str(
-            ins['outpoint']['index']) for ins in tx['ins'])
+        tx_utxo_set = set((ins.prevout.hash[::-1], ins.prevout.n) for ins in tx.vin)
         if not tx_utxo_set.issuperset(set(self.utxos[None])):
             return (False, "my utxos are not contained")
         # Check that the sequence numbers of all inputs are unaltered
@@ -1087,10 +1089,10 @@ class P2EPTaker(Taker):
         # Note that this is hacky and is most elegantly addressed by
         # use of PSBT (although any object encapsulation of tx input
         # would serve the same purpose).
-        if tx["locktime"] == 0:
+        if tx.nLockTime == 0:
             return (False, "Invalid PayJoin v0 transaction: locktime 0")
-        for i in tx["ins"]:
-            if i["sequence"] != 0xffffffff - 1:
+        for i in tx.vin:
+            if i.nSequence != 0xffffffff - 1:
                 return (False, "Invalid PayJoin v0 transaction: "+\
                         "sequence is not 0xffffffff -1")
 
@@ -1099,7 +1101,7 @@ class P2EPTaker(Taker):
         # not) of PayJoin to sweep utxos at no cost.
         # (TODO This is very kludgy, more sophisticated approach
         # should be used in future):
-        if len(tx["ins"]) - len (self.utxos[None]) > 5:
+        if len(tx.vin) - len (self.utxos[None]) > 5:
             return (False,
                     "proposed tx has more than 5 inputs from "
                     "the recipient, which is too expensive.")
@@ -1122,9 +1124,8 @@ class P2EPTaker(Taker):
         # checking input validity and transaction balance.
         retrieve_utxos = {}
         ctr = 0
-        for index, ins in enumerate(tx['ins']):
-            utxo_for_checking = ins['outpoint']['hash'] + ':' + str(
-                ins['outpoint']['index'])
+        for index, ins in enumerate(tx.vin):
+            utxo_for_checking = (ins.prevout.hash[::-1], ins.prevout.n)
             if utxo_for_checking in self.utxos[None]:
                 continue
             retrieve_utxos[ctr] = [index, utxo_for_checking]
@@ -1140,32 +1141,11 @@ class P2EPTaker(Taker):
             if utxo_data[i] is None:
                 return (False, "Proposed transaction contains invalid utxos")
             total_receiver_input += utxo_data[i]["value"]
-            scriptCode = None
-            ver_amt = None
             idx = retrieve_utxos[i][0]
-            if "txinwitness" in tx["ins"][idx]:
-                ver_amt = utxo_data[i]["value"]
-                try:
-                    ver_sig, ver_pub = tx["ins"][idx]["txinwitness"]
-                except Exception as e:
-                    print("Segwit error: ", repr(e))
-                    return (False, "Segwit input not of expected type, "
-                            "either p2sh-p2wpkh or p2wpkh")
-                # note that the scriptCode is the same whether nested or not
-                # also note that the scriptCode has to be inferred if we are
-                # only given a transaction serialization.
-                scriptCode = "76a914" + btc.hash160(unhexlify(ver_pub)) + "88ac"
-            else:
-                scriptSig = btc.deserialize_script(tx["ins"][idx]["script"])
-                if len(scriptSig) != 2:
-                    return (False,
-                    "Proposed transaction contains unsupported input type")
-                ver_sig, ver_pub = scriptSig
-            if not btc.verify_tx_input(txhex, idx,
-                                       utxo_data[i]['script'],
-                                       ver_sig, ver_pub,
-                                       scriptCode=scriptCode,
-                                       amount=ver_amt):
+            if not btc.verify_tx_input(tx, idx, tx.vin[idx].scriptSig,
+                                       btc.CScript(utxo_data[i]['script']),
+                                       amount=utxo_data[i]["value"],
+                                       witness=tx.wit.vtxinwit[idx].scriptWitness):
                 return (False,
                         "Proposed transaction is not correctly signed.")
         payment = new_cj_amount - total_receiver_input
@@ -1185,7 +1165,7 @@ class P2EPTaker(Taker):
         # our fee estimator. Its return value will be governed by our own fee settings
         # in joinmarket.cfg; allow either (a) automatic agreement for any value within
         # a range of 0.3 to 3x this figure, or (b) user to agree on prompt.
-        fee_est = estimate_tx_fee(len(tx['ins']), len(tx['outs']),
+        fee_est = estimate_tx_fee(len(tx.vin), len(tx.vout),
                                   txtype=self.wallet_service.get_txtype())
         fee_ok = False
         if btc_fee > 0.3 * fee_est and btc_fee < 3 * fee_est:

jmclient/jmclient/taker_utils.py

@@ -5,15 +5,13 @@ import os
 import sys
 import time
 import numbers
-
-from jmbase import get_log, jmprint
+from jmbase import get_log, jmprint, bintohex
 from .configure import jm_single, validate_address, is_burn_destination
 from .schedule import human_readable_schedule_entry, tweak_tumble_schedule,\
     schedule_to_text
 from .wallet import BaseWallet, estimate_tx_fee, compute_tx_locktime, \
     FidelityBondMixin
-from jmbitcoin import deserialize, make_shuffled_tx, serialize, txhash,\
-    amount_to_str, mk_burn_script, bin_hash160
+from jmbitcoin import make_shuffled_tx, amount_to_str, mk_burn_script
 from jmbase.support import EXIT_SUCCESS
 log = get_log()
 
@@ -136,13 +134,20 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False,
     log.info("Using a fee of : " + amount_to_str(fee_est) + ".")
     if amount != 0:
         log.info("Using a change value of: " + amount_to_str(changeval) + ".")
-    txsigned = sign_tx(wallet_service, make_shuffled_tx(
-        list(utxos.keys()), outs, False, 2, tx_locktime), utxos)
+    tx = make_shuffled_tx(list(utxos.keys()), outs, 2, compute_tx_locktime())
+        list(utxos.keys()), outs, 2, tx_locktime), utxos)
+    inscripts = {}
+    for i, txinp in enumerate(tx.vin):
+        u = (txinp.prevout.hash[::-1], txinp.prevout.n)
+        inscripts[i] = (utxos[u]["script"], utxos[u]["value"])
+    success, msg = wallet_service.sign_tx(tx, inscripts)
+    if not success:
+        log.error("Failed to sign transaction, quitting. Error msg: " + msg)
+        return
     log.info("Got signed transaction:\n")
-    log.info(pformat(txsigned))
-    tx = serialize(txsigned)
+    log.info(pformat(str(tx)))
     log.info("In serialized form (for copy-paste):")
-    log.info(tx)
+    log.info(bintohex(tx.serialize()))
     actual_amount = amount if amount != 0 else total_inputs_val - fee_est
     log.info("Sends: " + amount_to_str(actual_amount) + " to destination: " + destination)
     if not answeryes:
@@ -151,12 +156,12 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False,
                 log.info("You chose not to broadcast the transaction, quitting.")
                 return False
         else:
-            accepted = accept_callback(pformat(txsigned), destination, actual_amount,
+            accepted = accept_callback(pformat(str(tx)), destination, actual_amount,
                                        fee_est)
             if not accepted:
                 return False
-    jm_single().bc_interface.pushtx(tx)
-    txid = txhash(tx)
+    jm_single().bc_interface.pushtx(tx.serialize())
+    txid = bintohex(tx.GetTxid()[::-1])
     successmsg = "Transaction sent: " + txid
     cb = log.info if not info_callback else info_callback
     cb(successmsg)

jmclient/jmclient/wallet.py

@@ -27,7 +27,7 @@ from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH,\
 from .support import get_random_bytes
 from . import mn_encode, mn_decode
 import jmbitcoin as btc
-from jmbase import JM_WALLET_NAME_PREFIX
+from jmbase import JM_WALLET_NAME_PREFIX, bintohex
 
 
 """
@@ -198,7 +198,8 @@ class UTXOManager(object):
         assert isinstance(index, numbers.Integral)
         assert isinstance(mixdepth, numbers.Integral)
 
-        return self._utxo[mixdepth].pop((txid, index))
+        x = self._utxo[mixdepth].pop((txid, index))
+        return x
 
     def add_utxo(self, txid, index, path, value, mixdepth, height=None):
         # Assumed: that we add a utxo only if we want it enabled,
@@ -433,19 +434,22 @@ class BaseWallet(object):
         Add signatures to transaction for inputs referenced by scripts.
 
         args:
-            tx: transaction dict
+            tx: CMutableTransaction object
             scripts: {input_index: (output_script, amount)}
             kwargs: additional arguments for engine.sign_transaction
         returns:
-            input transaction dict with added signatures, hex-encoded.
+            True, None if success.
+            False, msg if signing failed, with error msg.
         """
         for index, (script, amount) in scripts.items():
             assert amount > 0
             path = self.script_to_path(script)
             privkey, engine = self._get_key_from_path(path)
-            tx = btc.deserialize(engine.sign_transaction(tx, index, privkey,
-                                                         amount, **kwargs))
-        return tx
+            sig, msg = engine.sign_transaction(tx, index, privkey,
+                                                         amount, **kwargs)
+            if not sig:
+                return False, msg
+        return True, None
 
     @deprecated
     def get_key_from_addr(self, addr):
@@ -455,7 +459,7 @@ class BaseWallet(object):
         script = self._ENGINE.address_to_script(addr)
         path = self.script_to_path(script)
         privkey = self._get_key_from_path(path)[0]
-        return hexlify(privkey).decode('ascii')
+        return privkey
 
     def _get_addr_int_ext(self, address_type, mixdepth):
         if address_type == self.ADDRESS_TYPE_EXTERNAL:
@@ -578,32 +582,19 @@ class BaseWallet(object):
         """
         self.save()
 
-    @deprecated
     def remove_old_utxos(self, tx):
-        tx = deepcopy(tx)
-        for inp in tx['ins']:
-            inp['outpoint']['hash'] = unhexlify(inp['outpoint']['hash'])
-
-        ret = self.remove_old_utxos_(tx)
-
-        removed_utxos = {}
-        for (txid, index), val in ret.items():
-            val['address'] = self.get_address_from_path(val['path'])
-            removed_utxos[hexlify(txid).decode('ascii') + ':' + str(index)] = val
-        return removed_utxos
-
-    def remove_old_utxos_(self, tx):
         """
         Remove all own inputs of tx from internal utxo list.
 
         args:
-            tx: transaction dict
+            tx: CMutableTransaction
         returns:
             {(txid, index): {'script': bytes, 'path': str, 'value': int} for all removed utxos
         """
         removed_utxos = {}
-        for inp in tx['ins']:
-            txid, index = inp['outpoint']['hash'], inp['outpoint']['index']
+        for inp in tx.vin:
+            txid = inp.prevout.hash[::-1]
+            index = inp.prevout.n
             md = self._utxos.have_utxo(txid, index)
             if md is False:
                 continue
@@ -614,46 +605,32 @@ class BaseWallet(object):
                                             'value': value}
         return removed_utxos
 
-    @deprecated
-    def add_new_utxos(self, tx, txid, height=None):
-        tx = deepcopy(tx)
-        for out in tx['outs']:
-            out['script'] = unhexlify(out['script'])
-
-        ret = self.add_new_utxos_(tx, unhexlify(txid), height=height)
-
-        added_utxos = {}
-        for (txid_bin, index), val in ret.items():
-            addr = self.get_address_from_path(val['path'])
-            val['address'] = addr
-            added_utxos[txid + ':' + str(index)] = val
-        return added_utxos
-
-    def add_new_utxos_(self, tx, txid, height=None):
+    def add_new_utxos(self, tx, height=None):
         """
         Add all outputs of tx for this wallet to internal utxo list.
-
+        They are also returned in standard dict form.
         args:
-            tx: transaction dict
+            tx: CMutableTransaction
             height: blockheight in which tx was included, or None
                     if unconfirmed.
         returns:
-            {(txid, index): {'script': bytes, 'path': tuple, 'value': int}
+            {(txid, index): {'script': bytes, 'path': tuple, 'value': int,
+            'address': str}
                 for all added utxos
         """
-        assert isinstance(txid, bytes) and len(txid) == self._utxos.TXID_LEN
         added_utxos = {}
-        for index, outs in enumerate(tx['outs']):
+        txid = tx.GetTxid()[::-1]
+        for index, outs in enumerate(tx.vout):
+            spk = outs.scriptPubKey
+            val = outs.nValue
             try:
-                self.add_utxo(txid, index, outs['script'], outs['value'],
-                              height=height)
+                self.add_utxo(txid, index, spk, val, height=height)
             except WalletError:
                 continue
 
-            path = self.script_to_path(outs['script'])
-            added_utxos[(txid, index)] = {'script': outs['script'],
-                                          'path': path,
-                                          'value': outs['value']}
+            path = self.script_to_path(spk)
+            added_utxos[(txid, index)] = {'script': spk, 'path': path, 'value': val,
+                'address': self._ENGINE.script_to_address(spk)}
         return added_utxos
 
     def add_utxo(self, txid, index, script, value, height=None):
@@ -669,9 +646,10 @@ class BaseWallet(object):
         mixdepth = self._get_mixdepth_from_path(path)
         self._utxos.add_utxo(txid, index, path, value, mixdepth, height=height)
 
-    def process_new_tx(self, txd, txid, height=None):
-        """ Given a newly seen transaction, deserialized as txd and
-        with transaction id, process its inputs and outputs and update
+    def process_new_tx(self, txd, height=None):
+        """ Given a newly seen transaction, deserialized as
+        CMutableTransaction txd,
+        process its inputs and outputs and update
         the utxo contents of this wallet accordingly.
         NOTE: this should correctly handle transactions that are not
         actually related to the wallet; it will not add (or remove,
@@ -679,30 +657,15 @@ class BaseWallet(object):
         functions check this condition.
         """
         removed_utxos = self.remove_old_utxos(txd)
-        added_utxos = self.add_new_utxos(txd, txid, height=height)
+        added_utxos = self.add_new_utxos(txd, height=height)
         return (removed_utxos, added_utxos)
 
-    @deprecated
-    def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None,
-                     maxheight=None):
-        utxo_filter_new = None
-        if utxo_filter:
-            utxo_filter_new = [(unhexlify(utxo[:64]), int(utxo[65:]))
-                               for utxo in utxo_filter]
-        ret = self.select_utxos_(mixdepth, amount, utxo_filter_new, select_fn,
-                                 maxheight=maxheight)
-        ret_conv = {}
-        for utxo, data in ret.items():
-            addr = self.get_address_from_path(data['path'])
-            utxo_txt = hexlify(utxo[0]).decode('ascii') + ':' + str(utxo[1])
-            ret_conv[utxo_txt] = {'address': addr, 'value': data['value']}
-        return ret_conv
-
-    def select_utxos_(self, mixdepth, amount, utxo_filter=None,
-                      select_fn=None, maxheight=None):
+    def select_utxos(self, mixdepth, amount, utxo_filter=None,
+                      select_fn=None, maxheight=None, includeaddr=False):
         """
         Select a subset of available UTXOS for a given mixdepth whose value is
-        greater or equal to amount.
+        greater or equal to amount. If `includeaddr` is True, adds an `address`
+        key to the returned dict.
 
         args:
             mixdepth: int, mixdepth to select utxos from, must be smaller or
@@ -713,6 +676,7 @@ class BaseWallet(object):
 
         returns:
             {(txid, index): {'script': bytes, 'path': tuple, 'value': int}}
+
         """
         assert isinstance(mixdepth, numbers.Integral)
         assert isinstance(amount, numbers.Integral)
@@ -728,7 +692,8 @@ class BaseWallet(object):
 
         for data in ret.values():
             data['script'] = self.get_script_from_path(data['path'])
-
+            if includeaddr:
+                data["address"] = self.get_address_from_path(data["path"])
         return ret
 
     def disable_utxo(self, txid, index, disable=True):
@@ -758,21 +723,7 @@ class BaseWallet(object):
                                             include_disabled=include_disabled,
                                             maxheight=maxheight)
 
-    @deprecated
-    def get_utxos_by_mixdepth(self, verbose=True, includeheight=False):
-        # TODO: verbose
-        ret = self.get_utxos_by_mixdepth_(includeheight=includeheight)
-
-        utxos_conv = collections.defaultdict(dict)
-        for md, utxos in ret.items():
-            for utxo, data in utxos.items():
-                utxo_str = hexlify(utxo[0]).decode('ascii') + ':' + str(utxo[1])
-                addr = self.get_address_from_path(data['path'])
-                data['address'] = addr
-                utxos_conv[md][utxo_str] = data
-        return utxos_conv
-
-    def get_utxos_by_mixdepth_(self, include_disabled=False, includeheight=False):
+    def get_utxos_by_mixdepth(self, include_disabled=False, includeheight=False):
         """
         Get all UTXOs for active mixdepths.
 
@@ -791,9 +742,11 @@ class BaseWallet(object):
                 if not include_disabled and self._utxos.is_disabled(*utxo):
                     continue
                 script = self.get_script_from_path(path)
+                addr = self.get_address_from_path(path)
                 script_utxos[md][utxo] = {'script': script,
                                           'path': path,
-                                          'value': value}
+                                          'value': value,
+                                          'address': addr}
                 if includeheight:
                     script_utxos[md][utxo]['height'] = height
         return script_utxos
@@ -1101,7 +1054,6 @@ class ImportWalletMixin(object):
             raise WalletError("Unsupported key type for imported keys.")
 
         privkey, key_type_wif = self._ENGINE.wif_to_privkey(wif)
-
         # FIXME: there is no established standard for encoding key type in wif
         #if key_type is not None and key_type_wif is not None and \
         #        key_type != key_type_wif:

jmclient/jmclient/wallet_service.py

@@ -16,8 +16,9 @@ from jmclient.output import fmt_tx_data
 from jmclient.blockchaininterface import (INF_HEIGHT, BitcoinCoreInterface,
     BitcoinCoreNoHistoryInterface)
 from jmclient.wallet import FidelityBondMixin
-from jmbase.support import jmprint, EXIT_SUCCESS
-import jmbitcoin as btc
+from jmbase.support import jmprint, EXIT_SUCCESS, utxo_to_utxostr, bintohex, hextobin
+from jmbitcoin import lx
+
 """Wallet service
 
 The purpose of this independent service is to allow
@@ -173,9 +174,9 @@ class WalletService(Service):
         and return type boolean.
         `txinfo` - either a txid expected for the transaction, if
         known, or a tuple of the ordered output set, of the form
-        (('script': script), ('value': value), ..). This can be
-        constructed from jmbitcoin.deserialize output, key "outs",
-        using tuple(). See WalletService.transaction_monitor().
+        ((CScript, int), ..). This is be constructed from the
+        CMutableTransaction vout list.
+        See WalletService.transaction_monitor().
         `cb_type` - must be one of "all", "unconfirmed", "confirmed";
         the first type will be called back once for every new
         transaction, the second only once when the number of
@@ -223,7 +224,9 @@ class WalletService(Service):
             self.bci.import_addresses([address], self.EXTERNAL_WALLET_LABEL,
                                   restart_cb=self.restart_callback)
 
-    def default_autofreeze_warning_cb(self, utxostr):
+    def default_autofreeze_warning_cb(self, utxo):
+        success, utxostr = utxo_to_utxostr(utxo)
+        assert success, "Autofreeze warning cb called with invalid utxo."
         jlog.warning("WARNING: new utxo has been automatically "
              "frozen to prevent forced address reuse: ")
         jlog.warning(utxostr)
@@ -267,9 +270,7 @@ class WalletService(Service):
                 utxo]["value"] <= freeze_threshold:
                 # freezing of coins must be communicated to user:
                 self.autofreeze_warning_cb(utxo)
-                # process_new_tx returns added utxos in str format:
-                txidstr, idx = utxo.split(":")
-                self.disable_utxo(binascii.unhexlify(txidstr), int(idx))
+                self.disable_utxo(*utxo)
 
     def transaction_monitor(self):
         """Keeps track of any changes in the wallet (new transactions).
@@ -298,7 +299,7 @@ class WalletService(Service):
 
         for tx in new_txs:
             txid = tx["txid"]
-            res = self.bci.get_transaction(txid)
+            res = self.bci.get_transaction(hextobin(txid))
             if not res:
                 continue
             confs = res["confirmations"]
@@ -317,7 +318,7 @@ class WalletService(Service):
             txd = self.bci.get_deser_from_gettransaction(res)
             if txd is None:
                 continue
-            removed_utxos, added_utxos = self.wallet.process_new_tx(txd, txid, height)
+            removed_utxos, added_utxos = self.wallet.process_new_tx(txd, height)
             if txid not in self.processed_txids:
                 # apply checks to disable/freeze utxos to reused addrs if needed:
                 self.check_for_reuse(added_utxos)
@@ -339,16 +340,9 @@ class WalletService(Service):
                     f(txd, txid)
 
             # The tuple given as the second possible key for the dict
-            # is such because dict keys must be hashable types, so a simple
-            # replication of the entries in the list tx["outs"], where tx
-            # was generated via jmbitcoin.deserialize, is unacceptable to
-            # Python, since they are dicts. However their keyset is deterministic
-            # so it is sufficient to convert these dicts to tuples with fixed
-            # ordering, thus it can be used as a key into the self.callbacks
-            # dicts. (This is needed because txid is not always available
+            # is such because txid is not always available
             # at the time of callback registration).
-            possible_keys = [txid, tuple(
-                    (x["script"], x["value"]) for x in txd["outs"])]
+            possible_keys = [txid, tuple((x.scriptPubKey, x.nValue) for x in txd.vout)]
 
             # note that len(added_utxos) > 0 is not a sufficient condition for
             # the tx being new, since wallet.add_new_utxos will happily re-add
@@ -406,8 +400,9 @@ class WalletService(Service):
         def report_changed(x, utxos):
             if len(utxos.keys()) > 0:
                 jlog.info(x + ' utxos=\n{}'.format('\n'.join(
-                    '{} - {}'.format(u, fmt_tx_data(tx_data, self))
-                    for u, tx_data in utxos.items())))
+                    '{} - {}'.format(utxo_to_utxostr(u)[1],
+                        fmt_tx_data(tx_data, self)) for u,
+                    tx_data in utxos.items())))
 
         report_changed("Removed", removed_utxos)
         report_changed("Added", added_utxos)
@@ -752,14 +747,15 @@ class WalletService(Service):
     def _add_unspent_txo(self, utxo, height):
         """
         Add a UTXO as returned by rpc's listunspent call to the wallet.
-
+        Note that these are returned as little endian outpoint txids, so
+        are converted.
         params:
             utxo: single utxo dict as returned by listunspent
             current_blockheight: blockheight as integer, used to
             set the block in which a confirmed utxo is included.
         """
-        txid = binascii.unhexlify(utxo['txid'])
-        script = binascii.unhexlify(utxo['scriptPubKey'])
+        txid = hextobin(utxo['txid'])
+        script = hextobin(utxo['scriptPubKey'])
         value = int(Decimal(str(utxo['amount'])) * Decimal('1e8'))
         self.add_utxo(txid, int(utxo['vout']), script, value, height)
 
@@ -774,11 +770,9 @@ class WalletService(Service):
         self.wallet.save()
 
     def get_utxos_by_mixdepth(self, include_disabled=False,
-                              verbose=False, hexfmt=True, includeconfs=False):
+                              verbose=False, includeconfs=False):
         """ Returns utxos by mixdepth in a dict, optionally including
         information about how many confirmations each utxo has.
-        TODO clean up underlying wallet.get_utxos_by_mixdepth (including verbosity
-        and formatting options) to make this less confusing.
         """
         def height_to_confs(x):
             # convert height entries to confirmations:
@@ -793,36 +787,30 @@ class WalletService(Service):
                         confs = self.current_blockheight - h + 1
                     ubym_conv[m][u]["confs"] = confs
             return ubym_conv
+        ubym = self.wallet.get_utxos_by_mixdepth(
+            include_disabled=include_disabled, includeheight=includeconfs)
+        if not includeconfs:
+            return ubym
+        else:
+            return height_to_confs(ubym)
 
-        if hexfmt:
-            ubym = self.wallet.get_utxos_by_mixdepth(verbose=verbose,
-                                                     includeheight=includeconfs)
-            if not includeconfs:
-                return ubym
-            else:
-                return height_to_confs(ubym)
+    def minconfs_to_maxheight(self, minconfs):
+        if minconfs is None:
+            return None
         else:
-            ubym = self.wallet.get_utxos_by_mixdepth_(
-            include_disabled=include_disabled, includeheight=includeconfs)
-            if not includeconfs:
-                return ubym
-            else:
-                return height_to_confs(ubym)
+            return self.current_blockheight - minconfs + 1
 
     def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None,
-                     minconfs=None):
+                     minconfs=None, includeaddr=False):
         """ Request utxos from the wallet in a particular mixdepth to satisfy
         a certain total amount, optionally set the selector function (or use
         the currently configured function set by the wallet, and optionally
         require a minimum of minconfs confirmations (default none means
         unconfirmed are allowed).
         """
-        if minconfs is None:
-            maxheight = None
-        else:
-            maxheight = self.current_blockheight - minconfs + 1
         return self.wallet.select_utxos(mixdepth, amount, utxo_filter=utxo_filter,
-                                    select_fn=select_fn, maxheight=maxheight)
+                select_fn=select_fn, maxheight=self.minconfs_to_maxheight(minconfs),
+                includeaddr=includeaddr)
 
     def get_balance_by_mixdepth(self, verbose=True,
                                 include_disabled=False,

jmclient/jmclient/wallet_utils.py

@@ -16,7 +16,8 @@ from jmclient import (get_network, WALLET_IMPLEMENTATIONS, Storage, podle,
     LegacyWallet, SegwitWallet, FidelityBondMixin, FidelityBondWatchonlyWallet,
     is_native_segwit_mode, load_program_config, add_base_options, check_regtest)
 from jmclient.wallet_service import WalletService
-from jmbase.support import get_password, jmprint, EXIT_FAILURE, EXIT_ARGERROR
+from jmbase.support import (get_password, jmprint, EXIT_FAILURE,
+                            EXIT_ARGERROR, utxo_to_utxostr)
 
 from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH, \
     TYPE_SEGWIT_LEGACY_WALLET_FIDELITY_BONDS
@@ -369,22 +370,26 @@ def wallet_showutxos(wallet, showprivkey):
     utxos = wallet.get_utxos_by_mixdepth(includeconfs=True)
     for md in utxos:
         for u, av in utxos[md].items():
+            success, us = utxo_to_utxostr(u)
+            assert success
             key = wallet.get_key_from_addr(av['address'])
             tries = podle.get_podle_tries(u, key, max_tries)
             tries_remaining = max(0, max_tries - tries)
-            unsp[u] = {'address': av['address'], 'value': av['value'],
+            unsp[us] = {'address': av['address'], 'value': av['value'],
                        'tries': tries, 'tries_remaining': tries_remaining,
                        'external': False,
                        'confirmations': av['confs']}
             if showprivkey:
-                unsp[u]['privkey'] = wallet.get_wif_path(av['path'])
+                unsp[us]['privkey'] = wallet.get_wif_path(av['path'])
 
     used_commitments, external_commitments = podle.get_podle_commitments()
     for u, ec in iteritems(external_commitments):
+        success, us = utxo_to_utxostr(u)
+        assert success
         tries = podle.get_podle_tries(utxo=u, max_tries=max_tries,
                                           external=True)
         tries_remaining = max(0, max_tries - tries)
-        unsp[u] = {'tries': tries, 'tries_remaining': tries_remaining,
+        unsp[us] = {'tries': tries, 'tries_remaining': tries_remaining,
                    'external': True}
 
     return json.dumps(unsp, indent=4)
@@ -431,7 +436,7 @@ def wallet_display(wallet_service, showprivkey, displayall=False,
     acctlist = []
     # TODO - either optionally not show disabled utxos, or
     # mark them differently in display (labels; colors)
-    utxos = wallet_service.get_utxos_by_mixdepth(include_disabled=True, hexfmt=False)
+    utxos = wallet_service.get_utxos_by_mixdepth(include_disabled=True)
     for m in range(wallet_service.mixdepth + 1):
         branchlist = []
         for address_type in [0, 1]:

jmclient/jmclient/yieldgenerator.py

@@ -132,7 +132,8 @@ class YieldGeneratorBasic(YieldGenerator):
 
         change_addr = self.wallet_service.get_internal_addr(mixdepth)
 
-        utxos = self.wallet_service.select_utxos(mixdepth, total_amount, minconfs=1)
+        utxos = self.wallet_service.select_utxos(mixdepth, total_amount,
+                                        minconfs=1, includeaddr=True)
         my_total_in = sum([va['value'] for va in utxos.values()])
         real_cjfee = calc_cj_fee(offer["ordertype"], offer["cjfee"], amount)
         change_value = my_total_in - amount - offer["txfee"] + real_cjfee
@@ -141,7 +142,8 @@ class YieldGeneratorBasic(YieldGenerator):
                        'finding new utxos').format(change_value))
             try:
                 utxos = self.wallet_service.select_utxos(mixdepth,
-                    total_amount + jm_single().DUST_THRESHOLD, minconfs=1)
+                    total_amount + jm_single().DUST_THRESHOLD,
+                    minconfs=1, includeaddr=True)
             except Exception:
                 jlog.info('dont have the required UTXOs to make a '
                           'output above the dust threshold, quitting')

jmclient/test/commontest.py

@@ -6,11 +6,13 @@ import binascii
 import random
 from decimal import Decimal
 
-from jmbase import get_log
+from jmbase import (get_log, hextobin, utxostr_to_utxo,
+                    utxo_to_utxostr, listchanger, dictchanger)
+
 from jmclient import (
     jm_single, open_test_wallet_maybe, estimate_tx_fee,
     BlockchainInterface, get_p2sh_vbyte, BIP32Wallet,
-    SegwitLegacyWallet, WalletService)
+    SegwitLegacyWallet, WalletService, BTC_P2SH_P2WPKH)
 from jmbase.support import chunks
 import jmbitcoin as btc
 
@@ -45,9 +47,11 @@ class DummyBlockchainInterface(BlockchainInterface):
         pass
     def is_address_imported(self, addr):
         pass
-    
+
+    def get_current_block_height(self):
+        return 10**6
+
     def pushtx(self, txhex):
-        print("pushing: " + str(txhex))
         return True
     
     def insert_fake_query_results(self, fqr):
@@ -56,7 +60,7 @@ class DummyBlockchainInterface(BlockchainInterface):
     def setQUSFail(self, state):
         self.qusfail = state
     
-    def query_utxo_set(self, txouts,includeconf=False):
+    def query_utxo_set(self, txouts, includeconf=False):
         if self.qusfail:
             #simulate failure to find the utxo
             return [None]
@@ -72,12 +76,14 @@ class DummyBlockchainInterface(BlockchainInterface):
         known_outs = {"03243f4a659e278a1333f8308f6aaf32db4692ee7df0340202750fd6c09150f6:1": "03a2d1cbe977b1feaf8d0d5cc28c686859563d1520b28018be0c2661cf1ebe4857",
                       "498faa8b22534f3b443c6b0ce202f31e12f21668b4f0c7a005146808f250d4c3:0": "02b4b749d54e96b04066b0803e372a43d6ffa16e75a001ae0ed4b235674ab286be",
                       "3f3ea820d706e08ad8dc1d2c392c98facb1b067ae4c671043ae9461057bd2a3c:1": "023bcbafb4f68455e0d1d117c178b0e82a84e66414f0987453d78da034b299c3a9"}
+        known_outs = dictchanger(known_outs)
         #our wallet utxos, faked, for podle tests: utxos are doctored (leading 'f'),
         #and the lists are (amt, age)
         wallet_outs = {'f34b635ed8891f16c4ec5b8236ae86164783903e8e8bb47fa9ef2ca31f3c2d7a:0': [10000000, 2],
                        'f780d6e5e381bff01a3519997bb4fcba002493103a198fde334fd264f9835d75:1': [20000000, 6],
                        'fe574db96a4d43a99786b3ea653cda9e4388f377848f489332577e018380cff1:0': [50000000, 3],
                        'fd9711a2ef340750db21efb761f5f7d665d94b312332dc354e252c77e9c48349:0': [50000000, 6]}
+        wallet_outs = dictchanger(wallet_outs)
         
         if includeconf and set(txouts).issubset(set(wallet_outs)):
             #includeconf used as a trigger for a podle check;
@@ -88,16 +94,16 @@ class DummyBlockchainInterface(BlockchainInterface):
                                 'confirms': wallet_outs[to][1]})
             return results
         if txouts[0] in known_outs:
-            addr = btc.pubkey_to_p2sh_p2wpkh_address(
-                        known_outs[txouts[0]], get_p2sh_vbyte())
+            scr = BTC_P2SH_P2WPKH.pubkey_to_script(known_outs[txouts[0]])
+            addr = btc.CCoinAddress.from_scriptPubKey(scr)
             return [{'value': 200000000,
                      'address': addr,
-                     'script': btc.address_to_script(addr),
+                     'script': scr,
                      'confirms': 20}]
         for t in txouts:
             result_dict = {'value': 10000000000,
                            'address': "mrcNu71ztWjAQA6ww9kHiW3zBWSQidHXTQ",
-                           'script': '76a91479b000887626b294a914501a4cd226b58b23598388ac'}
+                           'script': hextobin('76a91479b000887626b294a914501a4cd226b58b23598388ac')}
             if includeconf:
                 result_dict['confirms'] = 20
             result.append(result_dict)        
@@ -110,18 +116,10 @@ class DummyBlockchainInterface(BlockchainInterface):
 def create_wallet_for_sync(wallet_structure, a, **kwargs):
     #We need a distinct seed for each run so as not to step over each other;
     #make it through a deterministic hash
-    seedh = btc.sha256("".join([str(x) for x in a]))[:32]
+    seedh = btc.b2x(btc.Hash("".join([str(x) for x in a]).encode("utf-8")))[:32]
     return make_wallets(
         1, [wallet_structure], fixed_seeds=[seedh], **kwargs)[0]['wallet']
 
-
-def binarize_tx(tx):
-    for o in tx['outs']:
-        o['script'] = binascii.unhexlify(o['script'])
-    for i in tx['ins']:
-        i['outpoint']['hash'] = binascii.unhexlify(i['outpoint']['hash'])
-
-
 def make_sign_and_push(ins_full,
                        wallet_service,
                        amount,
@@ -130,7 +128,12 @@ def make_sign_and_push(ins_full,
                        hashcode=btc.SIGHASH_ALL,
                        estimate_fee = False):
     """Utility function for easily building transactions
-    from wallets
+    from wallets.
+    `ins_full` should be a list of dicts in format returned
+    by wallet.select_utxos:
+    {(txid, index): {"script":..,"value":..,"path":..}}
+    ... although the path is not used.
+    The "script" and "value" data is used to allow signing.
     """
     assert isinstance(wallet_service, WalletService)
     total = sum(x['value'] for x in ins_full.values())
@@ -143,22 +146,21 @@ def make_sign_and_push(ins_full,
              'address': output_addr}, {'value': total - amount - fee_est,
                                        'address': change_addr}]
 
-    de_tx = btc.deserialize(btc.mktx(ins, outs))
+    tx = btc.mktx(ins, outs)
     scripts = {}
-    for index, ins in enumerate(de_tx['ins']):
-        utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
-        script = wallet_service.addr_to_script(ins_full[utxo]['address'])
-        scripts[index] = (script, ins_full[utxo]['value'])
-    binarize_tx(de_tx)
-    de_tx = wallet_service.sign_tx(de_tx, scripts, hashcode=hashcode)
+    for i, j in enumerate(ins):
+        scripts[i] = (ins_full[j]["script"], ins_full[j]["value"])
+
+    success, msg = wallet_service.sign_tx(tx, scripts, hashcode=hashcode)
+    if not success:
+        return False
     #pushtx returns False on any error
-    push_succeed = jm_single().bc_interface.pushtx(btc.serialize(de_tx))
+    push_succeed = jm_single().bc_interface.pushtx(tx.serialize())
     if push_succeed:
-        txid = btc.txhash(btc.serialize(de_tx))
         # in normal operation this happens automatically
         # but in some tests there is no monitoring loop:
-        wallet_service.process_new_tx(de_tx, txid)
-        return txid
+        wallet_service.process_new_tx(tx)
+        return tx.GetTxid()[::-1]
     else:
         return False
 

jmclient/test/test_client_protocol.py

@@ -1,7 +1,8 @@
 #! /usr/bin/env python
 '''test client-protocol interfacae.'''
 
-from jmbase import get_log
+from jmbase import get_log, bintohex, hextobin
+from jmbase.commands import *
 from jmclient import load_test_config, Taker,\
     JMClientProtocolFactory, jm_single, Maker, WalletService
 from jmclient.client_protocol import JMTakerClientProtocol
@@ -14,12 +15,10 @@ from twisted.protocols.amp import UnknownRemoteError
 from twisted.protocols import amp
 from twisted.trial import unittest
 from twisted.test import proto_helpers
-from jmbase.commands import *
 from taker_test_data import t_raw_signed_tx
 from commontest import default_max_cj_fee
 import json
 import jmbitcoin as bitcoin
-
 import twisted
 twisted.internet.base.DelayedCall.debug = True
 
@@ -97,7 +96,7 @@ class DummyMaker(Maker):
 
     def on_auth_received(self, nick, offer, commitment, cr, amount, kphex):
         # success, utxos, auth_pub, cj_addr, change_addr, btc_sig
-        return True, [], '', '', '', ''
+        return True, [], b"", '', '', ''
 
     def on_tx_received(self, nick, txhex, offerinfo):
         # success, sigs
@@ -227,8 +226,8 @@ class JMTestServerProtocol(JMBaseProtocol):
         self.defaultCallbacks(d2)
         #To test, this must include a valid ecdsa sig
         fullmsg = "fullmsgforverify"
-        priv = "aa"*32 + "01"
-        pub = bitcoin.privkey_to_pubkey(priv)
+        priv = b"\xaa"*32 + b"\x01"
+        pub = bintohex(bitcoin.privkey_to_pubkey(priv))
         sig = bitcoin.ecdsa_sign(fullmsg, priv)
         d3 = self.callRemote(JMRequestMsgSigVerify,
                                         msg="msgforverify",
@@ -262,7 +261,7 @@ class JMTestServerProtocolFactory(protocol.ServerFactory):
 
 class DummyClientProtocolFactory(JMClientProtocolFactory):
     def buildProtocol(self, addr):
-        return JMTakerClientProtocol(self, self.client, nick_priv="aa"*32)
+        return JMTakerClientProtocol(self, self.client, nick_priv=b"\xaa"*32 + b"\x01")
 
 
 class TrialTestJMClientProto(unittest.TestCase):
@@ -364,8 +363,8 @@ class TestMakerClientProtocol(unittest.TestCase):
     @inlineCallbacks
     def test_JMRequestMsgSigVerify(self):
         fullmsg = 'fullmsgforverify'
-        priv = 'aa'*32 + '01'
-        pub = bitcoin.privkey_to_pubkey(priv)
+        priv = b"\xaa"*32 + b"\x01"
+        pub = bintohex(bitcoin.privkey_to_pubkey(priv))
         sig = bitcoin.ecdsa_sign(fullmsg, priv)
         yield self.init_client()
         yield self.callClient(

jmclient/test/test_coinjoin.py

@@ -6,14 +6,15 @@ Test doing full coinjoins, bypassing IRC
 import os
 import sys
 import pytest
+import copy
 from twisted.internet import reactor
 
-from jmbase import get_log
+from jmbase import get_log, hextobin, bintohex
 from jmclient import load_test_config, jm_single,\
     YieldGeneratorBasic, Taker, LegacyWallet, SegwitLegacyWallet,\
     NO_ROUNDING
 from jmclient.podle import set_commitment_file
-from commontest import make_wallets, binarize_tx, default_max_cj_fee
+from commontest import make_wallets, default_max_cj_fee
 from test_taker import dummy_filter_orderbook
 import jmbitcoin as btc
 
@@ -85,14 +86,14 @@ def init_coinjoin(taker, makers, orderbook, cj_amount):
         ioauth_data = list(response[1:])
         ioauth_data[0] = list(ioauth_data[0].keys())
         # maker_pk which is set up by jmdaemon
-        ioauth_data.append(None)
+        ioauth_data.append("00")
         maker_data[mid] = ioauth_data
 
         # this is handled by jmdaemon
         active_orders[mid]['utxos'] = response[1]
         active_orders[mid]['cjaddr'] = ioauth_data[2]
         active_orders[mid]['changeaddr'] = ioauth_data[3]
-        active_orders[mid]['offer'] = m.offerlist[0]
+        active_orders[mid]['offer'] = copy.deepcopy(m.offerlist[0])
         active_orders[mid]['amount'] = cj_amount
     return active_orders, maker_data
 
@@ -194,13 +195,12 @@ def test_coinjoin_mixdepth_wrap_taker(monkeypatch, tmpdir, setup_cj):
     taker_final_result = do_tx_signing(taker, makers, active_orders, txdata)
     assert taker_final_result is not False
 
-    tx = btc.deserialize(txdata[2])
-    binarize_tx(tx)
+    tx = btc.CMutableTransaction.deserialize(hextobin(txdata[2]))
 
     wallet_service = wallet_services[-1]
     # TODO change for new tx monitoring:
-    wallet_service.remove_old_utxos_(tx)
-    wallet_service.add_new_utxos_(tx, b'\x00' * 32)  # fake txid
+    wallet_service.remove_old_utxos(tx)
+    wallet_service.add_new_utxos(tx)
 
     balances = wallet_service.get_balance_by_mixdepth()
     assert balances[0] == cj_amount
@@ -250,14 +250,13 @@ def test_coinjoin_mixdepth_wrap_maker(monkeypatch, tmpdir, setup_cj):
     taker_final_result = do_tx_signing(taker, makers, active_orders, txdata)
     assert taker_final_result is not False
 
-    tx = btc.deserialize(txdata[2])
-    binarize_tx(tx)
+    tx = btc.CMutableTransaction.deserialize(hextobin(txdata[2]))
 
     for i in range(MAKER_NUM):
         wallet_service = wallet_services[i]
         # TODO as above re: monitoring
-        wallet_service.remove_old_utxos_(tx)
-        wallet_service.add_new_utxos_(tx, b'\x00' * 32)  # fake txid
+        wallet_service.remove_old_utxos(tx)
+        wallet_service.add_new_utxos(tx)
 
         balances = wallet_service.get_balance_by_mixdepth()
         assert balances[0] == cj_amount

jmclient/test/test_commitment_utils.py

@@ -4,11 +4,14 @@ import pytest
 
 from jmclient import (load_test_config, jm_single)
 from jmclient.commitment_utils import get_utxo_info, validate_utxo_data
-
+from jmbitcoin import select_chain_params
 
 def test_get_utxo_info():
     load_test_config()
+    # this test tests mainnet keys, so temporarily switch network
+    select_chain_params("bitcoin")
     jm_single().config.set("BLOCKCHAIN", "network", "mainnet")
+
     dbci = DummyBlockchainInterface()
     privkey = "L1RrrnXkcKut5DEMwtDthjwRcTTwED36thyL1DebVrKuwvohjMNi"
     #to verify use from_wif_privkey and privkey_to_address
@@ -28,7 +31,7 @@ def test_get_utxo_info():
     with pytest.raises(Exception) as e_info:
         u, priv = get_utxo_info(fakeutxo + privkey)
     #invalid index
-    fu2 = "ab"*32 + ":00004"
+    fu2 = "ab"*32 + ":-1"
     with pytest.raises(Exception) as e_info:
         u, priv = get_utxo_info(fu2 + "," + privkey)
     #invalid privkey
@@ -53,3 +56,5 @@ def test_get_utxo_info():
     retval = validate_utxo_data(utxodatas, False)
     assert not retval
     dbci.setQUSFail(False)
+    select_chain_params("bitcoin/regtest")
+    jm_single().config.set("BLOCKCHAIN", "network", "mainnet")

jmclient/test/test_maker.py

@@ -29,8 +29,7 @@ def construct_tx_offerlist(cjaddr, changeaddr, maker_utxos, maker_utxos_value,
         'txfee': 0
     }
 
-    utxos = { utxo['outpoint']['hash'] + ':' + str(utxo['outpoint']['index']):
-                {'utxo': utxo, 'value': maker_utxos_value} for utxo in maker_utxos }
+    utxos = { utxo: {'utxo': utxo, 'value': maker_utxos_value} for utxo in maker_utxos }
 
     offerlist = {
         'utxos': utxos,
@@ -46,59 +45,56 @@ def construct_tx_offerlist(cjaddr, changeaddr, maker_utxos, maker_utxos_value,
 def create_tx_inputs(count=1):
     inp = []
     for i in range(count):
-        inp.append({'outpoint': {'hash': '0'*64, 'index': i},
-                    'script': '',
-                    'sequence': 4294967295})
+        inp.append((b"\x00"*32, i))
     return inp
 
 
-def create_tx_outputs(*scripts_amount):
+def create_tx_outputs(*addrs_amount):
     outp = []
-    for script, amount in scripts_amount:
-        outp.append({'script': script, 'value': amount})
+    for addr, amount in addrs_amount:
+        outp.append({'address': addr, 'value': amount})
     return outp
 
 
 def address_p2pkh_generator():
-    return get_address_generator(b'\x76\xa9\x14', b'\x88\xac', get_p2pk_vbyte())
+    return get_address_generator(b'\x76\xa9\x14', b'\x88\xac')
 
 
 def address_p2sh_generator():
-    return get_address_generator(b'\xa9\x14', b'\x87', get_p2sh_vbyte())
+    return get_address_generator(b'\xa9\x14', b'\x87', p2sh=True)
 
 
-def get_address_generator(script_pre, script_post, vbyte):
+def get_address_generator(script_pre, script_post, p2sh=False):
     counter = 0
     while True:
         script = script_pre + struct.pack(b'=LQQ', 0, 0, counter) + script_post
-        addr = btc.script_to_address(script, vbyte)
-        yield addr, binascii.hexlify(script).decode('ascii')
+        if p2sh:
+            addr = btc.CCoinAddress.from_scriptPubKey(
+                btc.CScript(script).to_p2sh_scriptPubKey())
+        else:
+            addr = btc.CCoinAddress.from_scriptPubKey(btc.CScript(script))
+        yield str(addr), binascii.hexlify(script).decode('ascii')
         counter += 1
 
 
-def create_tx_and_offerlist(cj_addr, cj_change_addr, other_output_scripts,
-                            cj_script=None, cj_change_script=None, offertype='swreloffer'):
-    assert len(other_output_scripts) % 2 == 0, "bug in test"
+def create_tx_and_offerlist(cj_addr, cj_change_addr, other_output_addrs,
+                            offertype='swreloffer'):
+    assert len(other_output_addrs) % 2 == 0, "bug in test"
 
     cj_value = 100000000
     maker_total_value = cj_value*3
 
-    if cj_script is None:
-        cj_script = btc.address_to_script(cj_addr)
-    if cj_change_script is None:
-        cj_change_script = btc.address_to_script(cj_change_addr)
-
     inputs = create_tx_inputs(3)
     outputs = create_tx_outputs(
-        (cj_script, cj_value),
-        (cj_change_script, maker_total_value - cj_value),  # cjfee=0, txfee=0
-        *((script, cj_value + (i%2)*(50000000+i)) \
-            for i, script in enumerate(other_output_scripts))
+        (cj_addr, cj_value),
+        (cj_change_addr, maker_total_value - cj_value),  # cjfee=0, txfee=0
+        *((addr, cj_value + (i%2)*(50000000+i)) \
+            for i, addr in enumerate(other_output_addrs))
     )
 
     maker_utxos = [inputs[0]]
 
-    tx = btc.deserialize(btc.mktx(inputs, outputs))
+    tx = btc.mktx(inputs, outputs)
     offerlist = construct_tx_offerlist(cj_addr, cj_change_addr, maker_utxos,
                                        maker_total_value, cj_value, offertype)
 
@@ -119,21 +115,20 @@ def test_verify_unsigned_tx_sw_valid(setup_env_nodeps):
 
     # test standard cj
     tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr,
-        [next(p2sh_gen)[1] for s in range(4)], cj_script, cj_change_script)
+        [next(p2sh_gen)[0] for s in range(4)])
 
     assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "standard sw cj"
 
     # test cj with mixed outputs
     tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr,
-        list(chain((next(p2sh_gen)[1] for s in range(3)),
-                   (next(p2pkh_gen)[1] for s in range(1)))),
-        cj_script, cj_change_script)
+        list(chain((next(p2sh_gen)[0] for s in range(3)),
+                   (next(p2pkh_gen)[0] for s in range(1)))))
 
     assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "sw cj with p2pkh output"
 
     # test cj with only p2pkh outputs
     tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr,
-        [next(p2pkh_gen)[1] for s in range(4)], cj_script, cj_change_script)
+        [next(p2pkh_gen)[0] for s in range(4)])
 
     assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "sw cj with only p2pkh outputs"
 
@@ -152,21 +147,20 @@ def test_verify_unsigned_tx_nonsw_valid(setup_env_nodeps):
 
     # test standard cj
     tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr,
-        [next(p2pkh_gen)[1] for s in range(4)], cj_script, cj_change_script, 'reloffer')
+        [next(p2pkh_gen)[0] for s in range(4)], offertype='reloffer')
 
     assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "standard nonsw cj"
 
     # test cj with mixed outputs
     tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr,
-        list(chain((next(p2sh_gen)[1] for s in range(1)),
-                   (next(p2pkh_gen)[1] for s in range(3)))),
-        cj_script, cj_change_script, 'reloffer')
+        list(chain((next(p2sh_gen)[0] for s in range(1)),
+                   (next(p2pkh_gen)[0] for s in range(3)))), offertype='reloffer')
 
     assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "nonsw cj with p2sh output"
 
     # test cj with only p2sh outputs
     tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr,
-        [next(p2sh_gen)[1] for s in range(4)], cj_script, cj_change_script, 'reloffer')
+        [next(p2sh_gen)[0] for s in range(4)], offertype='reloffer')
 
     assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "nonsw cj with only p2sh outputs"
 
@@ -175,4 +169,5 @@ def test_verify_unsigned_tx_nonsw_valid(setup_env_nodeps):
 def setup_env_nodeps(monkeypatch):
     monkeypatch.setattr(jmclient.configure, 'get_blockchain_interface_instance',
                         lambda x: DummyBlockchainInterface())
+    btc.select_chain_params("bitcoin/regtest")
     load_test_config()

jmclient/test/test_payjoin.py

@@ -75,14 +75,11 @@ def final_checks(wallet_services, amount, txfee, tsb, msb, source_mixdepth=0):
     return True
 
 @pytest.mark.parametrize('wallet_cls, wallet_structures, mean_amt',
-        [([LegacyWallet, LegacyWallet],
-          [[4, 0, 0, 0, 0]] * 2, 1.0),
+        [ # note we have removed LegacyWallet test cases.
          ([SegwitLegacyWallet, SegwitLegacyWallet],
           [[1, 3, 0, 0, 0]] * 2, 2.0),
          ([SegwitWallet, SegwitWallet],
           [[1, 0, 0, 0, 0]] * 2, 4.0),
-         ([LegacyWallet, SegwitWallet],
-          [[4, 0, 0, 0, 0]] * 2, 1.0),
          ([SegwitLegacyWallet, SegwitWallet],
           [[1, 3, 0, 0, 0]] * 2, 2.0),
          ([SegwitWallet, SegwitLegacyWallet],

jmclient/test/test_podle.py

@@ -28,8 +28,8 @@ def test_commitment_retries(setup_podle):
     """
     allowed = jm_single().config.getint("POLICY", "taker_utxo_retries")
     #make some pretend commitments
-    dummy_priv_utxo_pairs = [(bitcoin.sha256(os.urandom(10)),
-           bitcoin.sha256(os.urandom(10))+":0") for _ in range(10)]
+    dummy_priv_utxo_pairs = [(bitcoin.Hash(os.urandom(10)),
+           bitcoin.b2x(bitcoin.Hash(os.urandom(10)))+":0") for _ in range(10)]
     #test a single commitment request of all 10
     for x in dummy_priv_utxo_pairs:
         p = generate_podle([x], allowed)
@@ -46,15 +46,15 @@ def generate_single_podle_sig(priv, i):
     This calls the underlying 'raw' code based on the class PoDLE, not the
     library 'generate_podle' which intelligently searches and updates commitments.
     """
-    dummy_utxo = bitcoin.sha256(priv) + ":3"
-    podle = PoDLE(dummy_utxo, binascii.hexlify(priv).decode('ascii'))
+    dummy_utxo = bitcoin.b2x(bitcoin.Hash(priv)) + ":3"
+    podle = PoDLE(dummy_utxo, priv)
     r = podle.generate_podle(i)
     return (r['P'], r['P2'], r['sig'],
             r['e'], r['commit'])
 
 def test_rand_commitments(setup_podle):
     for i in range(20):
-        priv = os.urandom(32)
+        priv = os.urandom(32)+b"\x01"
         Pser, P2ser, s, e, commitment = generate_single_podle_sig(priv, 1 + i%5)
         assert verify_podle(Pser, P2ser, s, e, commitment)
         #tweak commitments to verify failure
@@ -90,7 +90,7 @@ def test_external_commitments(setup_podle):
     tries = jm_single().config.getint("POLICY","taker_utxo_retries")
     for i in range(10):
         priv = os.urandom(32)
-        dummy_utxo = bitcoin.sha256(priv)+":2"
+        dummy_utxo = (bitcoin.Hash(priv), 2)
         ecs[dummy_utxo] = {}
         ecs[dummy_utxo]['reveal']={}
         for j in range(tries):
@@ -104,16 +104,16 @@ def test_external_commitments(setup_podle):
         assert external[u]['P'] == ecs[u]['P']
         for i in range(tries):
             for x in ['P2', 's', 'e']:
-                assert external[u]['reveal'][str(i)][x] == ecs[u]['reveal'][i][x]
+                assert external[u]['reveal'][i][x] == ecs[u]['reveal'][i][x]
     
     #add a dummy used commitment, then try again
-    update_commitments(commitment="ab"*32)
+    update_commitments(commitment=b"\xab"*32)
     ecs = {}
     known_commits = []
     known_utxos = []
     tries = 3
     for i in range(1, 6):
-        u = binascii.hexlify(struct.pack(b'B', i)*32).decode('ascii')
+        u = (struct.pack(b'B', i)*32, i+3)
         known_utxos.append(u)
         priv = struct.pack(b'B', i)*32+b"\x01"
         ecs[u] = {}
@@ -131,8 +131,9 @@ def test_external_commitments(setup_podle):
     #this should find the remaining one utxo and return from it
     assert generate_podle([], max_tries=tries, allow_external=known_utxos)
     #test commitment removal
-    to_remove = ecs[binascii.hexlify(struct.pack(b'B', 3)*32).decode('ascii')]
-    update_commitments(external_to_remove={binascii.hexlify(struct.pack(b'B', 3)*32).decode('ascii'):to_remove})
+    tru = (struct.pack(b"B", 3)*32, 3+3)
+    to_remove = {tru: ecs[tru]}
+    update_commitments(external_to_remove=to_remove)
     #test that an incorrectly formatted file raises
     with open(get_commitment_file(), "rb") as f:
         validjson = json.loads(f.read().decode('utf-8'))
@@ -152,14 +153,14 @@ def test_podle_constructor(setup_podle):
     """Tests rules about construction of PoDLE object
     are conformed to.
     """
-    priv  = "aa"*32
+    priv  = b"\xaa"*32
     #pub and priv together not allowed
     with pytest.raises(PoDLEError) as e_info:
         p = PoDLE(priv=priv, P="dummypub")
     #no pub or priv is allowed, i forget if this is useful for something
     p = PoDLE()
     #create from priv
-    p = PoDLE(priv=priv+"01", u="dummyutxo")
+    p = PoDLE(priv=priv+b"\x01", u=(struct.pack(b"B", 7)*32, 4))
     pdict = p.generate_podle(2)
     assert all([k in pdict for k in ['used', 'utxo', 'P', 'P2', 'commit', 'sig', 'e']])
     #using the valid data, serialize/deserialize test
@@ -181,7 +182,7 @@ def test_podle_constructor(setup_podle):
     with pytest.raises(PoDLEError) as e_info:
         p.generate_podle(0)
     #Test construction from pubkey
-    pub = bitcoin.privkey_to_pubkey(priv+"01")
+    pub = bitcoin.privkey_to_pubkey(priv+b"\x01")
     p = PoDLE(P=pub)
     with pytest.raises(PoDLEError) as e_info:
         p.get_commitment()

jmclient/test/test_privkeys.py

@@ -2,13 +2,16 @@
 '''Public and private key validity and formatting tests.'''
 
 import jmbitcoin as btc
+from jmclient import (BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH,
+                      jm_single, load_test_config)
 import binascii
+import struct
 import json
 import pytest
 import os
 testdir = os.path.dirname(os.path.realpath(__file__))
 
-def test_read_raw_privkeys():
+def test_read_raw_privkeys(setup_keys):
     badkeys = [b'', b'\x07'*31,b'\x07'*34, b'\x07'*33]
     for b in badkeys:
         with pytest.raises(Exception) as e_info:
@@ -18,7 +21,7 @@ def test_read_raw_privkeys():
         c, k = btc.read_privkey(g[0])
         assert c == g[1]
 
-def test_wif_privkeys_invalid():
+def test_wif_privkeys_invalid(setup_keys):
     #first try to create wif privkey from key of wrong length
     bad_privs = [b'\x01\x02'*17] #some silly private key but > 33 bytes
 
@@ -27,7 +30,7 @@ def test_wif_privkeys_invalid():
     
     for priv in bad_privs:
         with pytest.raises(Exception) as e_info:
-            fake_wif = btc.wif_compressed_privkey(binascii.hexlify(priv).decode('ascii'))
+            fake_wif = BTCEngine.privkey_to_wif(priv)
 
     #Create a wif with wrong length
     bad_wif1 = btc.bin_to_b58check(b'\x01\x02'*34, b'\x80')
@@ -35,7 +38,7 @@ def test_wif_privkeys_invalid():
     bad_wif2 = btc.bin_to_b58check(b'\x07'*33, b'\x80')
     for bw in [bad_wif1, bad_wif2]:
         with pytest.raises(Exception) as e_info:
-            fake_priv = btc.from_wif_privkey(bw)
+            fake_priv, keytype = BTCEngine.wif_to_privkey(bw)
 
     #Some invalid b58 from bitcoin repo;
     #none of these are valid as any kind of key or address
@@ -49,15 +52,14 @@ def test_wif_privkeys_invalid():
             print('testing this key: ' + bad_key)
             #should throw exception
             with pytest.raises(Exception) as e_info:
-                from_wif_key = btc.from_wif_privkey(bad_key,
-                                                    btc.get_version_byte(bad_key))
+                from_wif_key, keytype = BTCEngine.wif_to_privkey(bad_key)
                 #in case the b58 check encoding is valid, we should
                 #also check if the leading version byte is in the
                 #expected set, and throw an error if not.
                 if chr(btc.get_version_byte(bad_key)) not in b'\x80\xef':
                     raise Exception("Invalid version byte")
 
-def test_wif_privkeys_valid():
+def test_wif_privkeys_valid(setup_keys):
     with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f:
         json_data = f.read()
     valid_keys_list = json.loads(json_data)
@@ -65,16 +67,20 @@ def test_wif_privkeys_valid():
         key, hex_key, prop_dict = a
         if prop_dict["isPrivkey"]:
             netval = "testnet" if prop_dict["isTestnet"] else "mainnet"
+            jm_single().config.set("BLOCKCHAIN", "network", netval)
             print('testing this key: ' + key)
             assert btc.get_version_byte(
                 key) in b'\x80\xef', "not valid network byte"
             comp = prop_dict["isCompressed"]
-            from_wif_key = btc.from_wif_privkey(
-                key,
-                compressed=comp,
-                vbyte=btc.from_int_to_byte(btc.from_byte_to_int(btc.get_version_byte(key))-128))
-            expected_key = hex_key
-            if comp: expected_key += '01'
+            if not comp:
+                # we only handle compressed keys
+                continue
+            from_wif_key, keytype = BTCEngine.wif_to_privkey(key)
+            expected_key = binascii.unhexlify(hex_key) + b"\x01"
             assert from_wif_key == expected_key, "Incorrect key decoding: " + \
                    str(from_wif_key) + ", should be: " + str(expected_key)
+    jm_single().config.set("BLOCKCHAIN", "network", "testnet")
 
+@pytest.fixture(scope='module')
+def setup_keys():
+    load_test_config()

jmclient/test/test_taker.py

@@ -10,13 +10,21 @@ import pytest
 import json
 import struct
 from base64 import b64encode
+from jmbase import (utxostr_to_utxo, utxo_to_utxostr, hextobin,
+                    dictchanger, listchanger)
 from jmclient import load_test_config, jm_single, set_commitment_file,\
     get_commitment_file, SegwitLegacyWallet, Taker, VolatileStorage,\
-    get_network, WalletService, NO_ROUNDING
+    get_network, WalletService, NO_ROUNDING, BTC_P2PKH
 from taker_test_data import t_utxos_by_mixdepth, t_orderbook,\
     t_maker_response, t_chosen_orders, t_dummy_ext
 from commontest import default_max_cj_fee
 
+def convert_utxos(utxodict):
+    return_dict = {}
+    for uk, val in utxodict.items():
+        return_dict[utxostr_to_utxo(uk)[1]] = val
+    return return_dict
+
 class DummyWallet(SegwitLegacyWallet):
     def __init__(self):
         storage = VolatileStorage()
@@ -36,32 +44,30 @@ class DummyWallet(SegwitLegacyWallet):
                 script = self._ENGINE.address_to_script(data['address'])
                 self._script_map[script] = path
 
-    def get_utxos_by_mixdepth(self, verbose=True, includeheight=False):
-        return t_utxos_by_mixdepth
-
-    def get_utxos_by_mixdepth_(self, verbose=True, include_disabled=False,
-                               includeheight=False):
-        utxos = self.get_utxos_by_mixdepth(verbose)
-
-        utxos_conv = {}
-        for md, utxo_data in utxos.items():
-            md_utxo = utxos_conv.setdefault(md, {})
-            for i, (utxo_hex, data) in enumerate(utxo_data.items()):
-                utxo, index = utxo_hex.split(':')
-                data_conv = {
-                    'script': self._ENGINE.address_to_script(data['address']),
-                    'path': (b'dummy', md, i),
-                    'value': data['value']
-                }
-                md_utxo[(binascii.unhexlify(utxo), int(index))] = data_conv
-
-        return utxos_conv
+    def get_utxos_by_mixdepth(self, include_disabled=False, verbose=True,
+                              includeheight=False):
+        # utxostr conversion routines because taker_test_data uses hex:
+        retval = {}
+        for mixdepth, v in t_utxos_by_mixdepth.items():
+            retval[mixdepth] = {}
+            for i, (utxo, val) in enumerate(v.items()):
+                retval[mixdepth][utxostr_to_utxo(utxo)[1]] = val
+                val["script"] = self._ENGINE.address_to_script(val['address'])
+                val["path"] = (b'dummy', mixdepth, i)
+        return retval
 
     def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None,
-                     maxheight=None):
+                     maxheight=None, includeaddr=False):
         if amount > self.get_balance_by_mixdepth()[mixdepth]:
             raise Exception("Not enough funds")
-        return t_utxos_by_mixdepth[mixdepth]
+        # comment as for get_utxos_by_mixdepth:
+        retval = {}
+        for k, v in t_utxos_by_mixdepth[mixdepth].items():
+            success, u = utxostr_to_utxo(k)
+            assert success
+            retval[u] = v
+            retval[u]["script"] = self.addr_to_script(retval[u]["address"])
+        return retval
 
     def get_internal_addr(self, mixing_depth, bci=None):
         if self.inject_addr_get_failure:
@@ -70,7 +76,7 @@ class DummyWallet(SegwitLegacyWallet):
 
     def sign_tx(self, tx, addrs):
         print("Pretending to sign on addresses: " + str(addrs))
-        return tx
+        return True, None
 
     def sign(self, tx, i, priv, amount):
         """Sign a transaction; the amount field
@@ -97,10 +103,10 @@ class DummyWallet(SegwitLegacyWallet):
         musGZczug3BAbqobmYherywCwL9REgNaNm
         """
         for p in privs:
-            addrs[p] = bitcoin.privkey_to_address(p, False, magicbyte=0x6f)
+            addrs[p] = BTC_P2PKH.privkey_to_address(p)
         for p, a in iteritems(addrs):
             if a == addr:
-                return binascii.hexlify(p).decode('ascii')
+                return p
         raise ValueError("No such keypair")
 
     def _is_my_bip32_path(self, path):
@@ -170,7 +176,7 @@ def test_make_commitment(setup_taker, failquery, external):
     amount = 110000000
     taker = get_taker([(mixdepth, amount, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", NO_ROUNDING)])
     taker.cjamount = amount
-    taker.input_utxos = t_utxos_by_mixdepth[0]
+    taker.input_utxos = convert_utxos(t_utxos_by_mixdepth[0])
     if failquery:
         jm_single().bc_interface.setQUSFail(True)
     taker.make_commitment()
@@ -194,12 +200,13 @@ def test_auth_pub_not_found(setup_taker):
     res = taker.initialize(orderbook)
     taker.orderbook = copy.deepcopy(t_chosen_orders) #total_cjfee unaffected, all same
     maker_response = copy.deepcopy(t_maker_response)
-    utxos = ["03243f4a659e278a1333f8308f6aaf32db4692ee7df0340202750fd6c09150f6:1",
-             "498faa8b22534f3b443c6b0ce202f31e12f21668b4f0c7a005146808f250d4c3:0",
-             "3f3ea820d706e08ad8dc1d2c392c98facb1b067ae4c671043ae9461057bd2a3c:1"]
+    utxos = [utxostr_to_utxo(x)[1] for x in [
+        "03243f4a659e278a1333f8308f6aaf32db4692ee7df0340202750fd6c09150f6:1",
+        "498faa8b22534f3b443c6b0ce202f31e12f21668b4f0c7a005146808f250d4c3:0",
+        "3f3ea820d706e08ad8dc1d2c392c98facb1b067ae4c671043ae9461057bd2a3c:1"]]
     fake_query_results = [{'value': 200000000,
                            'address': "mrKTGvFfYUEqk52qPKUroumZJcpjHLQ6pn",
-                           'script': '76a914767c956efe6092a775fea39a06d1cac9aae956d788ac',
+                           'script': hextobin('76a914767c956efe6092a775fea39a06d1cac9aae956d788ac'),
                            'utxo': utxos[i],
                            'confirms': 20} for i in range(3)]
     jm_single().bc_interface.insert_fake_query_results(fake_query_results)
@@ -223,8 +230,8 @@ def test_auth_pub_not_found(setup_taker):
         ([(0, 199850001, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False,
          2, False, None, None), #trigger sub dust change for taker
         #edge case triggers that do fail
-        ([(0, 199850000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False,
-         2, False, None, None), #trigger negative change        
+        ([(0, 199851000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False,
+         2, False, None, None), #trigger negative change
         ([(0, 199599800, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False,
          2, False, None, None), #trigger sub dust change for maker
         ([(0, 20000000, 3, "INTERNAL", 0, NO_ROUNDING)], True, False,
@@ -232,7 +239,7 @@ def test_auth_pub_not_found(setup_taker):
         ([(0, 20000000, 3, "INTERNAL", 0, NO_ROUNDING)], False, False,
          7, False, None, None), #test not enough cp
         ([(0, 80000000, 3, "INTERNAL", 0, NO_ROUNDING)], False, False,
-         2, False, None, "30000"), #test failed commit       
+         2, False, None, "30000"), #test failed commit
         ([(0, 20000000, 3, "INTERNAL", 0, NO_ROUNDING)], False, False,
          2, True, None, None), #test unauthed response
         ([(0, 5000000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, True,
@@ -282,14 +289,15 @@ def test_taker_init(setup_taker, schedule, highfee, toomuchcoins, minmakers,
     if notauthed:
         #Doctor one of the maker response data fields
         maker_response["J659UPUSLLjHJpaB"][1] = "xx" #the auth pub
-    if schedule[0][1] == 199850000:
+    if schedule[0][1] == 199851000:
         #triggers negative change
-        #makers offer 3000 txfee; we estimate ~ 147*10 + 2*34 + 10=1548 bytes
-        #times 30k = 46440, so we pay 43440, plus maker fees = 3*0.0002*200000000
-        #roughly, gives required selected = amt + 163k, hence the above =
-        #2btc - 150k sats = 199850000 (tweaked because of aggressive coin selection)
+        #((109 + 4*64)*ins + 34 * outs + 8)/4. plug in 9 ins and 8 outs gives
+        #tx size estimate = 1101 bytes. Times 30 ~= 33030.
+        #makers offer 3000 txfee, so we pay 30030, plus maker fees = 3*0.0002*200000000
+        #roughly, gives required selected = amt + 120k+30k, hence the above =
+        #2btc - 140k sats = 199851000 (tweaked because of aggressive coin selection)
         #simulate the effect of a maker giving us a lot more utxos
-        taker.utxos["dummy_for_negative_change"] = ["a", "b", "c", "d", "e"]
+        taker.utxos["dummy_for_negative_change"] = [(struct.pack(b"B", a) *32, a+1) for a in range(7,12)]
         with pytest.raises(ValueError) as e_info:
             res = taker.receive_utxos(maker_response)
         return clean_up()
@@ -361,7 +369,7 @@ def test_taker_init(setup_taker, schedule, highfee, toomuchcoins, minmakers,
     [
         (7),
     ])
-def test_unconfirm_confirm(schedule_len):
+def test_unconfirm_confirm(setup_taker, schedule_len):
     """These functions are: do-nothing by default (unconfirm, for Taker),
     and merely update schedule index for confirm (useful for schedules/tumbles).
     This tests that the on_finished callback correctly reports the fromtx
@@ -369,20 +377,26 @@ def test_unconfirm_confirm(schedule_len):
     The exception to the above is that the txd passed in must match
     self.latest_tx, so we use a dummy value here for that.
     """
+    class DummyTx(object):
+        pass
     test_unconfirm_confirm.txflag = True
     def finished_for_confirms(res, fromtx=False, waittime=0, txdetails=None):
         assert res #confirmed should always send true
         test_unconfirm_confirm.txflag = fromtx
 
     taker = get_taker(schedule_len=schedule_len, on_finished=finished_for_confirms)
-    taker.latest_tx = {"outs": "blah"}
-    taker.unconfirm_callback({"ins": "foo", "outs": "blah"}, "b")
+    taker.latest_tx = DummyTx()
+    taker.latest_tx.vout = "blah"
+    fake_txd = DummyTx()
+    fake_txd.vin = "foo"
+    fake_txd.vout = "blah"
+    taker.unconfirm_callback(fake_txd, "b")
     for i in range(schedule_len-1):
         taker.schedule_index += 1
-        fromtx = taker.confirm_callback({"ins": "foo", "outs": "blah"}, "b", 1)
+        fromtx = taker.confirm_callback(fake_txd, "b", 1)
         assert test_unconfirm_confirm.txflag
     taker.schedule_index += 1
-    fromtx = taker.confirm_callback({"ins": "foo", "outs": "blah"}, "b", 1)
+    fromtx = taker.confirm_callback(fake_txd, "b", 1)
     assert not test_unconfirm_confirm.txflag
     
 @pytest.mark.parametrize(
@@ -396,16 +410,13 @@ def test_on_sig(setup_taker, dummyaddr, schedule):
     #then, create a signature with various inputs, pass in in b64 to on_sig.
     #in order for it to verify, the DummyBlockchainInterface will have to 
     #return the right values in query_utxo_set
-    
+    utxos = [(struct.pack(b"B", x) * 32, 1) for x in range(5)]
     #create 2 privkey + utxos that are to be ours
     privs = [x*32 + b"\x01" for x in [struct.pack(b'B', y) for y in range(1,6)]]
-    utxos = [str(x)*64+":1" for x in range(5)]
-    fake_query_results = [{'value': 200000000,
-                           'utxo': utxos[x],
-                        'address': bitcoin.privkey_to_address(privs[x], False, magicbyte=0x6f),
-                        'script': bitcoin.mk_pubkey_script(
-                            bitcoin.privkey_to_address(privs[x], False, magicbyte=0x6f)),
-                        'confirms': 20} for x in range(5)]
+    scripts = [BTC_P2PKH.privkey_to_script(privs[x]) for x in range(5)]
+    addrs = [BTC_P2PKH.privkey_to_address(privs[x]) for x in range(5)]
+    fake_query_results = [{'value': 200000000, 'utxo': utxos[x], 'address': addrs[x],
+                           'script': scripts[x], 'confirms': 20} for x in range(5)]
 
     dbci = DummyBlockchainInterface()
     dbci.insert_fake_query_results(fake_query_results)
@@ -414,47 +425,52 @@ def test_on_sig(setup_taker, dummyaddr, schedule):
     outs = [{'value': 100000000, 'address': dummyaddr},
             {'value': 899990000, 'address': dummyaddr}]
     tx = bitcoin.mktx(utxos, outs)
-    
-    de_tx = bitcoin.deserialize(tx)
+    # since tx will be updated as it is signed, unlike in real life
+    # (where maker signing operation doesn't happen here), we'll create
+    # a second copy without the signatures:
+    tx2 = bitcoin.mktx(utxos, outs)
+
     #prepare the Taker with the right intermediate data
     taker = get_taker(schedule=schedule)
     taker.nonrespondants=["cp1", "cp2", "cp3"]
-    taker.latest_tx = de_tx
+    taker.latest_tx = tx
     #my inputs are the first 2 utxos
     taker.input_utxos = {utxos[0]:
-                        {'address': bitcoin.privkey_to_address(privs[0], False, magicbyte=0x6f),
-                         'script': bitcoin.mk_pubkey_script(
-                            bitcoin.privkey_to_address(privs[0], False, magicbyte=0x6f)),
+                        {'address': addrs[0],
+                         'script': scripts[0],
                          'value': 200000000},
                         utxos[1]:
-                        {'address': bitcoin.privkey_to_address(privs[1], False, magicbyte=0x6f),
-                         'script': bitcoin.mk_pubkey_script(
-                            bitcoin.privkey_to_address(privs[1], False, magicbyte=0x6f)),
+                        {'address': addrs[1],
+                         'script': scripts[1],
                          'value': 200000000}}    
     taker.utxos = {None: utxos[:2], "cp1": [utxos[2]], "cp2": [utxos[3]], "cp3":[utxos[4]]}
     for i in range(2):
         # placeholders required for my inputs
-        taker.latest_tx['ins'][i]['script'] = 'deadbeef' 
+        taker.latest_tx.vin[i].scriptSig = bitcoin.CScript(hextobin('deadbeef'))
+        tx2.vin[i].scriptSig = bitcoin.CScript(hextobin('deadbeef'))
     #to prepare for my signing, need to mark cjaddr:
     taker.my_cj_addr = dummyaddr
     #make signatures for the last 3 fake utxos, considered as "not ours":
-    tx3 = bitcoin.sign(tx, 2, privs[2])
-    sig3 = b64encode(binascii.unhexlify(bitcoin.deserialize(tx3)['ins'][2]['script']))
+    sig, msg = bitcoin.sign(tx2, 2, privs[2])
+    assert sig, "Failed to sign: " + msg
+    sig3 = b64encode(tx2.vin[2].scriptSig)
     taker.on_sig("cp1", sig3)
     #try sending the same sig again; should be ignored
     taker.on_sig("cp1", sig3)
-    tx4 = bitcoin.sign(tx, 3, privs[3])
-    sig4 = b64encode(binascii.unhexlify(bitcoin.deserialize(tx4)['ins'][3]['script']))
+    sig, msg = bitcoin.sign(tx2, 3, privs[3])
+    assert sig, "Failed to sign: " + msg
+    sig4 = b64encode(tx2.vin[3].scriptSig)
     #try sending junk instead of cp2's correct sig
-    taker.on_sig("cp2", str("junk"))
+    assert not taker.on_sig("cp2", str("junk")), "incorrectly accepted junk signature"
     taker.on_sig("cp2", sig4)
-    tx5 = bitcoin.sign(tx, 4, privs[4])
+    sig, msg = bitcoin.sign(tx2, 4, privs[4])
+    assert sig, "Failed to sign: " + msg
     #Before completing with the final signature, which will trigger our own
     #signing, try with an injected failure of query utxo set, which should
     #prevent this signature being accepted.
     dbci.setQUSFail(True)
-    sig5 = b64encode(binascii.unhexlify(bitcoin.deserialize(tx5)['ins'][4]['script']))
-    taker.on_sig("cp3", sig5)
+    sig5 = b64encode(tx2.vin[4].scriptSig)
+    assert not taker.on_sig("cp3", sig5), "incorrectly accepted sig5"
     #allow it to succeed, and try again
     dbci.setQUSFail(False)
     #this should succeed and trigger the we-sign code
@@ -465,7 +481,7 @@ def test_on_sig(setup_taker, dummyaddr, schedule):
     [
         ([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw")]),
     ])
-def test_auth_counterparty(schedule):
+def test_auth_counterparty(setup_taker, schedule):
     taker = get_taker(schedule=schedule)
     first_maker_response = t_maker_response["J659UPUSLLjHJpaB"]
     utxo, auth_pub, cjaddr, changeaddr, sig, maker_pub = first_maker_response

jmclient/test/test_tx_creation.py

@@ -1,6 +1,9 @@
 #! /usr/bin/env python
 '''Test of unusual transaction types creation and push to
-network to check validity.'''
+   network to check validity.
+   Note as of Feb 2020: earlier versions included multisig
+   p2(w)sh tests, these have been removed since Joinmarket
+   does not use this feature.'''
 
 import time
 import binascii
@@ -24,62 +27,12 @@ vpubs = ["03e9a06e539d6bf5cf1ca5c41b59121fa3df07a338322405a312c67b6349a707e9",
          "028a2f126e3999ff66d01dcb101ab526d3aa1bf5cbdc4bde14950a4cead95f6fcb",
          "02bea84d70e74f7603746b62d79bf035e16d982b56e6a1ee07dfd3b9130e8a2ad9"]
 
-
-@pytest.mark.parametrize(
-    "nw, wallet_structures, mean_amt, sdev_amt, amount, pubs, k", [
-        (1, [[2, 1, 4, 0, 0]], 4, 1.4, 600000000, vpubs[1:4], 2),
-        (1, [[3, 3, 0, 0, 3]], 4, 1.4, 100000000, vpubs[:4], 3),
-    ])
-def test_create_p2sh_output_tx(setup_tx_creation, nw, wallet_structures,
-                               mean_amt, sdev_amt, amount, pubs, k):
-    wallets = make_wallets(nw, wallet_structures, mean_amt, sdev_amt)
-    for w in wallets.values():
-        w['wallet'].sync_wallet(fast=True)
-    for k, w in enumerate(wallets.values()):
-        wallet_service = w['wallet']
-        ins_full = wallet_service.select_utxos(0, amount)
-        script = bitcoin.mk_multisig_script(pubs, k)
-        output_addr = bitcoin.p2sh_scriptaddr(bitcoin.safe_from_hex(script),
-                                              magicbyte=196)
-        txid = make_sign_and_push(ins_full,
-                                  wallet_service,
-                                  amount,
-                                  output_addr=output_addr)
-        assert txid
-
-def test_script_to_address(setup_tx_creation):
-    sample_script = "a914307f099a3bfedec9a09682238db491bade1b467f87"
-    assert bitcoin.script_to_address(
-        sample_script, vbyte=5) == "367SYUMqo1Fi4tQsycnmCtB6Ces1Z7EZLH"
-    assert bitcoin.script_to_address(
-        sample_script, vbyte=196) == "2MwfecDHsQTm4Gg3RekQdpqAMR15BJrjfRF"
-
-def test_mktx(setup_tx_creation):
-    """Testing exceptional conditions; not guaranteed
-    to create valid tx objects"""
-    #outpoint structure must be {"outpoint":{"hash":hash, "index": num}}
-    ins = [{'outpoint': {"hash":x*64, "index":0},
-            "script": "", "sequence": 4294967295} for x in ["a", "b", "c"]]
-    pub = vpubs[0]
-    addr = bitcoin.pubkey_to_address(pub, magicbyte=get_p2pk_vbyte())
-    script = bitcoin.address_to_script(addr)
-    outs = [script + ":1000", addr+":2000",{"script":script, "value":3000}]
-    tx = bitcoin.mktx(ins, outs)
-    print(tx)
-    #rewrite with invalid output
-    outs.append({"foo": "bar"})
-    with pytest.raises(Exception) as e_info:
-        tx = bitcoin.mktx(ins, outs)
-
-def test_bintxhash(setup_tx_creation):
-    tx = "abcdef1234"
-    x = bitcoin.bin_txhash(tx)
-    assert binascii.hexlify(x).decode('ascii') == "121480fc2cccd5103434a9c88b037e08ef6c4f9f95dfb85b56f7043a344613fe"
-
 def test_all_same_priv(setup_tx_creation):
     #recipient
-    priv = "aa"*32 + "01"
-    addr = bitcoin.privkey_to_address(priv, magicbyte=get_p2pk_vbyte())
+    priv = b"\xaa"*32 + b"\x01"
+    pub = bitcoin.privkey_to_pubkey(priv)
+    addr = str(bitcoin.CCoinAddress.from_scriptPubKey(
+        bitcoin.CScript([bitcoin.OP_0, bitcoin.Hash160(pub)])))
     wallet_service = make_wallets(1, [[1,0,0,0,0]], 1)[0]['wallet']
     #make another utxo on the same address
     addrinwallet = wallet_service.get_addr(0,0,0)
@@ -89,52 +42,41 @@ def test_all_same_priv(setup_tx_creation):
     outs = [{"address": addr, "value": 1000000}]
     ins = list(insfull.keys())
     tx = bitcoin.mktx(ins, outs)
-    tx = bitcoin.signall(tx, wallet_service.get_key_from_addr(addrinwallet))
-
-@pytest.mark.parametrize(
-    "signall",
-    [
-        (True,),
-        (False,),
-    ])
-def test_verify_tx_input(setup_tx_creation, signall):
-    priv = "aa"*32 + "01"
-    addr = bitcoin.privkey_to_address(priv, magicbyte=get_p2pk_vbyte())
+    scripts = {}
+    for i, j in enumerate(ins):
+        scripts[i] = (insfull[j]["script"], insfull[j]["value"])
+    success, msg = wallet_service.sign_tx(tx, scripts)
+    assert success, msg
+
+def test_verify_tx_input(setup_tx_creation):
+    priv = b"\xaa"*32 + b"\x01"
+    pub = bitcoin.privkey_to_pubkey(priv)
+    script = bitcoin.pubkey_to_p2sh_p2wpkh_script(pub)
+    addr = str(bitcoin.CCoinAddress.from_scriptPubKey(script))
     wallet_service = make_wallets(1, [[2,0,0,0,0]], 1)[0]['wallet']
     wallet_service.sync_wallet(fast=True)
     insfull = wallet_service.select_utxos(0, 110000000)
-    print(insfull)
     outs = [{"address": addr, "value": 1000000}]
     ins = list(insfull.keys())
     tx = bitcoin.mktx(ins, outs)
-    desertx = bitcoin.deserialize(tx)
-    print(desertx)
-    if signall:
-        privdict = {}
-        for index, ins in enumerate(desertx['ins']):
-            utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
-            ad = insfull[utxo]['address']
-            priv = wallet_service.get_key_from_addr(ad)
-            privdict[utxo] = priv
-        tx = bitcoin.signall(tx, privdict)
-    else:
-        for index, ins in enumerate(desertx['ins']):
-            utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
-            ad = insfull[utxo]['address']
-            priv = wallet_service.get_key_from_addr(ad)
-            tx = bitcoin.sign(tx, index, priv)
-    desertx2 = bitcoin.deserialize(tx)
-    print(desertx2)
-    sig, pub = bitcoin.deserialize_script(desertx2['ins'][0]['script'])
-    print(sig, pub)
-    pubscript = bitcoin.address_to_script(bitcoin.pubkey_to_address(
-        pub, magicbyte=get_p2pk_vbyte()))
-    sig = binascii.unhexlify(sig)
-    pub = binascii.unhexlify(pub)
-    sig_good = bitcoin.verify_tx_input(tx, 0, pubscript,
-                                       sig, pub)
-    assert sig_good
-   
+    scripts = {0: (insfull[ins[0]]["script"], bitcoin.coins_to_satoshi(1))}
+    success, msg = wallet_service.sign_tx(tx, scripts)
+    assert success, msg
+    # testing Joinmarket's ability to verify transaction inputs
+    # of others: pretend we don't have a wallet owning the transaction,
+    # and instead verify an input using the (sig, pub, scriptCode) data
+    # that is sent by counterparties:
+    cScrWit = tx.wit.vtxinwit[0].scriptWitness
+    sig = cScrWit.stack[0]
+    pub = cScrWit.stack[1]
+    scriptSig = tx.vin[0].scriptSig
+    tx2 = bitcoin.mktx(ins, outs)
+    res = bitcoin.verify_tx_input(tx2, 0, scriptSig,
+                            bitcoin.pubkey_to_p2sh_p2wpkh_script(pub),
+                            amount = bitcoin.coins_to_satoshi(1),
+                            witness = bitcoin.CScript([sig, pub]))
+    assert res
+
 def test_absurd_fees(setup_tx_creation):
     """Test triggering of ValueError exception
     if the transaction fees calculated from the blockchain
@@ -157,7 +99,6 @@ def test_create_sighash_txs(setup_tx_creation):
         wallet_service.sync_wallet(fast=True)
         amount = 350000000
         ins_full = wallet_service.select_utxos(0, amount)
-        print("using hashcode: " + str(sighash))
         txid = make_sign_and_push(ins_full, wallet_service, amount, hashcode=sighash)
         assert txid
 
@@ -165,118 +106,39 @@ def test_create_sighash_txs(setup_tx_creation):
     with pytest.raises(Exception) as e_info:
         fake_utxos = wallet_service.select_utxos(4, 1000000000)
 
-
-def test_spend_p2sh_utxos(setup_tx_creation):
-    #make a multisig address from 3 privs
-    privs = [struct.pack(b'B', x) * 32 + b'\x01' for x in range(1, 4)]
-    pubs = [bitcoin.privkey_to_pubkey(binascii.hexlify(priv).decode('ascii')) for priv in privs]
-    script = bitcoin.mk_multisig_script(pubs, 2)
-    msig_addr = bitcoin.p2sh_scriptaddr(script, magicbyte=196)
-    #pay into it
-    wallet_service = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet']
-    wallet_service.sync_wallet(fast=True)
-    amount = 350000000
-    ins_full = wallet_service.select_utxos(0, amount)
-    txid = make_sign_and_push(ins_full, wallet_service, amount, output_addr=msig_addr)
-    assert txid
-    #wait for mining
-    time.sleep(1)
-    #spend out; the input can be constructed from the txid of previous
-    msig_in = txid + ":0"
-    ins = [msig_in]
-    #random output address and change addr
-    output_addr = wallet_service.get_internal_addr(1)
-    amount2 = amount - 50000
-    outs = [{'value': amount2, 'address': output_addr}]
-    tx = bitcoin.mktx(ins, outs)
-    sigs = []
-    for priv in privs[:2]:
-        sigs.append(bitcoin.get_p2sh_signature(tx, 0, script, binascii.hexlify(priv).decode('ascii')))
-    tx = bitcoin.apply_multisignatures(tx, 0, script, sigs)
-    txid = jm_single().bc_interface.pushtx(tx)
-    assert txid
-
 def test_spend_p2wpkh(setup_tx_creation):
     #make 3 p2wpkh outputs from 3 privs
     privs = [struct.pack(b'B', x) * 32 + b'\x01' for x in range(1, 4)]
-    pubs = [bitcoin.privkey_to_pubkey(
-        binascii.hexlify(priv).decode('ascii')) for priv in privs]
+    pubs = [bitcoin.privkey_to_pubkey(priv) for priv in privs]
     scriptPubKeys = [bitcoin.pubkey_to_p2wpkh_script(pub) for pub in pubs]
-    addresses = [bitcoin.pubkey_to_p2wpkh_address(pub) for pub in pubs]
+    addresses = [str(bitcoin.CCoinAddress.from_scriptPubKey(
+        spk)) for spk in scriptPubKeys]
     #pay into it
     wallet_service = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet']
     wallet_service.sync_wallet(fast=True)
     amount = 35000000
     p2wpkh_ins = []
-    for addr in addresses:
+    for i, addr in enumerate(addresses):
         ins_full = wallet_service.select_utxos(0, amount)
         txid = make_sign_and_push(ins_full, wallet_service, amount, output_addr=addr)
         assert txid
-        p2wpkh_ins.append(txid + ":0")
+        p2wpkh_ins.append((txid, 0))
+        txhex = jm_single().bc_interface.get_transaction(txid)
         #wait for mining
-        time.sleep(1)
+        jm_single().bc_interface.tick_forward_chain(1)
     #random output address
     output_addr = wallet_service.get_internal_addr(1)
     amount2 = amount*3 - 50000
     outs = [{'value': amount2, 'address': output_addr}]
     tx = bitcoin.mktx(p2wpkh_ins, outs)
-    sigs = []
-    for i, priv in enumerate(privs):
-        # sign each of 3 inputs
-        tx = bitcoin.p2wpkh_sign(tx, i, binascii.hexlify(priv),
-                                 amount, native=True)
-        # check that verify_tx_input correctly validates;
-        # to do this, we need to extract the signature and get the scriptCode
-        # of this pubkey
-        scriptCode = bitcoin.pubkey_to_p2pkh_script(pubs[i])
-        witness = bitcoin.deserialize(tx)['ins'][i]['txinwitness']
-        assert len(witness) == 2
-        assert witness[1] == pubs[i]
-        sig = witness[0]
-        assert bitcoin.verify_tx_input(tx, i, scriptPubKeys[i], sig,
-                        pubs[i], scriptCode=scriptCode, amount=amount)
-    txid = jm_single().bc_interface.pushtx(tx)
-    assert txid
-
 
-def test_spend_p2wsh(setup_tx_creation):
-    #make 2 x 2 of 2multisig outputs; will need 4 privs
-    privs = [struct.pack(b'B', x) * 32 + b'\x01' for x in range(1, 5)]
-    privs = [binascii.hexlify(priv).decode('ascii') for priv in privs]
-    pubs = [bitcoin.privkey_to_pubkey(priv) for priv in privs]
-    redeemScripts = [bitcoin.mk_multisig_script(pubs[i:i+2], 2) for i in [0, 2]]
-    scriptPubKeys = [bitcoin.pubkeys_to_p2wsh_multisig_script(pubs[i:i+2]) for i in [0, 2]]
-    addresses = [bitcoin.pubkeys_to_p2wsh_multisig_address(pubs[i:i+2]) for i in [0, 2]]
-    #pay into it
-    wallet_service = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet']
-    wallet_service.sync_wallet(fast=True)
-    amount = 35000000
-    p2wsh_ins = []
-    for addr in addresses:
-        ins_full = wallet_service.select_utxos(0, amount)
-        txid = make_sign_and_push(ins_full, wallet_service, amount, output_addr=addr)
-        assert txid
-        p2wsh_ins.append(txid + ":0")
-        #wait for mining
-        time.sleep(1)
-    #random output address and change addr
-    output_addr = wallet_service.get_internal_addr(1)
-    amount2 = amount*2 - 50000
-    outs = [{'value': amount2, 'address': output_addr}]
-    tx = bitcoin.mktx(p2wsh_ins, outs)
-    sigs = []
-    for i in range(2):
-        sigs = []
-        for priv in privs[i*2:i*2+2]:
-            # sign input j with each of 2 keys
-            sig = bitcoin.get_p2sh_signature(tx, i, redeemScripts[i], priv, amount=amount)
-            sigs.append(sig)
-            # check that verify_tx_input correctly validates;
-            assert bitcoin.verify_tx_input(tx, i, scriptPubKeys[i], sig,
-                                           bitcoin.privkey_to_pubkey(priv),
-                                           scriptCode=redeemScripts[i], amount=amount)
-        tx = bitcoin.apply_p2wsh_multisignatures(tx, i, redeemScripts[i], sigs)
-    txid = jm_single().bc_interface.pushtx(tx)
+    for i, priv in enumerate(privs):
+        # sign each of 3 inputs; note that bitcoin.sign
+        # automatically validates each signature it creates.
+        sig, msg = bitcoin.sign(tx, i, priv, amount=amount, native=True)
+        if not sig:
+            assert False, msg
+    txid = jm_single().bc_interface.pushtx(tx.serialize())
     assert txid
 
 def test_spend_freeze_script(setup_tx_creation):
@@ -320,7 +182,6 @@ def test_spend_freeze_script(setup_tx_creation):
         push_success = jm_single().bc_interface.pushtx(tx)
 
         assert push_success == required_success
-
 @pytest.fixture(scope="module")
 def setup_tx_creation():
     load_test_config()

jmclient/test/test_valid_addresses.py

@@ -1,10 +1,18 @@
 from jmclient.configure import validate_address, load_test_config
 from jmclient import jm_single
+import jmbitcoin as btc
 import json
 import pytest
 import os
 testdir = os.path.dirname(os.path.realpath(__file__))
 
+def address_valid_somewhere(addr):
+    for x in ["bitcoin", "bitcoin/testnet", "bitcoin/regtest"]:
+        btc.select_chain_params(x)
+        if validate_address(addr)[0]:
+            return True
+    return False
+
 def test_non_addresses(setup_addresses):
     #could flesh this out with other examples
     res, msg = validate_address(2)
@@ -17,9 +25,8 @@ def test_b58_invalid_addresses(setup_addresses):
     invalid_key_list = json.loads(json_data)
     for k in invalid_key_list:
         bad_key = k[0]
-        res, message = validate_address(bad_key)
-        assert res == False, "Incorrectly validated address: " + bad_key + " with message: " + message
-
+        res = address_valid_somewhere(bad_key)
+        assert res == False, "Incorrectly validated address: " + bad_key
 
 def test_b58_valid_addresses():
     with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f:
@@ -30,30 +37,36 @@ def test_b58_valid_addresses():
         if not prop_dict["isPrivkey"]:
             if prop_dict["isTestnet"]:
                 jm_single().config.set("BLOCKCHAIN", "network", "testnet")
+                btc.select_chain_params("bitcoin/testnet")
             else:
                 jm_single().config.set("BLOCKCHAIN", "network", "mainnet")
+                btc.select_chain_params("bitcoin")
             #if using pytest -s ; sanity check to see what's actually being tested
-            print('testing this address: ', addr)
             res, message = validate_address(addr)
             assert res == True, "Incorrectly failed to validate address: " + addr + " with message: " + message
     jm_single().config.set("BLOCKCHAIN", "network", "testnet")
+    btc.select_chain_params("bitcoin/regtest")
 
 def test_valid_bech32_addresses():
     valids = ["BC1QW508D6QEJXTDG4Y5R3ZARVARY0C5XW7KV8F3T4",
     "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sl5k7",
-    "bc1pw508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7k7grplx",
-    "BC1SW50QA3JX3S",
-    "bc1zw508d6qejxtdg4y5r3zarvaryvg6kdaj",
+    # TODO these are valid bech32 addresses but rejected by bitcointx
+    # because they are not witness version 0; add others.
+    #"bc1pw508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7k7grplx",
+    #"BC1SW50QA3JX3S",
+    #"bc1zw508d6qejxtdg4y5r3zarvaryvg6kdaj",
     "tb1qqqqqp399et2xygdj5xreqhjjvcmzhxw4aywxecjdzew6hylgvsesrxh6hy"]
     for va in valids:
-        print("Testing this address: ", va)
         if va.lower()[:2] == "bc":
             jm_single().config.set("BLOCKCHAIN", "network", "mainnet")
+            btc.select_chain_params("bitcoin")
         else:
             jm_single().config.set("BLOCKCHAIN", "network", "testnet")
+            btc.select_chain_params("bitcoin/testnet")
         res, message = validate_address(va)
         assert res == True, "Incorrect failed to validate address: " + va + " with message: " + message
     jm_single().config.set("BLOCKCHAIN", "network", "testnet")
+    btc.select_chain_params("bitcoin/regtest")
 
 def test_invalid_bech32_addresses():
     invalids = [
@@ -68,8 +81,7 @@ def test_invalid_bech32_addresses():
     "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3pjxtptv",
     "bc1gmk9yu"]
     for iva in invalids:
-        print("Testing this address: ", iva)
-        res, message = validate_address(iva)
+        res = address_valid_somewhere(iva)
         assert res == False, "Incorrectly validated address: " + iva
 
 @pytest.fixture(scope="module")

jmclient/test/test_wallet.py

@@ -6,12 +6,14 @@ from binascii import hexlify, unhexlify
 
 import pytest
 import jmbitcoin as btc
-from commontest import binarize_tx, ensure_bip65_activated
-from jmbase import get_log
+from commontest import ensure_bip65_activated
+from jmbase import (get_log, utxostr_to_utxo, utxo_to_utxostr,
+                    hextobin, bintohex)
 from jmclient import load_test_config, jm_single, \
     SegwitLegacyWallet,BIP32Wallet, BIP49Wallet, LegacyWallet,\
     VolatileStorage, get_network, cryptoengine, WalletError,\
     SegwitWallet, WalletService, SegwitLegacyWalletFidelityBonds,\
+    BTC_P2PKH, BTC_P2SH_P2WPKH,\
     FidelityBondMixin, FidelityBondWatchonlyWallet, wallet_gettimelockaddress
 from test_blockchaininterface import sync_test_wallet
 
@@ -20,10 +22,7 @@ log = get_log()
 
 
 def signed_tx_is_segwit(tx):
-    for inp in tx['ins']:
-        if 'txinwitness' not in inp:
-            return False
-    return True
+    return tx.has_witness()
 
 
 def assert_segwit(tx):
@@ -47,10 +46,11 @@ def get_populated_wallet(amount=10**8, num=3):
 
 
 def fund_wallet_addr(wallet, addr, value_btc=1):
-    txin_id = jm_single().bc_interface.grab_coins(addr, value_btc)
+    # special case, grab_coins returns hex from rpc:
+    txin_id = hextobin(jm_single().bc_interface.grab_coins(addr, value_btc))
     txinfo = jm_single().bc_interface.get_transaction(txin_id)
-    txin = btc.deserialize(unhexlify(txinfo['hex']))
-    utxo_in = wallet.add_new_utxos_(txin, unhexlify(txin_id), 1)
+    txin = btc.CMutableTransaction.deserialize(btc.x(txinfo["hex"]))
+    utxo_in = wallet.add_new_utxos(txin, 1)
     assert len(utxo_in) == 1
     return list(utxo_in.keys())[0]
 
@@ -367,13 +367,15 @@ def test_signing_imported(setup_wallet, wif, keytype, type_check):
     path = wallet.import_private_key(MIXDEPTH, wif, keytype)
     utxo = fund_wallet_addr(wallet, wallet.get_address_from_path(path))
     # The dummy output is constructed as an unspendable p2sh:
-    tx = btc.deserialize(btc.mktx(['{}:{}'.format(
-        hexlify(utxo[0]).decode('ascii'), utxo[1])],
-        [btc.p2sh_scriptaddr(b"\x00",magicbyte=196) + ':' + str(10**8 - 9000)]))
+    tx = btc.mktx([utxo],
+                [{"address": str(btc.CCoinAddress.from_scriptPubKey(
+                    btc.CScript(b"\x00").to_p2sh_scriptPubKey())),
+                  "value": 10**8 - 9000}])    
     script = wallet.get_script_from_path(path)
-    tx = wallet.sign_tx(tx, {0: (script, 10**8)})
+    success, msg = wallet.sign_tx(tx, {0: (script, 10**8)})
+    assert success, msg
     type_check(tx)
-    txout = jm_single().bc_interface.pushtx(btc.serialize(tx))
+    txout = jm_single().bc_interface.pushtx(tx.serialize())
     assert txout
 
 
@@ -385,17 +387,19 @@ def test_signing_imported(setup_wallet, wif, keytype, type_check):
 def test_signing_simple(setup_wallet, wallet_cls, type_check):
     jm_single().config.set('BLOCKCHAIN', 'network', 'testnet')
     storage = VolatileStorage()
-    wallet_cls.initialize(storage, get_network())
+    wallet_cls.initialize(storage, get_network(), entropy=b"\xaa"*16)
     wallet = wallet_cls(storage)
     utxo = fund_wallet_addr(wallet, wallet.get_internal_addr(0))
     # The dummy output is constructed as an unspendable p2sh:
-    tx = btc.deserialize(btc.mktx(['{}:{}'.format(
-        hexlify(utxo[0]).decode('ascii'), utxo[1])],
-        [btc.p2sh_scriptaddr(b"\x00",magicbyte=196) + ':' + str(10**8 - 9000)]))
+    tx = btc.mktx([utxo],
+            [{"address": str(btc.CCoinAddress.from_scriptPubKey(
+                btc.CScript(b"\x00").to_p2sh_scriptPubKey())),
+              "value": 10**8 - 9000}])    
     script = wallet.get_script(0, 1, 0)
-    tx = wallet.sign_tx(tx, {0: (script, 10**8)})
+    success, msg = wallet.sign_tx(tx, {0: (script, 10**8)})
+    assert success, msg
     type_check(tx)
-    txout = jm_single().bc_interface.pushtx(btc.serialize(tx))
+    txout = jm_single().bc_interface.pushtx(tx.serialize())
     assert txout
 
 def test_timelocked_output_signing(setup_wallet):
@@ -450,18 +454,18 @@ def test_add_utxos(setup_wallet):
     for md in range(1, wallet.max_mixdepth + 1):
         assert balances[md] == 0
 
-    utxos = wallet.get_utxos_by_mixdepth_()
+    utxos = wallet.get_utxos_by_mixdepth()
     assert len(utxos[0]) == num_tx
     for md in range(1, wallet.max_mixdepth + 1):
         assert not utxos[md]
 
     with pytest.raises(Exception):
         # no funds in mixdepth
-        wallet.select_utxos_(1, amount)
+        wallet.select_utxos(1, amount)
 
     with pytest.raises(Exception):
         # not enough funds
-        wallet.select_utxos_(0, amount * (num_tx + 1))
+        wallet.select_utxos(0, amount * (num_tx + 1))
 
     wallet.reset_utxos()
     assert wallet.get_balance_by_mixdepth()[0] == 0
@@ -472,12 +476,12 @@ def test_select_utxos(setup_wallet):
     amount = 10**8
 
     wallet = get_populated_wallet(amount)
-    utxos = wallet.select_utxos_(0, amount // 2)
+    utxos = wallet.select_utxos(0, amount // 2)
 
     assert len(utxos) == 1
     utxos = list(utxos.keys())
 
-    more_utxos = wallet.select_utxos_(0, int(amount * 1.5), utxo_filter=utxos)
+    more_utxos = wallet.select_utxos(0, int(amount * 1.5), utxo_filter=utxos)
     assert len(more_utxos) == 2
     assert utxos[0] not in more_utxos
 
@@ -488,14 +492,11 @@ def test_add_new_utxos(setup_wallet):
 
     scripts = [wallet.get_new_script(x, True) for x in range(3)]
     tx_scripts = list(scripts)
-    tx_scripts.append(b'\x22'*17)
-
-    tx = btc.deserialize(btc.mktx(
-        ['0'*64 + ':2'], [{'script': hexlify(s).decode('ascii'), 'value': 10**8}
-                          for s in tx_scripts]))
-    binarize_tx(tx)
-    txid = b'\x01' * 32
-    added = wallet.add_new_utxos_(tx, txid, 1)
+    tx = btc.mktx(
+            [(b"\x00"*32, 2)],
+            [{"address": wallet.script_to_addr(s),
+              "value": 10**8} for s in tx_scripts])
+    added = wallet.add_new_utxos(tx, 1)
     assert len(added) == len(scripts)
 
     added_scripts = {x['script'] for x in added.values()}
@@ -517,21 +518,20 @@ def test_remove_old_utxos(setup_wallet):
     for i in range(3):
         txin = jm_single().bc_interface.grab_coins(
             wallet.get_internal_addr(1), 1)
-        wallet.add_utxo(unhexlify(txin), 0, wallet.get_script(1, 1, i), 10**8, 1)
+        wallet.add_utxo(btc.x(txin), 0, wallet.get_script(1, 1, i), 10**8, 1)
 
-    inputs = wallet.select_utxos_(0, 10**8)
-    inputs.update(wallet.select_utxos_(1, 2 * 10**8))
+    inputs = wallet.select_utxos(0, 10**8)
+    inputs.update(wallet.select_utxos(1, 2 * 10**8))
     assert len(inputs) == 3
 
     tx_inputs = list(inputs.keys())
     tx_inputs.append((b'\x12'*32, 6))
 
-    tx = btc.deserialize(btc.mktx(
-        ['{}:{}'.format(hexlify(txid).decode('ascii'), i) for txid, i in tx_inputs],
-        ['0' * 36 + ':' + str(3 * 10**8 - 1000)]))
-    binarize_tx(tx)
+    tx = btc.mktx(tx_inputs,
+        [{"address": "2N9gfkUsFW7Kkb1Eurue7NzUxUt7aNJiS1U",
+          "value": 3 * 10**8 - 1000}])
 
-    removed = wallet.remove_old_utxos_(tx)
+    removed = wallet.remove_old_utxos(tx)
     assert len(removed) == len(inputs)
 
     for txid in removed:
@@ -760,6 +760,14 @@ def test_wallet_mixdepth_decrease(setup_wallet):
     max_mixdepth = wallet.max_mixdepth
     assert max_mixdepth >= 1, "bad default value for mixdepth for this test"
     utxo = fund_wallet_addr(wallet, wallet.get_internal_addr(max_mixdepth), 1)
+    bci = jm_single().bc_interface
+    unspent_list = bci.rpc('listunspent', [0])
+    # filter on label, but note (a) in certain circumstances (in-
+    # wallet transfer) it is possible for the utxo to be labeled
+    # with the external label, and (b) the wallet will know if it
+    # belongs or not anyway (is_known_addr):
+    our_unspent_list = [x for x in unspent_list if (
+        bci.is_address_labeled(x, wallet.get_wallet_name()))]
     assert wallet.get_balance_by_mixdepth()[max_mixdepth] == 10**8
     wallet.close()
     storage_data = wallet._storage.file_data
@@ -777,7 +785,7 @@ def test_wallet_mixdepth_decrease(setup_wallet):
 
     # wallet.select_utxos will still return utxos from higher mixdepths
     # because we explicitly ask for a specific mixdepth
-    assert utxo in new_wallet.select_utxos_(max_mixdepth, 10**7)
+    assert utxo in new_wallet.select_utxos(max_mixdepth, 10**7)
 
 def test_watchonly_wallet(setup_wallet):
     jm_single().config.set('BLOCKCHAIN', 'network', 'testnet')
@@ -819,6 +827,7 @@ def test_watchonly_wallet(setup_wallet):
 @pytest.fixture(scope='module')
 def setup_wallet():
     load_test_config()
+    btc.select_chain_params("bitcoin/regtest")
     #see note in cryptoengine.py:
     cryptoengine.BTC_P2WPKH.VBYTE = 100
     jm_single().bc_interface.tick_forward_chain_interval = 2

jmclient/test/test_wallets.py

@@ -8,7 +8,8 @@ from commontest import create_wallet_for_sync, make_sign_and_push
 import json
 
 import pytest
-from jmbase import get_log
+from jmbase import get_log, hextobin
+import jmbitcoin as btc
 from jmclient import (
     load_test_config, jm_single,
     estimate_tx_fee, BitcoinCoreInterface, Mnemonic)
@@ -43,9 +44,10 @@ def test_query_utxo_set(setup_wallets):
     txid = do_tx(wallet_service, 90000000)
     txid2 = do_tx(wallet_service, 20000000)
     print("Got txs: ", txid, txid2)
-    res1 = jm_single().bc_interface.query_utxo_set(txid + ":0", includeunconf=True)
+    res1 = jm_single().bc_interface.query_utxo_set(
+        (txid, 0), includeunconf=True)
     res2 = jm_single().bc_interface.query_utxo_set(
-        [txid + ":0", txid2 + ":1"],
+        [(txid, 0), (txid2, 1)],
         includeconf=True, includeunconf=True)
     assert len(res1) == 1
     assert len(res2) == 2
@@ -53,7 +55,7 @@ def test_query_utxo_set(setup_wallets):
     assert not 'confirms' in res1[0]
     assert 'confirms' in res2[0]
     assert 'confirms' in res2[1]
-    res3 = jm_single().bc_interface.query_utxo_set("ee" * 32 + ":25")
+    res3 = jm_single().bc_interface.query_utxo_set((b"\xee" * 32, 25))
     assert res3 == [None]
 
 
@@ -69,11 +71,11 @@ def test_wrong_network_bci(setup_wallets):
 def test_pushtx_errors(setup_wallets):
     """Ensure pushtx fails return False
     """
-    badtxhex = "aaaa"
-    assert not jm_single().bc_interface.pushtx(badtxhex)
+    badtx = b"\xaa\xaa"
+    assert not jm_single().bc_interface.pushtx(badtx)
     #Break the authenticated jsonrpc and try again
     jm_single().bc_interface.jsonRpc.port = 18333
-    assert not jm_single().bc_interface.pushtx(t_raw_signed_tx)
+    assert not jm_single().bc_interface.pushtx(hextobin(t_raw_signed_tx))
     #rebuild a valid jsonrpc inside the bci
     load_test_config()
 

jmclient/test/test_walletservice.py

@@ -39,7 +39,6 @@ def test_address_reuse_freezing(setup_walletservice):
     """
     context = {'cb_called': 0}
     def reuse_callback(utxostr):
-        print("Address reuse freezing callback on utxo: ", utxostr)
         context['cb_called'] += 1
     # we must fund after initial sync (for imports), hence
     # "populated" with no coins

jmclient/test/test_yieldgenerator.py

@@ -1,6 +1,6 @@
 
 import unittest
-
+from jmbitcoin import CMutableTxOut, CMutableTransaction
 from jmclient import load_test_config, jm_single,\
     SegwitLegacyWallet, VolatileStorage, YieldGeneratorBasic, \
     get_network, WalletService
@@ -28,14 +28,12 @@ class CustomUtxoWallet(SegwitLegacyWallet):
             self.add_utxo_at_mixdepth(m, b)
 
     def add_utxo_at_mixdepth(self, mixdepth, balance):
-        tx = {'outs': [{'script': self.get_internal_script(mixdepth),
-                        'value': balance}]}
-        # We need to generate a fake "txid" that has to be unique for all the
-        # UTXOs that are added to the wallet.  For that, we simply use the
-        # script, and make it fit the required length (32 bytes).
-        txid = tx['outs'][0]['script'] + b'x' * 32
-        txid = txid[:32]
-        self.add_new_utxos_(tx, txid, 1)
+        txout = CMutableTxOut(balance, self.get_internal_script(mixdepth))
+        tx = CMutableTransaction()
+        tx.vout = [txout]
+        # (note: earlier requirement that txid be generated uniquely is now
+        # automatic; tx.GetTxid() functions correctly within the wallet).
+        self.add_new_utxos(tx, 1)
 
     def assert_utxos_from_mixdepth(self, utxos, expected):
         """Asserts that the list of UTXOs (as returned from UTXO selection

jmdaemon/test/test_message_channel.py

@@ -15,6 +15,7 @@ import base64
 import struct
 import traceback
 import threading
+import binascii
 import jmbitcoin as bitcoin
 from dummy_mc import DummyMessageChannel
 
@@ -22,10 +23,11 @@ from dummy_mc import DummyMessageChannel
 jlog = get_log()
 
 def make_valid_nick(i=0):
-    nick_priv = hashlib.sha256(struct.pack(b'B', i)*16).hexdigest() + '01'
-    nick_pubkey = bitcoin.privtopub(nick_priv)
-    nick_pkh_raw = hashlib.sha256(nick_pubkey.encode('ascii')).digest()[:NICK_HASH_LENGTH]
-    nick_pkh = bitcoin.b58encode(nick_pkh_raw)
+    nick_priv = hashlib.sha256(struct.pack(b'B', i)*16).digest() + b"\x01"
+    nick_pubkey = bitcoin.privkey_to_pubkey(nick_priv)
+    nick_pkh_raw = hashlib.sha256(binascii.hexlify(
+        nick_pubkey)).digest()[:NICK_HASH_LENGTH]
+    nick_pkh = bitcoin.base58.encode(nick_pkh_raw)
     #right pad to maximum possible; b58 is not fixed length.
     #Use 'O' as one of the 4 not included chars in base58.
     nick_pkh += 'O' * (NICK_MAX_ENCODED - len(nick_pkh))

scripts/add-utxo.py

@@ -17,7 +17,7 @@ import jmbitcoin as btc
 from jmclient import load_program_config, jm_single, get_p2pk_vbyte,\
     open_wallet, WalletService, add_external_commitments, update_commitments,\
     PoDLE, get_podle_commitments, get_utxo_info, validate_utxo_data, quit,\
-    get_wallet_path, add_base_options
+    get_wallet_path, add_base_options, BTCEngine, BTC_P2SH_P2WPKH
 from jmbase.support import EXIT_SUCCESS, EXIT_FAILURE, EXIT_ARGERROR, jmprint
 
 
@@ -32,9 +32,10 @@ def add_ext_commitments(utxo_datas):
         This calls the underlying 'raw' code based on the class PoDLE, not the
         library 'generate_podle' which intelligently searches and updates commitments.
         """
-        #Convert priv to hex
-        hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte())
-        podle = PoDLE(u, hexpriv)
+        #Convert priv from wif; require P2SH-P2WPKH keys
+        rawpriv, keytype = BTCEngine.wif_to_privkey(priv)
+        assert keytype == BTC_P2SH_P2WPKH
+        podle = PoDLE(u, rawpriv)
         r = podle.generate_podle(i)
         return (r['P'], r['P2'], r['sig'],
                 r['e'], r['commit'])

scripts/convert_old_wallet.py

@@ -7,10 +7,9 @@ from binascii import hexlify, unhexlify
 from collections import defaultdict
 from pyaes import AESModeOfOperationCBC, Decrypter
 from jmbase import JM_APP_NAME
-from jmclient import Storage, load_program_config
+from jmclient import Storage, load_program_config, BTCEngine
 from jmclient.wallet_utils import get_password, get_wallet_cls,\
     cli_get_wallet_passphrase_check, get_wallet_path
-from jmbitcoin import wif_compressed_privkey
 
 
 class ConvertException(Exception):
@@ -99,7 +98,7 @@ def new_wallet_from_data(data, file_name):
         for md in data['imported']:
             for privkey in data['imported'][md]:
                 privkey += b'\x01'
-                wif = wif_compressed_privkey(hexlify(privkey).decode('ascii'))
+                wif = BTCEngine.privkey_to_wif(privkey)
                 wallet.import_private_key(md, wif)
 
     wallet.save()

scripts/joinmarket-qt.py

@@ -75,7 +75,7 @@ from jmclient import load_program_config, get_network, update_persist_config,\
     wallet_generate_recover_bip39, wallet_display, get_utxos_enabled_disabled,\
     NO_ROUNDING, get_max_cj_fee_values, get_default_max_absolute_fee, \
     get_default_max_relative_fee, RetryableStorageError, add_base_options, \
-    FidelityBondMixin
+    BTCEngine, BTC_P2SH_P2WPKH, FidelityBondMixin
 from qtsupport import ScheduleWizard, TumbleRestartWizard, config_tips,\
     config_types, QtHandler, XStream, Buttons, OkButton, CancelButton,\
     PasswordDialog, MyTreeWidget, JMQtMessageBox, BLUE_FG,\
@@ -1530,14 +1530,18 @@ class JMMainWindow(QMainWindow):
         done = False
 
         def privkeys_thread():
+            # To explain this (given setting was already done in
+            # load_program_config), see:
+            # https://github.com/Simplexum/python-bitcointx/blob/9f1fa67a5445f8c187ef31015a4008bc5a048eea/bitcointx/__init__.py#L242-L243
+            # note, we ignore the return value as we only want to apply
+            # the chainparams setting logic:
+            get_blockchain_interface_instance(jm_single().config)
             for addr in addresses:
                 time.sleep(0.1)
                 if done:
                     break
                 priv = self.wallet_service.get_key_from_addr(addr)
-                private_keys[addr] = btc.wif_compressed_privkey(
-                    priv,
-                    vbyte=get_p2pk_vbyte())
+                private_keys[addr] = BTCEngine.privkey_to_wif(priv)
                 self.computing_privkeys_signal.emit()
             self.show_privkeys_signal.emit()
 
@@ -1569,10 +1573,13 @@ class JMMainWindow(QMainWindow):
                                    privkeys_fn + '.json'), "wb") as f:
                 for addr, pk in private_keys.items():
                     #sanity check
-                    if not addr == btc.pubkey_to_p2sh_p2wpkh_address(
-                                    btc.privkey_to_pubkey(
-                                        btc.from_wif_privkey(pk, vbyte=get_p2pk_vbyte())
-                                    ), get_p2sh_vbyte()):
+                    rawpriv, keytype = BTCEngine.wif_to_privkey(pk)
+                    if not keytype == BTC_P2SH_P2WPKH:
+                        JMQtMessageBox(None, "Failed to create privkey export, "
+                                       "should be keytype p2sh-p2wpkh but is not.",
+                                       mbtype='crit')
+                        return
+                    if not addr == self.wallet_service._ENGINE.privkey_to_address(rawpriv):
                         JMQtMessageBox(None, "Failed to create privkey export -" +\
                                        " critical error in key parsing.",
                                        mbtype='crit')
@@ -2009,7 +2016,7 @@ if isinstance(jm_single().bc_interface, RegtestBitcoinCoreInterface):
     jm_single().bc_interface.simulating = True
     jm_single().maker_timeout_sec = 15
     #trigger start with a fake tx
-    jm_single().bc_interface.pushtx("00"*20)
+    jm_single().bc_interface.pushtx(b"\x00"*20)
 
 #prepare for logging
 for dname in ['logs', 'wallets', 'cmtdata']:

scripts/sendtomany.py

@@ -8,10 +8,10 @@ for other reasons).
 from pprint import pformat
 from optparse import OptionParser
 import jmbitcoin as btc
-from jmbase import get_log, jmprint
+from jmbase import get_log, jmprint, bintohex, utxostr_to_utxo
 from jmclient import load_program_config, estimate_tx_fee, jm_single,\
     get_p2pk_vbyte, validate_address, get_utxo_info, add_base_options,\
-    validate_utxo_data, quit
+    validate_utxo_data, quit, BTCEngine, BTC_P2SH_P2WPKH, BTC_P2PKH
 
 
 log = get_log()
@@ -39,10 +39,12 @@ def sign(utxo, priv, destaddrs, segwit=True):
     log.info("Using fee: " + str(fee))
     for i, addr in enumerate(destaddrs):
         outs.append({'address': addr, 'value': share})
-    unsigned_tx = btc.mktx(ins, outs)
+    tx = btc.mktx(ins, outs)
     amtforsign = amt if segwit else None
-    return btc.sign(unsigned_tx, 0, btc.from_wif_privkey(
-        priv, vbyte=get_p2pk_vbyte()), amount=amtforsign)
+    rawpriv, _ = BTCEngine.wif_to_privkey(priv)
+    success, msg = btc.sign(tx, 0, rawpriv, amount=amtforsign)
+    assert success, msg
+    return tx
     
 def main():
     parser = OptionParser(
@@ -106,17 +108,20 @@ def main():
     for d in destaddrs:
         if not validate_address(d):
             quit(parser, "Address was not valid; wrong network?: " + d)
-    txsigned = sign(u, priv, destaddrs, segwit = not options.nonsegwit)
+    success, utxo = utxostr_to_utxo(u)
+    if not success:
+        quit(parser, "Failed to load utxo from string: " + utxo)
+    txsigned = sign(utxo, priv, destaddrs, segwit = not options.nonsegwit)
     if not txsigned:
         log.info("Transaction signing operation failed, see debug messages for details.")
         return
-    log.debug("Got signed transaction:\n" + txsigned)
+    log.info("Got signed transaction:\n" + bintohex(txsigned.serialize()))
     log.debug("Deserialized:")
-    log.debug(pformat(btc.deserialize(txsigned)))
+    log.debug(pformat(str(txsigned)))
     if input('Would you like to push to the network? (y/n):')[0] != 'y':
         log.info("You chose not to broadcast the transaction, quitting.")
         return
-    jm_single().bc_interface.pushtx(txsigned)
+    jm_single().bc_interface.pushtx(txsigned.serialize())
 
 if __name__ == "__main__":
     main()

test/test_segwit.py

@@ -7,7 +7,7 @@ from common import make_wallets
 from pprint import pformat
 import jmbitcoin as btc
 import pytest
-from jmbase import get_log
+from jmbase import get_log, hextobin, bintohex
 from jmclient import load_test_config, jm_single, LegacyWallet
 log = get_log()
 
@@ -19,9 +19,9 @@ def test_segwit_valid_txs(setup_segwit):
     for j in valid_txs:
         if len(j) < 2:
             continue
-        deserialized_tx = btc.deserialize(str(j[1]))
+        deserialized_tx = btc.CMutableTransaction.deserialize(hextobin(j[1]))
         print(pformat(deserialized_tx))
-        assert btc.serialize(deserialized_tx) == str(j[1])
+        assert deserialized_tx.serialize() == hextobin(j[1])
         #TODO use bcinterface to decoderawtransaction
         #and compare the json values
 
@@ -63,8 +63,8 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount,
     sw_wallet_service = make_wallets(1, [[len(segwit_ins), 0, 0, 0, 0]], segwit_amt)[0]['wallet']
     sw_wallet_service.sync_wallet(fast=True)
 
-    nsw_utxos = nsw_wallet_service.get_utxos_by_mixdepth(hexfmt=False)[MIXDEPTH]
-    sw_utxos = sw_wallet_service.get_utxos_by_mixdepth(hexfmt=False)[MIXDEPTH]
+    nsw_utxos = nsw_wallet_service.get_utxos_by_mixdepth()[MIXDEPTH]
+    sw_utxos = sw_wallet_service.get_utxos_by_mixdepth()[MIXDEPTH]
     assert len(o_ins) <= len(nsw_utxos), "sync failed"
     assert len(segwit_ins) <= len(sw_utxos), "sync failed"
 
@@ -92,8 +92,8 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount,
 
     # FIXME: encoding mess, mktx should accept binary input formats
     tx_ins = []
-    for i, (txid, data) in sorted(all_ins.items(), key=lambda x: x[0]):
-        tx_ins.append('{}:{}'.format(binascii.hexlify(txid[0]).decode('ascii'), txid[1]))
+    for i, (txin, data) in sorted(all_ins.items(), key=lambda x: x[0]):
+        tx_ins.append(txin)
 
     # create outputs
     FEE = 50000
@@ -104,11 +104,11 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount,
     change_amt = total_amt_in_sat - amount - FEE
 
     tx_outs = [
-        {'script': binascii.hexlify(cj_script).decode('ascii'),
+        {'address': nsw_wallet_service.script_to_addr(cj_script),
          'value': amount},
-        {'script': binascii.hexlify(change_script).decode('ascii'),
+        {'address': nsw_wallet_service.script_to_addr(change_script),
          'value': change_amt}]
-    tx = btc.deserialize(btc.mktx(tx_ins, tx_outs))
+    tx = btc.mktx(tx_ins, tx_outs)
 
     # import new addresses to bitcoind
     jm_single().bc_interface.import_addresses(
@@ -120,18 +120,20 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount,
     for nsw_in_index in o_ins:
         inp = nsw_ins[nsw_in_index][1]
         scripts[nsw_in_index] = (inp['script'], inp['value'])
-    tx = nsw_wallet_service.sign_tx(tx, scripts)
+    success, msg = nsw_wallet_service.sign_tx(tx, scripts)
+    assert success, msg
 
     scripts = {}
     for sw_in_index in segwit_ins:
         inp = sw_ins[sw_in_index][1]
         scripts[sw_in_index] = (inp['script'], inp['value'])
-    tx = sw_wallet_service.sign_tx(tx, scripts)
+    success, msg = sw_wallet_service.sign_tx(tx, scripts)
+    assert success, msg
 
     print(tx)
 
     # push and verify
-    txid = jm_single().bc_interface.pushtx(btc.serialize(tx))
+    txid = jm_single().bc_interface.pushtx(tx.serialize())
     assert txid
 
     balances = jm_single().bc_interface.get_received_by_addr(