Native segwit support including basic BIP84 wallet.

* parsing of scripts and addresses of all segwit types.
* ability to verify arbitrary tx inputs for all segwit types.
* simplify mktx syntax (only allow [],[] args)
* add p2wpkh and p2wsh spending test and fixes to sign calls in wallet
* simplify wallet signing calls in cryptoengine
* add p2wpkh engine, add bip84 wallet

jmbitcoin/jmbitcoin/secp256k1_main.py

@@ -435,6 +435,32 @@ def privkey_to_pubkey(priv, usehex=True):
 
 privtopub = privkey_to_pubkey
 
+@hexbin
+def is_valid_pubkey(pubkey, usehex, require_compressed=False):
+    """ Returns True if the serialized pubkey is a valid secp256k1
+    pubkey serialization or False if not; returns False for an
+    uncompressed encoding if require_compressed is True.
+    """
+    # sanity check for public key
+    # see https://github.com/bitcoin/bitcoin/blob/master/src/pubkey.h
+    if require_compressed:
+        valid_uncompressed = False
+    elif len(pubkey) == 65 and pubkey[:1] in (b'\x04', b'\x06', b'\x07'):
+        valid_uncompressed = True
+    else:
+        valid_uncompressed = False
+
+    if not ((len(pubkey) == 33 and pubkey[:1] in (b'\x02', b'\x03')) or
+    valid_uncompressed):
+        return False
+    # serialization is valid, but we must ensure it corresponds
+    # to a valid EC point:
+    try:
+        dummy = secp256k1.PublicKey(pubkey)
+    except:
+        return False
+    return True
+
 @hexbin
 def multiply(s, pub, usehex, rawpub=True, return_serialized=True):
     '''Input binary compressed pubkey P(33 bytes)

jmbitcoin/jmbitcoin/secp256k1_transaction.py

@@ -8,9 +8,15 @@ import binascii
 import copy
 import re
 import os
+import struct
 from jmbitcoin.secp256k1_main import *
 from jmbitcoin.bech32 import *
 
+P2PKH_PRE, P2PKH_POST = b'\x76\xa9\x14', b'\x88\xac'
+P2SH_P2WPKH_PRE, P2SH_P2WPKH_POST = b'\xa9\x14', b'\x87'
+P2WPKH_PRE = b'\x00\x14'
+P2WSH_PRE = b'\x00\x16'
+
 # Transaction serialization and deserialization
 
 def deserialize(txinp):
@@ -108,12 +114,11 @@ def deserialize(txinp):
     return obj
 
 def serialize(tx):
-    """Assumes a deserialized transaction in which all
-    dictionary values are decoded hex strings or numbers.
-    Rationale: mixing raw bytes/hex/strings in dict objects causes
-    complexity; since the dict tx object has to be inspected
-    in this function, avoid complicated logic elsewhere by making
-    all conversions to raw byte strings happen here.
+    """ Accepts a dict in which the "outpoint","hash"
+    "script" and "txinwitness" entries can be in binary
+    or hex. If any "hash" field is in hex, the serialized
+    output is in hex, otherwise the serialization output
+    is in binary. Below table assumes all hex.
     Table of dictionary keys and type for the value:
     ================================================
     version: int
@@ -129,7 +134,9 @@ def serialize(tx):
     outs[0]["value"]: int
     locktime: int
     =================================================
-    Returned serialized transaction is a byte string.
+    Returned serialized transaction is a byte string,
+    or a hex encoded string, according to the above
+    hash check.
     """
     #Because we are manipulating the dict in-place, need
     #to work on a copy
@@ -184,6 +191,9 @@ def serialize(tx):
             items = inp["txinwitness"]
             o.write(num_to_var_int(len(items)))
             for item in items:
+                if item is None:
+                    o.write(b'\x00')
+                    continue
                 if isinstance(item, basestring) and not isinstance(item, bytes):
                     item = binascii.unhexlify(item)
                 o.write(num_to_var_int(len(item)) + item)
@@ -203,7 +213,7 @@ SIGHASH_ANYONECANPAY = 0x80
 
 def segwit_signature_form(txobj, i, script, amount, hashcode=SIGHASH_ALL,
                           decoder_func=binascii.unhexlify):
-    """Given a deserialized transaction txobj, an input index i,
+    """ Given a deserialized transaction txobj, an input index i,
     which spends from a witness,
     a script for redemption and an amount in satoshis, prepare
     the version of the transaction to be hashed and signed.
@@ -310,8 +320,11 @@ def segwit_txid(tx, hashcode=None):
     return txhash(reserialized_tx, hashcode)
 
 def txhash(tx, hashcode=None, check_sw=True):
-    """Creates the appropriate sha256 hash as required
+    """ Creates the appropriate sha256 hash as required
     either for signing or calculating txids.
+    The hashcode argument is used to distinguish the case
+    where we are hashing for signing (sighashing); by default
+    it is None, and this indicates we are calculating a txid.
     If check_sw is True it checks the serialized format for
     segwit flag bytes, and produces the correct form for txid (not wtxid).
     """
@@ -356,9 +369,7 @@ def ecdsa_tx_verify(tx, sig, pub, hashcode=SIGHASH_ALL):
 
 # Scripts
 
-
 def mk_pubkey_script(addr):
-    # Keep the auxiliary functions around for altcoins' sake
     return '76a914' + b58check_to_hex(addr) + '88ac'
 
 
@@ -366,20 +377,36 @@ def mk_scripthash_script(addr):
     return 'a914' + b58check_to_hex(addr) + '87'
 
 def segwit_scriptpubkey(witver, witprog):
-    """Construct a Segwit scriptPubKey for a given witness program."""
-    x = bytes([witver + 0x50 if witver else 0, len(witprog)] + witprog)
-    return x
+    """ Construct a Segwit scriptPubKey for a given witness program.
+    """
+    return bytes([witver + 0x50 if witver else 0, len(witprog)] + witprog)
 
-def mk_native_segwit_script(addr):
-    hrp = addr[:2]
+def mk_native_segwit_script(hrp, addr):
+    """ Returns scriptPubKey as hex encoded string,
+    given a valid bech32 address and human-readable-part,
+    else throws an Exception if the arguments do not
+    match this pattern.
+    """
     ver, prog =  bech32addr_decode(hrp, addr)
+    if ver is None:
+        # This error cannot occur if this function is
+        # called from `address_to_script`
+        raise Exception("Invalid native segwit script")
     scriptpubkey = segwit_scriptpubkey(ver, prog)
     return binascii.hexlify(scriptpubkey).decode('ascii')
+
 # Address representation to output script
 
 def address_to_script(addr):
-    if addr[:2] in ['bc', 'tb']:
-        return mk_native_segwit_script(addr)
+    """ Returns scriptPubKey as a hex-encoded string for
+    a given Bitcoin address.
+    """
+    x = bech32_decode(addr)
+    # Any failure (because it's not a bech32 address)
+    # returns (None, None)
+    if x[0] and x[1]:
+        return mk_native_segwit_script(x[0], addr)
+
     if addr[0] == '3' or addr[0] == '2':
         return mk_scripthash_script(addr)
     else:
@@ -388,57 +415,145 @@ def address_to_script(addr):
 # Output script to address representation
 
 def is_p2pkh_script(script):
-    if script[:3] == b'\x76\xa9\x14' and script[-2:] == b'\x88\xac' and len(
+    """ Given a script as bytes, returns True if the script
+    matches the pay-to-pubkey-hash pattern (using opcodes),
+    otherwise returns False.
+    """
+    if not isinstance(script, bytes):
+        script = binascii.unhexlify(script)
+    if script[:3] == P2PKH_PRE and script[-2:] == P2PKH_POST and len(
             script) == 25:
         return True
     return False
 
 def is_segwit_native_script(script):
-    """Is scriptPubkey of form P2WPKH or P2WSH"""
-    if script[:2] in [b'\x00\x14', b'\x00\x20']:
+    """Is script, as bytes, of form P2WPKH or P2WSH;
+    see BIP141 for definitions of 2 current valid scripts.
+    """
+    if not isinstance(script, bytes):
+        script = binascii.unhexlify(script)
+
+    if (script[:2] == P2WPKH_PRE and len(script) == 22) or (
+        script[:2] == P2WSH_PRE and len(script) == 34):
         return True
     return False
 
 def script_to_address(script, vbyte=0, witver=0):
+    """ Given a hex or bytes script, and optionally a version byte
+    (for P2SH) and/or a witness version (for native segwit witness
+    programs), convert to a valid address (either bech32 or Base58CE).
+    An important tacit assumption: anything which does not match
+    the parsing rules of native segwit, or pay-to-pubkey-hash, is
+    assumed to be p2sh.
+    Note also that this translates scriptPubKeys to addresses, not
+    redeemscripts to p2sh addresses; for that, use p2sh_scriptaddr.
+    """
     if not isinstance(script, bytes):
         script = binascii.unhexlify(script)
     if is_segwit_native_script(script):
-        #hrp interpreted from the vbyte entry, TODO this should be cleaner.
+        #hrp interpreted from the vbyte entry, TODO: better way?
         if vbyte in [0, 5]:
             hrp = 'bc'
+        elif vbyte == 100:
+            hrp = 'bcrt'
         else:
             hrp = 'tb'
         return bech32addr_encode(hrp=hrp, witver=witver,
-                                 witprog=[ord(x) for x in script[2:]])
+        witprog=struct.unpack('{}B'.format(len(script[2:])).encode(
+            'ascii'), script[2:]))
     if is_p2pkh_script(script):
-        return bin_to_b58check(script[3:-2], vbyte)  # pubkey hash addresses
+        return bin_to_b58check(script[3:-2], vbyte)
     else:
         # BIP0016 scripthash addresses: requires explicit vbyte set
         if vbyte == 0: raise Exception("Invalid version byte for P2SH")
         return bin_to_b58check(script[2:-1], vbyte)
 
+def pubkey_to_script(pubkey, script_pre, script_post=b'',
+                     require_compressed=False):
+    """ Generic conversion from a binary pubkey serialization
+    to a corresponding binary scriptPubKey for the pubkeyhash case.
+    """
+    if not is_valid_pubkey(pubkey, False,
+                           require_compressed=require_compressed):
+        raise Exception("Invalid pubkey.")
+    h = bin_hash160(pubkey)
+    assert len(h) == 0x14
+    assert script_pre[-1:] == b'\x14'
+    return script_pre + h + script_post
+
+def p2sh_scriptaddr(script, magicbyte=5):
+    if not isinstance(script, bytes):
+        script = binascii.unhexlify(script)
+    return hex_to_b58check(hash160(script), magicbyte)
+
+def pubkey_to_p2pkh_script(pub, require_compressed=False):
+    """ Construct a pay-to-pubkey-hash scriptPubKey
+    given a single pubkey. Script returned in binary.
+    The require compressed flag may be used to disallow
+    uncompressed keys, e.g. in constructing a segwit
+    scriptCode field.
+    """
+    if not isinstance(pub, bytes):
+        pub = binascii.unhexlify(pub)
+    return pubkey_to_script(pub, P2PKH_PRE, P2PKH_POST)
+
 def pubkey_to_p2sh_p2wpkh_script(pub):
+    """ Construct a nested-pay-to-witness-pubkey-hash
+    scriptPubKey given a single pubkey.
+    Script returned in binary.
+    """
     if not isinstance(pub, bytes):
         pub = binascii.unhexlify(pub)
-    return "0014" + hash160(pub)
+    wscript = pubkey_to_p2wpkh_script(pub)
+    return P2SH_P2WPKH_PRE + bin_hash160(wscript) + P2SH_P2WPKH_POST
 
 def pubkey_to_p2sh_p2wpkh_address(pub, magicbyte=5):
+    """ Construct a nested-pay-to-witness-pubkey-hash
+    address given a single pubkey; magicbyte defines
+    network as for any p2sh address.
+    """    
     if not isinstance(pub, bytes):
         pub = binascii.unhexlify(pub)
-    script = pubkey_to_p2sh_p2wpkh_script(pub)
+    script = pubkey_to_p2wpkh_script(pub)
     return p2sh_scriptaddr(script, magicbyte=magicbyte)
 
-def p2sh_scriptaddr(script, magicbyte=5):
-    if not isinstance(script, bytes):
-        script = binascii.unhexlify(script)
-    return hex_to_b58check(hash160(script), magicbyte)
-
-
-scriptaddr = p2sh_scriptaddr
-
+def pubkey_to_p2wpkh_script(pub):
+    """ Construct a pay-to-witness-pubkey-hash
+    scriptPubKey given a single pubkey. Note that
+    this is the witness program (version 0). Script
+    is returned in binary.
+    """
+    if not isinstance(pub, bytes):
+        pub = binascii.unhexlify(pub)
+    return pubkey_to_script(pub, P2WPKH_PRE,
+                            require_compressed=True)
+
+def pubkey_to_p2wpkh_address(pub):
+    """ Construct a pay-to-witness-pubkey-hash
+    address (bech32) given a single pubkey.
+    """    
+    script = pubkey_to_p2wpkh_script(pub)
+    return script_to_address(script)
+
+def pubkeys_to_p2wsh_script(pubs):
+    """ Given a list of N pubkeys, constructs an N of N
+    multisig scriptPubKey of type pay-to-witness-script-hash.
+    No other scripts than N-N multisig supported as of now.
+    """
+    N = len(pubs)
+    script = mk_multisig_script(pubs, N)
+    return P2WSH_PRE + bin_sha256(binascii.unhexlify(script))
+
+def pubkeys_to_p2wsh_address(pubs):
+    """ Given a list of N pubkeys, constructs an N of N
+    multisig address of type pay-to-witness-script-hash.
+    No other scripts than N-N multisig supported as of now.
+    """
+    script = pubkeys_to_p2wsh_script(pubs)
+    return script_to_address(script)
 
 def deserialize_script(scriptinp):
-    """Note that this is not used internally, in
+    """ Note that this is not used internally, in
     the jmbitcoin package, to deserialize() transactions;
     its function is only to allow parsing of scripts by
     external callers. Thus, it returns in the format used
@@ -521,80 +636,118 @@ def serialize_script(script):
         return result
 
 
-def mk_multisig_script(*args):  # [pubs],k or pub1,pub2...pub[n],k
-    if isinstance(args[0], list):
-        pubs, k = args[0], int(args[1])
-    else:
-        pubs = list(filter(lambda x: len(str(x)) >= 32, args))
-        k = int(args[len(pubs)])
+def mk_multisig_script(pubs, k):
+    """ Given a list of pubkeys and an integer k,
+    construct a multisig script for k of N, where N is
+    the length of the list `pubs`; script is returned
+    as hex string.
+    """
     return serialize_script([k] + pubs + [len(pubs)]) + 'ae'
 
-# Signing and verifying
 
+# Signing and verifying
 
-def verify_tx_input(tx, i, script, sig, pub, witness=None, amount=None):
+def verify_tx_input(tx, i, script, sig, pub, scriptCode=None, amount=None):
+    """ Given a hex-serialized transaction tx, an integer index i,
+    a script (see more on this below), signature and pubkey, and optionally
+    a segwit scriptCode and  amount in satoshis (that flags segwit usage),
+    we return True if and only if the signature and pubkey is valid for
+    this tx input.
+    Note on 'script' and 'scriptCode':
+    For p2pkh, 'script' should be the output script (scriptPubKey), and
+    the scriptCode should be left as the default None.
+    For p2sh non-segwit multisig, the script should be the
+    output of mk_multisig_script (and scriptCode and amount None).
+    For either nested (p2sh) or not p2wpkh and p2wsh, the scriptCode is the
+    preimage of the scriptPubKey hash (the witness program, which here
+    has been passed in in the 'script' parameter).
+
+    Note that for the p2sh-p2wsh and p2wsh cases, a redeem script that is not
+    multisig should still be correctly handled here; the codebase restricts
+    the creation of such signatures to multisig only, but *verification* should
+    function correctly with any custom script.
+
+    TODO: Also note that the OP_CODESEPARATOR special case outlined in BIP143
+    is not yet covered.
+    """
+    assert isinstance(i, int)
+    assert i >= 0
     if not isinstance(tx, bytes):
         tx = binascii.unhexlify(tx)
     if not isinstance(script, bytes):
         script = binascii.unhexlify(script)
+    if scriptCode is not None and not isinstance(scriptCode, bytes):
+        scriptCode = binascii.unhexlify(scriptCode)
     if isinstance(sig, bytes):
         sig = binascii.hexlify(sig).decode('ascii')
     if isinstance(pub, bytes):
         pub = binascii.hexlify(pub).decode('ascii')
-    if witness:
-        if isinstance(witness, bytes):
-            witness = safe_hexlify(witness)
+
     hashcode = binascii.unhexlify(sig[-2:])
-    if witness and amount:
-        #TODO assumes p2sh wrapped segwit input; OK for JM wallets
-        scriptCode = binascii.unhexlify("76a914"+hash160(binascii.unhexlify(pub))+"88ac")
-        modtx = segwit_signature_form(deserialize(tx), int(i),
-                                      scriptCode, amount, hashcode, decoder_func=lambda x: x)
+
+    if amount:
+        modtx = segwit_signature_form(deserialize(tx), i,
+                    scriptCode, amount, hashcode, decoder_func=lambda x: x)
     else:
-        modtx = signature_form(tx, int(i), script, hashcode)
+        modtx = signature_form(tx, i, script, hashcode)
     return ecdsa_tx_verify(modtx, sig, pub, hashcode)
 
 
-def sign(tx, i, priv, hashcode=SIGHASH_ALL, usenonce=None, amount=None):
-    i = int(i)
+def sign(tx, i, priv, hashcode=SIGHASH_ALL, usenonce=None, amount=None,
+         native=False):
+    """
+    Given a serialized transaction tx, an input index i, and a privkey
+    in bytes or hex, returns a serialized transaction, in hex always,
+    into which the signature and/or witness has been inserted. The field
+    `amount` flags whether segwit signing is to be done, and the field
+    `native` flags that native segwit p2wpkh signing is to be done. Note
+    that signing multisig is to be done with the alternative functions
+    multisign or p2wsh_multisign (and non N of N multisig scripthash
+    signing is not currently supported).
+    """
     if isinstance(tx, basestring) and not isinstance(tx, bytes):
         tx = binascii.unhexlify(tx)
-        hexout = True
-    else:
-        hexout = False
     if len(priv) <= 33:
         priv = safe_hexlify(priv)
     if amount:
-        return p2sh_p2wpkh_sign(tx, i, priv, amount, hashcode=hashcode,
-                                usenonce=usenonce)
-    pub = privkey_to_pubkey(priv, True)
-    address = pubkey_to_address(pub)
-    signing_tx = signature_form(tx, i, mk_pubkey_script(address), hashcode)
-    sig = ecdsa_tx_sign(signing_tx, priv, hashcode, usenonce=usenonce)
-    txobj = deserialize(tx)
-    txobj["ins"][i]["script"] = serialize_script([sig, pub])
-    serobj = serialize(txobj)
-    if hexout and isinstance(serobj, basestring) and isinstance(serobj, bytes):
+        serobj = p2wpkh_sign(tx, i, priv, amount, hashcode=hashcode,
+                               usenonce=usenonce, native=native)
+    else:
+        pub = privkey_to_pubkey(priv, True)
+        address = pubkey_to_address(pub)
+        signing_tx = signature_form(tx, i, mk_pubkey_script(address), hashcode)
+        sig = ecdsa_tx_sign(signing_tx, priv, hashcode, usenonce=usenonce)
+        txobj = deserialize(tx)
+        txobj["ins"][i]["script"] = serialize_script([sig, pub])
+        serobj = serialize(txobj)
+    if isinstance(serobj, basestring) and isinstance(serobj, bytes):
         return binascii.hexlify(serobj).decode('ascii')
-    elif not hexout and not isinstance(serobj, bytes):
-        return binascii.unhexlify(serobj)
     else:
         return serobj
 
-def p2sh_p2wpkh_sign(tx, i, priv, amount, hashcode=SIGHASH_ALL, usenonce=None):
+def p2wpkh_sign(tx, i, priv, amount, hashcode=SIGHASH_ALL, native=False,
+                usenonce=None):
     """Given a serialized transaction, index, private key in hex,
     amount in satoshis and optionally hashcode, return the serialized
     transaction containing a signature and witness for this input; it's
-    assumed that the input is of type pay-to-witness-pubkey-hash nested in p2sh.
+    assumed that the input is of type pay-to-witness-pubkey-hash.
+    If native is False, it's treated as p2sh nested.
     """
     pub = privkey_to_pubkey(priv)
-    script = pubkey_to_p2sh_p2wpkh_script(pub)
-    scriptCode = "76a914"+hash160(binascii.unhexlify(pub))+"88ac"
+    # Convert the input tx and script to hex so that the deserialize()
+    # call creates hex-encoded fields
+    script = binascii.hexlify(pubkey_to_p2wpkh_script(pub)).decode('ascii')
+    scriptCode = binascii.hexlify(pubkey_to_p2pkh_script(pub)).decode('ascii')
+    if isinstance(tx, bytes):
+        tx = binascii.hexlify(tx).decode('ascii')
     signing_tx = segwit_signature_form(deserialize(tx), i, scriptCode, amount,
                                        hashcode=hashcode)
     sig = ecdsa_tx_sign(signing_tx, priv, hashcode, usenonce=usenonce)
     txobj = deserialize(tx)
-    txobj["ins"][i]["script"] = "16"+script
+    if not native:
+        txobj["ins"][i]["script"] = "16" + script
+    else:
+        txobj["ins"][i]["script"] = ""
     txobj["ins"][i]["txinwitness"] = [sig, pub]
     return serialize(txobj)
 
@@ -611,14 +764,45 @@ def signall(tx, priv):
     return tx
 
 
-def multisign(tx, i, script, pk, hashcode=SIGHASH_ALL):
+def multisign(tx, i, script, pk, amount=None, hashcode=SIGHASH_ALL):
+    """ Tx is assumed to be serialized. The script passed here is
+    the redeemscript, for example the output of mk_multisig_script.
+    pk is the private key, and must be passed in hex.
+    If amount is not None, the output of p2wsh_multisign is returned.
+    What is returned is a single signature.
+    """
     if isinstance(tx, str):
         tx = binascii.unhexlify(tx)
     if isinstance(script, str):
         script = binascii.unhexlify(script)
+    if amount:
+        return p2wsh_multisign(tx, i, script, pk, amount, hashcode)
     modtx = signature_form(tx, i, script, hashcode)
     return ecdsa_tx_sign(modtx, pk, hashcode)
 
+def p2wsh_multisign(tx, i, script, pk, amount, hashcode=SIGHASH_ALL):
+    """ See note to multisign for the value to pass in as `script`.
+    Tx is assumed to be serialized.
+    """
+    modtx = segwit_signature_form(deserialize(tx), i, script, amount,
+                                  hashcode, decoder_func=lambda x: x)
+    return ecdsa_tx_sign(modtx, pk, hashcode)
+
+def apply_p2wsh_multisignatures(tx, i, script, sigs):
+    """Sigs must be passed in as a list, and must be a
+    complete list for this multisig, in the same order
+    as the list of pubkeys when creating the scriptPubKey.
+    """
+    if isinstance(script, str):
+        script = binascii.unhexlify(script)
+    sigs = [binascii.unhexlify(x) if x[:2] == '30' else x for x in sigs]
+    if isinstance(tx, str):
+        return safe_hexlify(apply_p2wsh_multisignatures(
+            binascii.unhexlify(tx), i, script, sigs))
+    txobj = deserialize(tx)
+    txobj["ins"][i]["script"] = ""
+    txobj["ins"][i]["txinwitness"] = [None] + sigs + [script]
+    return serialize(txobj)
 
 def apply_multisignatures(*args):
     # tx,i,script,sigs OR tx,i,script,sig1,sig2...,sig[n]
@@ -636,21 +820,8 @@ def apply_multisignatures(*args):
     txobj["ins"][i]["script"] = serialize_script([None] + sigs + [script])
     return serialize(txobj)
 
-
-def is_inp(arg):
-    return len(arg) > 64 or "output" in arg or "outpoint" in arg
-
-
-def mktx(*args):
-    # [in0, in1...],[out0, out1...] or in0, in1 ... out0 out1 ...
-    ins, outs = [], []
-    for arg in args:
-        if isinstance(arg, list):
-            for a in arg:
-                (ins if is_inp(a) else outs).append(a)
-        else:
-            (ins if is_inp(arg) else outs).append(arg)
-    txobj = {"locktime": 0, "version": 1, "ins": [], "outs": []}
+def mktx(ins, outs, version=1):
+    txobj = {"locktime": 0, "version": version, "ins": [], "outs": []}
     for i in ins:
         if isinstance(i, dict) and "outpoint" in i:
             txobj["ins"].append(i)

jmclient/jmclient/__init__.py

@@ -14,7 +14,8 @@ from .old_mnemonic import mn_decode, mn_encode
 from .taker import Taker
 from .wallet import (Mnemonic, estimate_tx_fee, WalletError, BaseWallet, ImportWalletMixin,
                      BIP39WalletMixin, BIP32Wallet, BIP49Wallet, LegacyWallet,
-                     SegwitLegacyWallet, UTXOManager, WALLET_IMPLEMENTATIONS)
+                     SegwitWallet, SegwitLegacyWallet, UTXOManager,
+                     WALLET_IMPLEMENTATIONS)
 from .storage import (Argon2Hash, Storage, StorageError,
                       StoragePasswordError, VolatileStorage)
 from .cryptoengine import BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, EngineError

jmclient/jmclient/cryptoengine.py

@@ -3,7 +3,7 @@ from __future__ import (absolute_import, division,
 from builtins import * # noqa: F401
 
 
-from binascii import hexlify, unhexlify
+from binascii import unhexlify
 from collections import OrderedDict
 import struct
 
@@ -17,57 +17,21 @@ NET_MAP = {'mainnet': NET_MAINNET, 'testnet': NET_TESTNET}
 WIF_PREFIX_MAP = {'mainnet': b'\x80', 'testnet': b'\xef'}
 BIP44_COIN_MAP = {'mainnet': 2**31, 'testnet': 2**31 + 1}
 
-
-#
-# library stuff that should be in btc/jmbitcoin
-#
-
-
-P2PKH_PRE, P2PKH_POST = b'\x76\xa9\x14', b'\x88\xac'
-P2SH_P2WPKH_PRE, P2SH_P2WPKH_POST = b'\xa9\x14', b'\x87'
-P2WPKH_PRE = b'\x00\x14'
-
-
-def _pubkey_to_script(pubkey, script_pre, script_post=b''):
-    # sanity check for public key
-    # see https://github.com/bitcoin/bitcoin/blob/master/src/pubkey.h
-    if not ((len(pubkey) == 33 and pubkey[:1] in (b'\x02', b'\x03')) or
-            (len(pubkey) == 65 and pubkey[:1] in (b'\x04', b'\x06', b'\x07'))):
-        raise Exception("Invalid public key!")
-    h = btc.bin_hash160(pubkey)
-    assert len(h) == 0x14
-    assert script_pre[-1:] == b'\x14'
-    return script_pre + h + script_post
-
-
-def pubkey_to_p2pkh_script(pubkey):
-    return _pubkey_to_script(pubkey, P2PKH_PRE, P2PKH_POST)
-
-
-def pubkey_to_p2sh_p2wpkh_script(pubkey):
-    wscript = pubkey_to_p2wpkh_script(pubkey)
-    return P2SH_P2WPKH_PRE + btc.bin_hash160(wscript) + P2SH_P2WPKH_POST
-
-
-def pubkey_to_p2wpkh_script(pubkey):
-    return _pubkey_to_script(pubkey, P2WPKH_PRE)
-
-
 def detect_script_type(script):
-    if script.startswith(P2PKH_PRE) and script.endswith(P2PKH_POST) and\
-            len(script) == 0x14 + len(P2PKH_PRE) + len(P2PKH_POST):
+    if script.startswith(btc.P2PKH_PRE) and script.endswith(btc.P2PKH_POST) and\
+       len(script) == 0x14 + len(btc.P2PKH_PRE) + len(btc.P2PKH_POST):
         return TYPE_P2PKH
-    elif (script.startswith(P2SH_P2WPKH_PRE) and
-          script.endswith(P2SH_P2WPKH_POST) and
-          len(script) == 0x14 + len(P2SH_P2WPKH_PRE) + len(P2SH_P2WPKH_POST)):
+    elif (script.startswith(btc.P2SH_P2WPKH_PRE) and
+          script.endswith(btc.P2SH_P2WPKH_POST) and
+          len(script) == 0x14 + len(btc.P2SH_P2WPKH_PRE) + len(
+              btc.P2SH_P2WPKH_POST)):
         return TYPE_P2SH_P2WPKH
-    elif script.startswith(P2WPKH_PRE) and\
-            len(script) == 0x14 + len(P2WPKH_PRE):
+    elif script.startswith(btc.P2WPKH_PRE) and\
+            len(script) == 0x14 + len(btc.P2WPKH_PRE):
         return TYPE_P2WPKH
     raise EngineError("Unknown script type for script '{}'"
                       .format(hexlify(script)))
 
-
 class classproperty(object):
     """
     from https://stackoverflow.com/a/5192374
@@ -246,24 +210,17 @@ class BTC_P2PKH(BTCEngine):
 
     @classmethod
     def pubkey_to_script(cls, pubkey):
-        return pubkey_to_p2pkh_script(pubkey)
+        return btc.pubkey_to_p2pkh_script(pubkey)
+
+    @classmethod
+    def pubkey_to_script_code(cls, pubkey):
+        raise EngineError("Script code does not apply to legacy wallets")
 
     @classmethod
     def sign_transaction(cls, tx, index, privkey, *args, **kwargs):
         hashcode = kwargs.get('hashcode') or btc.SIGHASH_ALL
-
-        pubkey = cls.privkey_to_pubkey(privkey)
-        script = cls.pubkey_to_script(pubkey)
-
-        signing_tx = btc.serialize(btc.signature_form(tx, index, script,
-                                                      hashcode=hashcode))
-        # FIXME: encoding mess
-        sig = unhexlify(btc.ecdsa_tx_sign(signing_tx, hexlify(privkey).decode('ascii'),
-                                          **kwargs))
-
-        tx['ins'][index]['script'] = btc.serialize_script([sig, pubkey])
-
-        return tx
+        return btc.sign(btc.serialize(tx), index, privkey,
+                        hashcode=hashcode, amount=None, native=False)
 
 
 class BTC_P2SH_P2WPKH(BTCEngine):
@@ -276,64 +233,61 @@ class BTC_P2SH_P2WPKH(BTCEngine):
 
     @classmethod
     def pubkey_to_script(cls, pubkey):
-        return pubkey_to_p2sh_p2wpkh_script(pubkey)
+        return btc.pubkey_to_p2sh_p2wpkh_script(pubkey)
+
+    @classmethod
+    def pubkey_to_script_code(cls, pubkey):
+        """ As per BIP143, the scriptCode for the p2wpkh
+        case is "76a914+hash160(pub)+"88ac" as per the
+        scriptPubKey of the p2pkh case.
+        """
+        return btc.pubkey_to_p2pkh_script(pubkey, require_compressed=True)
 
     @classmethod
     def sign_transaction(cls, tx, index, privkey, amount,
                          hashcode=btc.SIGHASH_ALL, **kwargs):
         assert amount is not None
-
-        pubkey = cls.privkey_to_pubkey(privkey)
-        wpkscript = pubkey_to_p2wpkh_script(pubkey)
-        pkscript = pubkey_to_p2pkh_script(pubkey)
-
-        signing_tx = btc.segwit_signature_form(tx, index, pkscript, amount,
-                                               hashcode=hashcode,
-                                               decoder_func=lambda x: x)
-        # FIXME: encoding mess
-        sig = unhexlify(btc.ecdsa_tx_sign(signing_tx, hexlify(privkey).decode('ascii'),
-                                          hashcode=hashcode, **kwargs))
-
-        assert len(wpkscript) == 0x16
-        tx['ins'][index]['script'] = b'\x16' + wpkscript
-        tx['ins'][index]['txinwitness'] = [sig, pubkey]
-
-        return tx
-
+        return btc.sign(btc.serialize(tx), index, privkey,
+                        hashcode=hashcode, amount=amount, native=False)
 
 class BTC_P2WPKH(BTCEngine):
+
     @classproperty
     def VBYTE(cls):
-        return btc.BTC_P2SH_VBYTE[get_network()]
+        """Note that vbyte is needed in the native segwit case
+        to decide the value of the 'human readable part' of the
+        bech32 address. If it's 0 or 5 we use 'bc', else we use
+        'tb' for testnet bitcoin; so it doesn't matter if we use
+        the P2PK vbyte or the P2SH one.
+        However, regtest uses 'bcrt' only (and fails on 'tb'),
+        so bitcoin.script_to_address currently uses an artificial
+        value 100 to flag that case.
+        This means that for testing, this value must be explicitly
+        overwritten.
+        """
+        return btc.BTC_P2PK_VBYTE[get_network()]
 
     @classmethod
     def pubkey_to_script(cls, pubkey):
-        return pubkey_to_p2wpkh_script(pubkey)
+        return btc.pubkey_to_p2wpkh_script(pubkey)
+
+    @classmethod
+    def pubkey_to_script_code(cls, pubkey):
+        """ As per BIP143, the scriptCode for the p2wpkh
+        case is "76a914+hash160(pub)+"88ac" as per the
+        scriptPubKey of the p2pkh case.
+        """
+        return btc.pubkey_to_p2pkh_script(pubkey, require_compressed=True)
 
     @classmethod
     def sign_transaction(cls, tx, index, privkey, amount,
                          hashcode=btc.SIGHASH_ALL, **kwargs):
         assert amount is not None
-        raise NotImplementedError("The following code is completely untested")
-
-        pubkey = cls.privkey_to_pubkey(privkey)
-        script = cls.pubkey_to_script(pubkey)
-
-        signing_tx = btc.segwit_signature_form(tx, index, script, amount,
-                                               hashcode=hashcode,
-                                               decoder_func=lambda x: x)
-        # FIXME: encoding mess
-        sig = unhexlify(btc.ecdsa_tx_sign(signing_tx, hexlify(privkey),
-                                          hashcode=hashcode, **kwargs))
-
-        tx['ins'][index]['script'] = script
-        tx['ins'][index]['txinwitness'] = [sig, pubkey]
-
-        return tx
-
+        return btc.sign(btc.serialize(tx), index, privkey,
+                        hashcode=hashcode, amount=amount, native=True)
 
 ENGINES = {
     TYPE_P2PKH: BTC_P2PKH,
     TYPE_P2SH_P2WPKH: BTC_P2SH_P2WPKH,
     TYPE_P2WPKH: BTC_P2WPKH
-}
+}

jmclient/jmclient/maker.py

@@ -139,14 +139,30 @@ class Maker(object):
             our_inputs[index] = (script, amount)
 
         txs = self.wallet.sign_tx(btc.deserialize(unhexlify(txhex)), our_inputs)
-
         for index in our_inputs:
-            sigmsg = txs['ins'][index]['script']
+            sigmsg = unhexlify(txs['ins'][index]['script'])
             if 'txinwitness' in txs['ins'][index]:
-                #We prepend the witness data since we want (sig, pub, scriptCode);
-                #also, the items in witness are not serialize_script-ed.
-                sigmsg = b''.join(btc.serialize_script_unit(x)
-                                  for x in txs['ins'][index]['txinwitness']) + sigmsg
+                # Note that this flag only implies that the transaction
+                # *as a whole* is using segwit serialization; it doesn't
+                # imply that this specific input is segwit type (to be
+                # fully general, we allow that even our own wallet's
+                # inputs might be of mixed type). So, we catch the EngineError
+                # which is thrown by non-segwit types. This way the sigmsg
+                # will only contain the scriptSig field if the wallet object
+                # decides it's necessary/appropriate for this specific input
+                # If it is segwit, we prepend the witness data since we want
+                # (sig, pub, witnessprogram=scriptSig - note we could, better,
+                # pass scriptCode here, but that is not backwards compatible,
+                # as the taker uses this third field and inserts it into the
+                # transaction scriptSig), else (non-sw) the !sig message remains
+                # unchanged as (sig, pub).
+                try:
+                    scriptSig = btc.pubkey_to_p2wpkh_script(txs['ins'][index]['txinwitness'][1])
+                    sigmsg = b''.join(btc.serialize_script_unit(
+                x) for x in txs['ins'][index]['txinwitness'] + [scriptSig])
+                except IndexError:
+                    #the sigmsg was already set before the segwit check
+                    pass
             sigs.append(base64.b64encode(sigmsg).decode('ascii'))
         return (True, sigs)
 

jmclient/jmclient/taker.py

@@ -554,7 +554,7 @@ class Taker(object):
         for i, u in iteritems(utxo):
             if utxo_data[i] is None:
                 continue
-            #Check if the sender serialize_scripted the witness
+            #Check if the sender serialize_scripted the scriptCode
             #item into the sig message; if so, also pick up the amount
             #from the utxo data retrieved from the blockchain to verify
             #the segwit-style signature. Note that this allows a mixed
@@ -567,24 +567,50 @@ class Taker(object):
                 break
             if len(sig_deserialized) == 2:
                 ver_sig, ver_pub = sig_deserialized
-                wit = None
+                scriptCode = None
             elif len(sig_deserialized) == 3:
-                ver_sig, ver_pub, wit =  sig_deserialized
+                ver_sig, ver_pub, scriptCode =  sig_deserialized
             else:
                 jlog.debug("Invalid signature message - more than 3 items")
                 break
-            ver_amt = utxo_data[i]['value'] if wit else None
+            ver_amt = utxo_data[i]['value'] if scriptCode else None
             sig_good = btc.verify_tx_input(txhex, u[0], utxo_data[i]['script'],
-                                               ver_sig, ver_pub, witness=wit,
-                                               amount=ver_amt)
+                            ver_sig, ver_pub, scriptCode=scriptCode, amount=ver_amt)
+
+            if ver_amt is not None and not sig_good:
+                # Special case to deal with legacy bots 0.5.0 or lower:
+                # the third field in the sigmessage was originally *not* the
+                # scriptCode, but the contents of tx['ins'][index]['script'],
+                # i.e. the witness program 0014... ; for this we can verify
+                # implicitly, as verify_tx_input used to, by reconstructing
+                # from the public key. For these cases, we can *assume* that
+                # the input is of type p2sh-p2wpkh; we call the jmbitcoin method
+                # directly, as we cannot assume that *our* wallet handles this.
+                scriptCode = hexlify(btc.pubkey_to_p2pkh_script(
+                    ver_pub, True)).decode('ascii')
+                sig_good = btc.verify_tx_input(txhex, u[0], utxo_data[i]['script'],
+                        ver_sig, ver_pub, scriptCode=scriptCode, amount=ver_amt)
+
             if sig_good:
                 jlog.debug('found good sig at index=%d' % (u[0]))
-                if wit:
+                if ver_amt:
+                    # Note that, due to the complexity of handling multisig or other
+                    # arbitrary script (considering sending multiple signatures OTW),
+                    # there is an assumption of p2sh-p2wpkh or p2wpkh, for the segwit
+                    # case.
                     self.latest_tx["ins"][u[0]]["txinwitness"] = [ver_sig, ver_pub]
-                    self.latest_tx["ins"][u[0]]["script"] = "16" + wit
+                    if btc.is_segwit_native_script(utxo_data[i]['script']):
+                        scriptSig = ""
+                    else:
+                        scriptSig = btc.serialize_script_unit(
+                            btc.pubkey_to_p2wpkh_script(ver_pub))
+                    self.latest_tx["ins"][u[0]]["script"] = scriptSig
                 else:
+                    # Non segwit (as per above comments) is limited only to single key,
+                    # p2pkh case.
                     self.latest_tx["ins"][u[0]]["script"] = sig
                 inserted_sig = True
+
                 # check if maker has sent everything possible
                 try:
                     self.utxos[nick].remove(u[1])
@@ -740,13 +766,7 @@ class Taker(object):
             script = self.wallet.addr_to_script(self.input_utxos[utxo]['address'])
             amount = self.input_utxos[utxo]['value']
             our_inputs[index] = (script, amount)
-
-        # FIXME: ugly hack
-        tx_bin = btc.deserialize(unhexlify(btc.serialize(self.latest_tx)))
-        self.wallet.sign_tx(tx_bin, our_inputs)
-
-        self.latest_tx = btc.deserialize(hexlify(btc.serialize(tx_bin)).decode('ascii'))
-
+        self.latest_tx = self.wallet.sign_tx(self.latest_tx, our_inputs)
 
     def push(self):
         tx = btc.serialize(self.latest_tx)

jmclient/jmclient/taker_utils.py

@@ -7,7 +7,7 @@ import pprint
 import os
 import time
 import numbers
-from binascii import hexlify, unhexlify
+from binascii import unhexlify
 from jmbase import get_log
 from .configure import jm_single, validate_address
 from .schedule import human_readable_schedule_entry, tweak_tumble_schedule,\
@@ -79,11 +79,12 @@ def direct_send(wallet, amount, mixdepth, destaddr, answeryes=False,
     log.info("Using a fee of : " + str(fee_est) + " satoshis.")
     if amount != 0:
         log.info("Using a change value of: " + str(changeval) + " satoshis.")
-    tx = sign_tx(wallet, mktx(list(utxos.keys()), outs), utxos)
-    txsigned = deserialize(tx)
+    txsigned = sign_tx(wallet, mktx(list(utxos.keys()), outs), utxos)
     log.info("Got signed transaction:\n")
-    log.info(tx + "\n")
     log.info(pformat(txsigned))
+    tx = serialize(txsigned)
+    log.info("In serialized form (for copy-paste):")
+    log.info(tx)
     actual_amount = amount if amount != 0 else total_inputs_val - fee_est
     log.info("Sends: " + str(actual_amount) + " satoshis to address: " + destaddr)
     if not answeryes:
@@ -112,12 +113,7 @@ def sign_tx(wallet, tx, utxos):
         script = wallet.addr_to_script(utxos[utxo]['address'])
         amount = utxos[utxo]['value']
         our_inputs[index] = (script, amount)
-
-    # FIXME: ugly hack
-    tx_bin = deserialize(unhexlify(serialize(stx)))
-    wallet.sign_tx(tx_bin, our_inputs)
-    return hexlify(serialize(tx_bin)).decode('ascii')
-
+    return wallet.sign_tx(deserialize(unhexlify(serialize(stx))), our_inputs)
 
 def import_new_addresses(wallet, addr_list):
     # FIXME: same code as in maker.py and taker.py

jmclient/jmclient/wallet.py

@@ -20,7 +20,8 @@ from numbers import Integral
 from .configure import jm_single
 from .support import select_gradual, select_greedy, select_greediest, \
     select
-from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH, ENGINES
+from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH,\
+    TYPE_P2WPKH, ENGINES
 from .support import get_random_bytes
 from . import mn_encode, mn_decode
 import jmbitcoin as btc
@@ -325,13 +326,14 @@ class BaseWallet(object):
             scripts: {input_index: (output_script, amount)}
             kwargs: additional arguments for engine.sign_transaction
         returns:
-            input transaction dict with added signatures
+            input transaction dict with added signatures, hex-encoded.
         """
         for index, (script, amount) in scripts.items():
             assert amount > 0
             path = self.script_to_path(script)
             privkey, engine = self._get_priv_from_path(path)
-            engine.sign_transaction(tx, index, privkey, amount, **kwargs)
+            tx = btc.deserialize(engine.sign_transaction(tx, index, privkey,
+                                                         amount, **kwargs))
         return tx
 
     @deprecated
@@ -371,6 +373,10 @@ class BaseWallet(object):
     def addr_to_script(cls, addr):
         return cls._ENGINE.address_to_script(addr)
 
+    @classmethod
+    def pubkey_to_script(cls, pubkey):
+        return cls._ENGINE.pubkey_to_script(pubkey)
+
     @classmethod
     def pubkey_to_addr(cls, pubkey):
         return cls._ENGINE.pubkey_to_address(pubkey)
@@ -381,6 +387,19 @@ class BaseWallet(object):
         engine = self._get_priv_from_path(path)[1]
         return engine.script_to_address(script)
 
+    def get_script_code(self, script):
+        """
+        For segwit wallets, gets the value of the scriptCode
+        parameter required (see BIP143) for sighashing; this is
+        required for protocols (like Joinmarket) where signature
+        verification materials must be communicated between wallets.
+        For non-segwit wallets, raises EngineError.
+        """
+        path = self.script_to_path(script)
+        priv, engine = self._get_priv_from_path(path)
+        pub = engine.privkey_to_pubkey(priv)
+        return engine.pubkey_to_script_code(pub)
+
     @classmethod
     def pubkey_has_address(cls, pubkey, addr):
         return cls._ENGINE.pubkey_has_address(pubkey, addr)
@@ -1354,12 +1373,16 @@ class LegacyWallet(ImportWalletMixin, BIP32Wallet):
         return self._key_ident, 0
 
 
-class BIP49Wallet(BIP32Wallet):
-    _BIP49_PURPOSE = 2**31 + 49
-    _ENGINE = ENGINES[TYPE_P2SH_P2WPKH]
+
+class BIP32PurposedWallet(BIP32Wallet):
+    """ A class to encapsulate cases like
+    BIP44, 49 and 84, all of which are derivatives
+    of BIP32, and use specific purpose
+    fields to flag different wallet types.
+    """
 
     def _get_bip32_base_path(self):
-        return self._key_ident, self._BIP49_PURPOSE,\
+        return self._key_ident, self._PURPOSE,\
                self._ENGINE.BIP44_COIN_TYPE
 
     @classmethod
@@ -1373,12 +1396,22 @@ class BIP49Wallet(BIP32Wallet):
 
         return path[len(self._get_bip32_base_path())] - 2**31
 
+class BIP49Wallet(BIP32PurposedWallet):
+    _PURPOSE = 2**31 + 49
+    _ENGINE = ENGINES[TYPE_P2SH_P2WPKH]
+
+class BIP84Wallet(BIP32PurposedWallet):
+    _PURPOSE = 2**31 + 84
+    _ENGINE = ENGINES[TYPE_P2WPKH]
 
 class SegwitLegacyWallet(ImportWalletMixin, BIP39WalletMixin, BIP49Wallet):
     TYPE = TYPE_P2SH_P2WPKH
 
+class SegwitWallet(ImportWalletMixin, BIP39WalletMixin, BIP84Wallet):
+    TYPE = TYPE_P2WPKH
 
 WALLET_IMPLEMENTATIONS = {
     LegacyWallet.TYPE: LegacyWallet,
-    SegwitLegacyWallet.TYPE: SegwitLegacyWallet
+    SegwitLegacyWallet.TYPE: SegwitLegacyWallet,
+    SegwitWallet.TYPE: SegwitWallet
 }

jmclient/test/commontest.py

@@ -155,10 +155,10 @@ def make_sign_and_push(ins_full,
     binarize_tx(de_tx)
     de_tx = wallet.sign_tx(de_tx, scripts, hashcode=hashcode)
     #pushtx returns False on any error
-    tx = binascii.hexlify(btc.serialize(de_tx)).decode('ascii')
-    push_succeed = jm_single().bc_interface.pushtx(tx)
+    push_succeed = jm_single().bc_interface.pushtx(btc.serialize(de_tx))
     if push_succeed:
-        return btc.txhash(tx)
+        removed = wallet.remove_old_utxos(de_tx)
+        return btc.txhash(btc.serialize(de_tx))
     else:
         return False
 

jmclient/test/test_tx_creation.py

@@ -41,11 +41,7 @@ def test_create_p2sh_output_tx(setup_tx_creation, nw, wallet_structures,
         wallet = w['wallet']
         ins_full = wallet.select_utxos(0, amount)
         script = bitcoin.mk_multisig_script(pubs, k)
-        #try the alternative argument passing
-        pubs.append(k)
-        script2 = bitcoin.mk_multisig_script(*pubs)
-        assert script2 == script
-        output_addr = bitcoin.scriptaddr(script, magicbyte=196)
+        output_addr = bitcoin.script_to_address(script, vbyte=196)
         txid = make_sign_and_push(ins_full,
                                   wallet,
                                   amount,
@@ -97,28 +93,21 @@ def test_all_same_priv(setup_tx_creation):
     tx = bitcoin.signall(tx, wallet.get_key_from_addr(addrinwallet))
 
 @pytest.mark.parametrize(
-    "signall, mktxlist",
+    "signall",
     [
-        (True, False),
-        (False, True),
+        (True,),
+        (False,),
     ])
-def test_verify_tx_input(setup_tx_creation, signall, mktxlist):
+def test_verify_tx_input(setup_tx_creation, signall):
     priv = "aa"*32 + "01"
     addr = bitcoin.privkey_to_address(priv, magicbyte=get_p2pk_vbyte())
     wallet = make_wallets(1, [[2,0,0,0,0]], 1)[0]['wallet']
     sync_wallet(wallet, fast=True)
     insfull = wallet.select_utxos(0, 110000000)
-    print(insfull)    
-    if not mktxlist:
-        outs = [{"address": addr, "value": 1000000}]
-        ins = list(insfull.keys())
-        tx = bitcoin.mktx(ins, outs)
-    else:
-        out1 = addr+":1000000"
-        ins0, ins1 = list(insfull.keys())
-        print("INS0 is: " + str(ins0))
-        print("INS1 is: " + str(ins1))
-        tx = bitcoin.mktx(ins0, ins1, out1)
+    print(insfull)
+    outs = [{"address": addr, "value": 1000000}]
+    ins = list(insfull.keys())
+    tx = bitcoin.mktx(ins, outs)
     desertx = bitcoin.deserialize(tx)
     print(desertx)
     if signall:
@@ -134,11 +123,7 @@ def test_verify_tx_input(setup_tx_creation, signall, mktxlist):
             utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
             ad = insfull[utxo]['address']
             priv = wallet.get_key_from_addr(ad)
-            if index % 2:
-                tx = binascii.unhexlify(tx)
             tx = bitcoin.sign(tx, index, priv)
-            if index % 2:
-                tx = binascii.hexlify(tx).decode('ascii')
     desertx2 = bitcoin.deserialize(tx)
     print(desertx2)
     sig, pub = bitcoin.deserialize_script(desertx2['ins'][0]['script'])
@@ -187,7 +172,7 @@ def test_spend_p2sh_utxos(setup_tx_creation):
     privs = [struct.pack(b'B', x) * 32 + b'\x01' for x in range(1, 4)]
     pubs = [bitcoin.privkey_to_pubkey(binascii.hexlify(priv).decode('ascii')) for priv in privs]
     script = bitcoin.mk_multisig_script(pubs, 2)
-    msig_addr = bitcoin.scriptaddr(script, magicbyte=196)
+    msig_addr = bitcoin.p2sh_scriptaddr(script, magicbyte=196)
     #pay into it
     wallet = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet']
     sync_wallet(wallet, fast=True)
@@ -212,6 +197,89 @@ def test_spend_p2sh_utxos(setup_tx_creation):
     txid = jm_single().bc_interface.pushtx(tx)
     assert txid
 
+def test_spend_p2wpkh(setup_tx_creation):
+    #make 3 p2wpkh outputs from 3 privs
+    privs = [struct.pack(b'B', x) * 32 + b'\x01' for x in range(1, 4)]
+    pubs = [bitcoin.privkey_to_pubkey(
+        binascii.hexlify(priv).decode('ascii')) for priv in privs]
+    scriptPubKeys = [bitcoin.pubkey_to_p2wpkh_script(pub) for pub in pubs]
+    addresses = [bitcoin.pubkey_to_p2wpkh_address(pub) for pub in pubs]
+    #pay into it
+    wallet = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet']
+    sync_wallet(wallet, fast=True)
+    amount = 35000000
+    p2wpkh_ins = []
+    for addr in addresses:
+        ins_full = wallet.select_utxos(0, amount)
+        txid = make_sign_and_push(ins_full, wallet, amount, output_addr=addr)
+        assert txid
+        p2wpkh_ins.append(txid + ":0")
+        #wait for mining
+        time.sleep(1)
+    #random output address
+    output_addr = wallet.get_new_addr(1, 1)
+    amount2 = amount*3 - 50000
+    outs = [{'value': amount2, 'address': output_addr}]
+    tx = bitcoin.mktx(p2wpkh_ins, outs)
+    sigs = []
+    for i, priv in enumerate(privs):
+        # sign each of 3 inputs
+        tx = bitcoin.p2wpkh_sign(tx, i, binascii.hexlify(priv),
+                                 amount, native=True)
+        # check that verify_tx_input correctly validates;
+        # to do this, we need to extract the signature and get the scriptCode
+        # of this pubkey
+        scriptCode = bitcoin.pubkey_to_p2pkh_script(pubs[i])
+        witness = bitcoin.deserialize(tx)['ins'][i]['txinwitness']
+        assert len(witness) == 2
+        assert witness[1] == pubs[i]
+        sig = witness[0]
+        assert bitcoin.verify_tx_input(tx, i, scriptPubKeys[i], sig,
+                        pubs[i], scriptCode=scriptCode, amount=amount)
+    txid = jm_single().bc_interface.pushtx(tx)
+    assert txid
+
+
+def test_spend_p2wsh(setup_tx_creation):
+    #make 2 x 2 of 2multisig outputs; will need 4 privs
+    privs = [struct.pack(b'B', x) * 32 + b'\x01' for x in range(1, 5)]
+    privs = [binascii.hexlify(priv).decode('ascii') for priv in privs]
+    pubs = [bitcoin.privkey_to_pubkey(priv) for priv in privs]
+    redeemScripts = [bitcoin.mk_multisig_script(pubs[i:i+2], 2) for i in [0, 2]]
+    scriptPubKeys = [bitcoin.pubkeys_to_p2wsh_script(pubs[i:i+2]) for i in [0, 2]]
+    addresses = [bitcoin.pubkeys_to_p2wsh_address(pubs[i:i+2]) for i in [0, 2]]
+    #pay into it
+    wallet = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet']
+    sync_wallet(wallet, fast=True)
+    amount = 35000000
+    p2wsh_ins = []
+    for addr in addresses:
+        ins_full = wallet.select_utxos(0, amount)
+        txid = make_sign_and_push(ins_full, wallet, amount, output_addr=addr)
+        assert txid
+        p2wsh_ins.append(txid + ":0")
+        #wait for mining
+        time.sleep(1)
+    #random output address and change addr
+    output_addr = wallet.get_new_addr(1, 1)
+    amount2 = amount*2 - 50000
+    outs = [{'value': amount2, 'address': output_addr}]
+    tx = bitcoin.mktx(p2wsh_ins, outs)
+    sigs = []
+    for i in range(2):
+        sigs = []
+        for priv in privs[i*2:i*2+2]:
+            # sign input j with each of 2 keys
+            sig = bitcoin.multisign(tx, i, redeemScripts[i], priv, amount=amount)
+            sigs.append(sig)
+            # check that verify_tx_input correctly validates;
+            assert bitcoin.verify_tx_input(tx, i, scriptPubKeys[i], sig,
+                                           bitcoin.privkey_to_pubkey(priv),
+                                           scriptCode=redeemScripts[i], amount=amount)
+        tx = bitcoin.apply_p2wsh_multisignatures(tx, i, redeemScripts[i], sigs)
+    txid = jm_single().bc_interface.pushtx(tx)
+    assert txid
+
 
 @pytest.fixture(scope="module")
 def setup_tx_creation():

jmclient/test/test_wallet.py

@@ -13,7 +13,8 @@ from commontest import binarize_tx
 from jmbase import get_log
 from jmclient import load_program_config, jm_single, \
     SegwitLegacyWallet,BIP32Wallet, BIP49Wallet, LegacyWallet,\
-    VolatileStorage, get_network, cryptoengine, WalletError
+    VolatileStorage, get_network, cryptoengine, WalletError,\
+    SegwitWallet
 from test_blockchaininterface import sync_test_wallet
 
 testdir = os.path.dirname(os.path.realpath(__file__))
@@ -303,17 +304,17 @@ def test_signing_imported(setup_wallet, wif, keytype, type_check):
     utxo = fund_wallet_addr(wallet, wallet.get_addr_path(path))
     tx = btc.deserialize(btc.mktx(['{}:{}'.format(hexlify(utxo[0]).decode('ascii'), utxo[1])],
                                   ['00'*17 + ':' + str(10**8 - 9000)]))
-    binarize_tx(tx)
     script = wallet.get_script_path(path)
-    wallet.sign_tx(tx, {0: (script, 10**8)})
+    tx = wallet.sign_tx(tx, {0: (script, 10**8)})
     type_check(tx)
-    txout = jm_single().bc_interface.pushtx(hexlify(btc.serialize(tx)).decode('ascii'))
+    txout = jm_single().bc_interface.pushtx(btc.serialize(tx))
     assert txout
 
 
 @pytest.mark.parametrize('wallet_cls,type_check', [
     [LegacyWallet, assert_not_segwit],
-    [SegwitLegacyWallet, assert_segwit]
+    [SegwitLegacyWallet, assert_segwit],
+    [SegwitWallet, assert_segwit],
 ])
 def test_signing_simple(setup_wallet, wallet_cls, type_check):
     jm_single().config.set('BLOCKCHAIN', 'network', 'testnet')
@@ -321,13 +322,15 @@ def test_signing_simple(setup_wallet, wallet_cls, type_check):
     wallet_cls.initialize(storage, get_network())
     wallet = wallet_cls(storage)
     utxo = fund_wallet_addr(wallet, wallet.get_internal_addr(0))
-    tx = btc.deserialize(btc.mktx(['{}:{}'.format(hexlify(utxo[0]).decode('ascii'), utxo[1])],
-                                  ['00'*17 + ':' + str(10**8 - 9000)]))
-    binarize_tx(tx)
+    # The dummy output is of length 25 bytes, because, for SegwitWallet, we else
+    # trigger the tx-size-small DOS limit in Bitcoin Core (82 bytes is the
+    # smallest "normal" transaction size (non-segwit size, ie no witness)
+    tx = btc.deserialize(btc.mktx(['{}:{}'.format(hexlify(utxo[0]).decode('ascii'),
+                                utxo[1])], ['00'*25 + ':' + str(10**8 - 9000)]))
     script = wallet.get_script(0, 1, 0)
-    wallet.sign_tx(tx, {0: (script, 10**8)})
+    tx = wallet.sign_tx(tx, {0: (script, 10**8)})
     type_check(tx)
-    txout = jm_single().bc_interface.pushtx(hexlify(btc.serialize(tx)).decode('ascii'))
+    txout = jm_single().bc_interface.pushtx(btc.serialize(tx))
     assert txout
 
 
@@ -645,4 +648,6 @@ def test_wallet_mixdepth_decrease(setup_wallet):
 @pytest.fixture(scope='module')
 def setup_wallet():
     load_program_config()
+    #see note in cryptoengine.py:
+    cryptoengine.BTC_P2WPKH.VBYTE = 100
     jm_single().bc_interface.tick_forward_chain_interval = 2

test/test_segwit.py

@@ -112,7 +112,6 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount,
         {'script': binascii.hexlify(change_script).decode('ascii'),
          'value': change_amt}]
     tx = btc.deserialize(btc.mktx(tx_ins, tx_outs))
-    binarize_tx(tx)
 
     # import new addresses to bitcoind
     jm_single().bc_interface.import_addresses(
@@ -125,18 +124,18 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount,
     for nsw_in_index in o_ins:
         inp = nsw_ins[nsw_in_index][1]
         scripts[nsw_in_index] = (inp['script'], inp['value'])
-    nsw_wallet.sign_tx(tx, scripts)
+    tx = nsw_wallet.sign_tx(tx, scripts)
 
     scripts = {}
     for sw_in_index in segwit_ins:
         inp = sw_ins[sw_in_index][1]
         scripts[sw_in_index] = (inp['script'], inp['value'])
-    sw_wallet.sign_tx(tx, scripts)
+    tx = sw_wallet.sign_tx(tx, scripts)
 
     print(tx)
 
     # push and verify
-    txid = jm_single().bc_interface.pushtx(binascii.hexlify(btc.serialize(tx)).decode('ascii'))
+    txid = jm_single().bc_interface.pushtx(btc.serialize(tx))
     assert txid
 
     balances = jm_single().bc_interface.get_received_by_addr(