Adds psbt creation and signing support in JM wallet.

Subclassed btcointx.CKeyBase to create a privkey type
that uses our signing code.
These are then used to allow signing of inputs in
transactions owned by our wallet.
Created a PSBTWalletMixin to perform the above function,
and added it to our three wallet types.

Wrote a detailed creation, sign and broadcast test
for a psbt using utxos from within and outside the wallet.

Detailed tests across all 3 wallet types for psbt
Tests cover also mixed inputs of different
types and owned/unowned. Direct send now exported to be used
in tests rather than only script usage, also supports returning
a tx object rather than only a txid.

jmbitcoin/jmbitcoin/__init__.py

@@ -8,8 +8,10 @@ from bitcointx import select_chain_params
 from bitcointx.core import (x, b2x, b2lx, lx, COutPoint, CTxOut, CTxIn,
                             CTxInWitness, CTxWitness, CMutableTransaction,
                             Hash160, coins_to_satoshi, satoshi_to_coins)
+from bitcointx.core.key import KeyStore
 from bitcointx.core.script import (CScript, OP_0, SignatureHash, SIGHASH_ALL,
                                    SIGVERSION_WITNESS_V0, CScriptWitness)
 from bitcointx.wallet import (CBitcoinSecret, P2WPKHBitcoinAddress, CCoinAddress,
                               P2SHCoinAddress)
+from bitcointx.core.psbt import PartiallySignedTransaction
 

jmbitcoin/jmbitcoin/secp256k1_main.py

@@ -9,6 +9,7 @@ import coincurve as secp256k1
 
 from bitcointx import base58
 from bitcointx.core import Hash
+from bitcointx.core.key import CKeyBase, CPubKey
 from bitcointx.signmessage import BitcoinMessage
 
 #Required only for PoDLE calculation:
@@ -215,3 +216,49 @@ def ecdsa_raw_verify(msg, pub, sig, rawmsg=False):
     except Exception as e:
         return False
     return retval
+
+class JMCKey(bytes, CKeyBase):
+    """An encapsulated private key.
+    This subclasses specifically for JM's own signing code.
+
+    Attributes:
+
+    pub           - The corresponding CPubKey for this private key
+    secret_bytes  - Secret data, 32 bytes (needed because subclasses may have trailing data)
+
+    is_compressed() - True if compressed
+
+    """
+
+    def __init__(self, b):
+        CKeyBase.__init__(self, b, compressed=True)
+
+    def is_compressed(self):
+        return True
+
+    @property
+    def secret_bytes(self):
+        assert isinstance(self, bytes)
+        return self[:32]
+
+    def sign(self, hash):
+        assert isinstance(hash, (bytes, bytearray))
+        if len(hash) != 32:
+            raise ValueError('Hash must be exactly 32 bytes long')
+        # TODO: non default sighash flag.
+        return ecdsa_raw_sign(hash, self.secret_bytes + b"\x01", rawmsg=True)
+
+
+    def verify(self, hash, sig):
+        return self.pub.verify(hash, sig)
+
+    def verify_nonstrict(self, hash, sig):
+        return self.pub.verify_nonstrict(hash, sig)
+
+    @classmethod
+    def from_secret_bytes(cls, secret, compressed=True):
+        return cls(secret, compressed=compressed)
+
+    @classmethod
+    def from_bytes(cls, data):
+        raise NotImplementedError('subclasses must override from_bytes()')

jmclient/jmclient/__init__.py

@@ -42,7 +42,7 @@ from .schedule import (get_schedule, get_tumble_schedule, schedule_to_text,
 from .commitment_utils import get_utxo_info, validate_utxo_data, quit
 from .taker_utils import (tumbler_taker_finished_update, restart_waiter,
                              restart_wait, get_tumble_log, direct_send,
-                             tumbler_filter_orders_callback)
+                             tumbler_filter_orders_callback, direct_send)
 from .cli_options import (add_base_options, add_common_options,
                           get_tumbler_parser, get_max_cj_fee_values,
                           check_regtest, get_sendpayment_parser,

jmclient/jmclient/taker_utils.py

@@ -21,7 +21,8 @@ Currently re-used by CLI script tumbler.py and joinmarket-qt
 """
 
 def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False,
-                accept_callback=None, info_callback=None):
+                accept_callback=None, info_callback=None,
+                return_transaction=False):
     """Send coins directly from one mixdepth to one destination address;
     does not need IRC. Sweep as for normal sendpayment (set amount=0).
     If answeryes is True, callback/command line query is not performed.
@@ -38,7 +39,8 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False,
     pushed), and returns nothing.
 
     This function returns:
-    The txid if transaction is pushed, False otherwise
+    The txid if transaction is pushed, False otherwise,
+    or the full CMutableTransaction if return_transaction is True.
     """
     #Sanity checks
     assert validate_address(destination)[0] or is_burn_destination(destination)
@@ -134,8 +136,8 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False,
     log.info("Using a fee of : " + amount_to_str(fee_est) + ".")
     if amount != 0:
         log.info("Using a change value of: " + amount_to_str(changeval) + ".")
-    tx = make_shuffled_tx(list(utxos.keys()), outs, 2, compute_tx_locktime())
-        list(utxos.keys()), outs, 2, tx_locktime), utxos)
+    tx = make_shuffled_tx(list(utxos.keys()), outs, 2, tx_locktime)
+
     inscripts = {}
     for i, txinp in enumerate(tx.vin):
         u = (txinp.prevout.hash[::-1], txinp.prevout.n)
@@ -165,7 +167,8 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False,
     successmsg = "Transaction sent: " + txid
     cb = log.info if not info_callback else info_callback
     cb(successmsg)
-    return txid
+    txinfo = txid if not return_transaction else tx
+    return txinfo
 
 
 def sign_tx(wallet_service, tx, utxos):

jmclient/jmclient/wallet.py

@@ -985,6 +985,118 @@ class BaseWallet(object):
     def __del__(self):
         self.close()
 
+class DummyKeyStore(btc.KeyStore):
+    @classmethod
+    def from_iterable(cls, iterable, **kwargs):
+        kstore = cls(**kwargs)
+        for k in iterable:
+            kstore.add_key(k)
+        return kstore
+
+    def add_key(self, k):
+        if isinstance(k, btc.CKeyBase):
+            if k.pub.key_id in self._privkeys:
+                assert self._privkeys[k.pub.key_id] == k
+            else:
+                self._privkeys[k.pub.key_id] = k
+        else:
+            raise ValueError('object supplied to add_key is of unrecognized type')
+
+class PSBTWalletMixin(object):
+    """
+    Mixin for BaseWallet to provide BIP174
+    functions.
+    """
+    def __init__(self, storage, **kwargs):
+        super(PSBTWalletMixin, self).__init__(storage, **kwargs)
+
+    @staticmethod
+    def witness_utxos_to_psbt_utxos(utxos):
+        """ Given a dict of utxos as returned from select_utxos,
+        convert them to the format required to populate PSBT inputs,
+        namely CTxOut. Note that the non-segwit case is different, there
+        you should provide an entire CMutableTransaction object instead.
+        """
+        return [btc.CMutableTxOut(v["value"],
+                                  v["script"]) for _, v in utxos.items()]
+
+    def create_psbt_from_tx(self, tx, spent_outs=None):
+        """ Given a CMutableTransaction object, which should not currently
+        contain signatures, we create and return a new PSBT object of type
+        btc.PartiallySignedTransaction.
+        Optionally the information about the spent outputs that is stored
+        in PSBT_IN_NONWITNESS_UTXO, PSBT_IN_WITNESS_UTXO and PSBT_IN_REDEEM_SCRIPT
+        can also be provided, one item per input, in the tuple (spent_outs).
+        These objects should be either CMutableTransaction, CTxOut or None,
+        Note that redeem script information cannot be provided for inputs which
+        we don't own.
+        """
+        # TODO: verify tx contains no signatures as a sanity check?
+        new_psbt = btc.PartiallySignedTransaction(unsigned_tx=tx)
+        if spent_outs is None:
+            # user has not provided input script information; psbt
+            # will not yet be usable for signing.
+            return new_psbt
+        for i, txinput in enumerate(new_psbt.inputs):
+            if spent_outs[i] is None:
+                # as above, will not be signable in this case
+                continue
+            if isinstance(spent_outs[i], (btc.CMutableTransaction, btc.CMutableTxOut)):
+                # note that we trust the caller to choose Tx vs TxOut as according
+                # to non-witness/witness:
+                txinput.utxo = spent_outs[i]
+            else:
+                assert False, "invalid spent output type passed into PSBT creator"
+        # we now insert redeemscripts where that is possible and necessary:
+        for i, txinput in enumerate(new_psbt.inputs):
+            if isinstance(txinput.utxo, btc.CMutableTxOut):
+                # witness
+                if txinput.utxo.scriptPubKey.is_witness_scriptpubkey():
+                    # nothing needs inserting; the scriptSig is empty.
+                    continue
+                elif txinput.utxo.scriptPubKey.is_p2sh():
+                    try:
+                        path = self.script_to_path(txinput.utxo.scriptPubKey)
+                    except AssertionError:
+                        # this happens when an input is provided but it's not in
+                        # this wallet; in this case, we cannot set the redeem script.
+                        continue
+                    privkey, _ = self._get_priv_from_path(path)
+                    txinput.redeem_script = btc.pubkey_to_p2wpkh_script(
+                        btc.privkey_to_pubkey(privkey))
+        return new_psbt
+
+    def sign_psbt(self, in_psbt, with_sign_result=False):
+        """ Given a serialized PSBT in raw binary format,
+        iterate over the inputs and sign all that we can sign with this wallet.
+        NB IT IS UP TO CALLERS TO ENSURE THAT THEY ACTUALLY WANT TO SIGN
+        THIS TRANSACTION!
+        The above is important especially in coinjoin scenarios.
+        Return: (psbt, msg)
+        msg: error message or None
+        if not `with_sign_result`:
+        psbt: signed psbt in binary serialization, or None if error.
+        if `with_sign_result` True:
+        psbt: (PSBT_SignResult object, psbt (deserialized) object)
+        """
+        try:
+            new_psbt = btc.PartiallySignedTransaction.from_binary(in_psbt)
+        except:
+            return None, "Unable to deserialize the PSBT object, invalid format."
+        privkeys = []
+        for k, v in self._utxos._utxo.items():
+            for k2, v2 in v.items():
+                privkeys.append(self._get_priv_from_path(v2[0]))
+        jmckeys = list(btc.JMCKey(x[0][:-1]) for x in privkeys)
+        new_keystore = DummyKeyStore.from_iterable(jmckeys)
+        try:
+            signresult = new_psbt.sign(new_keystore)
+        except Exception as e:
+            return None, repr(e)
+        if not with_sign_result:
+            return new_psbt.serialize(), None
+        else:
+            return (signresult, new_psbt), None
 
 class ImportWalletMixin(object):
     """
@@ -1543,7 +1655,7 @@ class BIP32Wallet(BaseWallet):
         return self._get_mixdepth_from_path(path), path[-2], path[-1]
 
 
-class LegacyWallet(ImportWalletMixin, BIP32Wallet):
+class LegacyWallet(ImportWalletMixin, PSBTWalletMixin, BIP32Wallet):
     TYPE = TYPE_P2PKH
     _ENGINE = ENGINES[TYPE_P2PKH]
 
@@ -1804,10 +1916,10 @@ class BIP84Wallet(BIP32PurposedWallet):
     _PURPOSE = 2**31 + 84
     _ENGINE = ENGINES[TYPE_P2WPKH]
 
-class SegwitLegacyWallet(ImportWalletMixin, BIP39WalletMixin, BIP49Wallet):
+class SegwitLegacyWallet(ImportWalletMixin, BIP39WalletMixin, PSBTWalletMixin, BIP49Wallet):
     TYPE = TYPE_P2SH_P2WPKH
 
-class SegwitWallet(ImportWalletMixin, BIP39WalletMixin, BIP84Wallet):
+class SegwitWallet(ImportWalletMixin, BIP39WalletMixin, PSBTWalletMixin, BIP84Wallet):
     TYPE = TYPE_P2WPKH
 
 class SegwitLegacyWalletFidelityBonds(FidelityBondMixin, SegwitLegacyWallet):

jmclient/test/test_psbt_wallet.py

@@ -0,0 +1,176 @@
+#! /usr/bin/env python
+'''Test of psbt creation, update, signing and finalizing
+   using the functionality of the PSBT Wallet Mixin.
+   Note that Joinmarket's PSBT code is a wrapper around
+   bitcointx.core.psbt, and the basic test vectors for
+   BIP174 are tested there, not here.
+   '''
+
+import time
+import binascii
+import struct
+import copy
+from commontest import make_wallets
+
+import jmbitcoin as bitcoin
+import pytest
+from jmbase import get_log, bintohex, hextobin
+from jmclient import (load_test_config, jm_single, direct_send,
+                      SegwitLegacyWallet, SegwitWallet, LegacyWallet)
+
+log = get_log()
+
+def dummy_accept_callback(tx, destaddr, actual_amount, fee_est):
+    return True
+def dummy_info_callback(msg):
+    pass
+
+def test_create_and_sign_psbt_with_legacy(setup_psbt_wallet):
+    """ The purpose of this test is to check that we can create and
+    then partially sign a PSBT where we own one input and the other input
+    is of legacy p2pkh type.
+    """
+    wallet_service = make_wallets(1, [[1,0,0,0,0]], 1)[0]['wallet']
+    wallet_service.sync_wallet(fast=True)
+    utxos = wallet_service.select_utxos(0, bitcoin.coins_to_satoshi(0.5))
+    assert len(utxos) == 1
+    # create a legacy address and make a payment into it
+    legacy_addr = bitcoin.CCoinAddress.from_scriptPubKey(
+        bitcoin.pubkey_to_p2pkh_script(
+            bitcoin.privkey_to_pubkey(b"\x01"*33)))
+    tx = direct_send(wallet_service, bitcoin.coins_to_satoshi(0.3), 0,
+                       str(legacy_addr), accept_callback=dummy_accept_callback,
+                       info_callback=dummy_info_callback,
+                       return_transaction=True)
+    assert tx
+    # this time we will have one utxo worth <~ 0.7
+    my_utxos = wallet_service.select_utxos(0, bitcoin.coins_to_satoshi(0.5))
+    assert len(my_utxos) == 1
+    # find the outpoint for the legacy address we're spending
+    n = -1
+    for i, t in enumerate(tx.vout):
+        if bitcoin.CCoinAddress.from_scriptPubKey(t.scriptPubKey) == legacy_addr:
+            n = i
+    assert n > -1
+    utxos = copy.deepcopy(my_utxos)
+    utxos[(tx.GetTxid()[::-1], n)] ={"script": legacy_addr.to_scriptPubKey(),
+              "value": bitcoin.coins_to_satoshi(0.3)}
+    outs = [{"value": bitcoin.coins_to_satoshi(0.998),
+                 "address": wallet_service.get_addr(0,0,0)}]
+    tx2 = bitcoin.mktx(list(utxos.keys()), outs)
+    spent_outs = wallet_service.witness_utxos_to_psbt_utxos(my_utxos)
+    spent_outs.append(tx)
+    new_psbt = wallet_service.create_psbt_from_tx(tx2, spent_outs)
+    signed_psbt_and_signresult, err = wallet_service.sign_psbt(
+        new_psbt.serialize(), with_sign_result=True)
+    assert err is None
+    signresult, signed_psbt = signed_psbt_and_signresult
+    assert signresult.num_inputs_signed == 1
+    assert signresult.num_inputs_final == 1
+    assert not signresult.is_final
+
+@pytest.mark.parametrize('unowned_utxo, wallet_cls', [
+    (True, SegwitLegacyWallet),
+    (False, SegwitLegacyWallet),
+    (True, SegwitWallet),
+    (False, SegwitWallet),
+    (True, LegacyWallet),
+    (False, LegacyWallet),
+])
+def test_create_psbt_and_sign(setup_psbt_wallet, unowned_utxo, wallet_cls):
+    """ Plan of test:
+    1. Create a wallet and source 3 destination addresses.
+    2. Make, and confirm, transactions that fund the 3 addrs.
+    3. Create a new tx spending 2 of those 3 utxos and spending
+       another utxo we don't own (extra is optional per `unowned_utxo`).
+    4. Create a psbt using the above transaction and corresponding
+       `spent_outs` field to fill in the redeem script.
+    5. Compare resulting PSBT with expected structure.
+    6. Use the wallet's sign_psbt method to sign the whole psbt, which
+       means signing each input we own.
+    7. Check that each input is finalized as per expected. Check that the whole
+       PSBT is or is not finalized as per whether there is an unowned utxo.
+    8. In case where whole psbt is finalized, attempt to broadcast the tx.
+    """
+    # steps 1 and 2:
+    wallet_service = make_wallets(1, [[3,0,0,0,0]], 1,
+                                  wallet_cls=wallet_cls)[0]['wallet']
+    wallet_service.sync_wallet(fast=True)
+    utxos = wallet_service.select_utxos(0, bitcoin.coins_to_satoshi(1.5))
+    # for legacy wallets, psbt creation requires querying for the spending
+    # transaction:
+    if wallet_cls == LegacyWallet:
+        fulltxs = []
+        for utxo, v in utxos.items():
+            fulltxs.append(jm_single().bc_interface.get_deser_from_gettransaction(
+                jm_single().bc_interface.get_transaction(utxo[0])))
+
+    assert len(utxos) == 2
+    u_utxos = {}
+    if unowned_utxo:
+        # note: tx creation uses the key only; psbt creation uses the value,
+        # which can be fake here; we do not intend to attempt to fully
+        # finalize a psbt with an unowned input. See
+        # https://github.com/Simplexum/python-bitcointx/issues/30
+        # the redeem script creation (which is artificial) will be
+        # avoided in future.
+        priv = b"\xaa"*32 + b"\x01"
+        pub = bitcoin.privkey_to_pubkey(priv)
+        script = bitcoin.pubkey_to_p2sh_p2wpkh_script(pub)
+        redeem_script = bitcoin.pubkey_to_p2wpkh_script(pub)
+        u_utxos[(b"\xaa"*32, 12)] = {"value": 1000, "script": script}
+    utxos.update(u_utxos)
+    # outputs aren't interesting for this test (we selected 1.5 but will get 2):
+    outs = [{"value": bitcoin.coins_to_satoshi(1.999),
+             "address": wallet_service.get_addr(0,0,0)}]
+    tx = bitcoin.mktx(list(utxos.keys()), outs)
+
+    if wallet_cls != LegacyWallet:
+        spent_outs = wallet_service.witness_utxos_to_psbt_utxos(utxos)
+    else:
+        spent_outs = fulltxs
+        # the extra input is segwit:
+        if unowned_utxo:
+            spent_outs.extend(
+                wallet_service.witness_utxos_to_psbt_utxos(u_utxos))
+    newpsbt = wallet_service.create_psbt_from_tx(tx, spent_outs)
+    # see note above
+    if unowned_utxo:
+        newpsbt.inputs[-1].redeem_script = redeem_script
+    print(bintohex(newpsbt.serialize()))
+    # we cannot compare with a fixed expected result due to wallet randomization, but we can
+    # check psbt structure:
+    expected_inputs_length = 3 if unowned_utxo else 2
+    assert len(newpsbt.inputs) == expected_inputs_length
+    assert len(newpsbt.outputs) == 1
+    # note: redeem_script field is a CScript which is a bytes instance,
+    # so checking length is best way to check for existence (comparison
+    # with None does not work):
+    if wallet_cls == SegwitLegacyWallet:
+        assert len(newpsbt.inputs[0].redeem_script) != 0
+        assert len(newpsbt.inputs[1].redeem_script) != 0
+    if unowned_utxo:
+        assert newpsbt.inputs[2].redeem_script == redeem_script
+
+    signed_psbt_and_signresult, err = wallet_service.sign_psbt(
+        newpsbt.serialize(), with_sign_result=True)
+    assert err is None
+    signresult, signed_psbt = signed_psbt_and_signresult
+    expected_signed_inputs = len(utxos) if not unowned_utxo else len(utxos)-1
+    assert signresult.num_inputs_signed == expected_signed_inputs
+    assert signresult.num_inputs_final == expected_signed_inputs
+
+    if not unowned_utxo:
+        assert signresult.is_final
+        # only in case all signed do we try to broadcast:
+        extracted_tx = signed_psbt.extract_transaction().serialize()
+        assert jm_single().bc_interface.pushtx(extracted_tx)
+    else:
+        # transaction extraction must fail for not-fully-signed psbts:
+        with pytest.raises(ValueError) as e:
+            extracted_tx = signed_psbt.extract_transaction()
+        
+  
+@pytest.fixture(scope="module")
+def setup_psbt_wallet():
+    load_test_config()