diff --git a/jmbitcoin/jmbitcoin/__init__.py b/jmbitcoin/jmbitcoin/__init__.py index 00fc519..2a7f481 100644 --- a/jmbitcoin/jmbitcoin/__init__.py +++ b/jmbitcoin/jmbitcoin/__init__.py @@ -8,8 +8,10 @@ from bitcointx import select_chain_params from bitcointx.core import (x, b2x, b2lx, lx, COutPoint, CTxOut, CTxIn, CTxInWitness, CTxWitness, CMutableTransaction, Hash160, coins_to_satoshi, satoshi_to_coins) +from bitcointx.core.key import KeyStore from bitcointx.core.script import (CScript, OP_0, SignatureHash, SIGHASH_ALL, SIGVERSION_WITNESS_V0, CScriptWitness) from bitcointx.wallet import (CBitcoinSecret, P2WPKHBitcoinAddress, CCoinAddress, P2SHCoinAddress) +from bitcointx.core.psbt import PartiallySignedTransaction diff --git a/jmbitcoin/jmbitcoin/secp256k1_main.py b/jmbitcoin/jmbitcoin/secp256k1_main.py index db0e7cb..45a5bd6 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_main.py +++ b/jmbitcoin/jmbitcoin/secp256k1_main.py @@ -9,6 +9,7 @@ import coincurve as secp256k1 from bitcointx import base58 from bitcointx.core import Hash +from bitcointx.core.key import CKeyBase, CPubKey from bitcointx.signmessage import BitcoinMessage #Required only for PoDLE calculation: @@ -215,3 +216,49 @@ def ecdsa_raw_verify(msg, pub, sig, rawmsg=False): except Exception as e: return False return retval + +class JMCKey(bytes, CKeyBase): + """An encapsulated private key. + This subclasses specifically for JM's own signing code. + + Attributes: + + pub - The corresponding CPubKey for this private key + secret_bytes - Secret data, 32 bytes (needed because subclasses may have trailing data) + + is_compressed() - True if compressed + + """ + + def __init__(self, b): + CKeyBase.__init__(self, b, compressed=True) + + def is_compressed(self): + return True + + @property + def secret_bytes(self): + assert isinstance(self, bytes) + return self[:32] + + def sign(self, hash): + assert isinstance(hash, (bytes, bytearray)) + if len(hash) != 32: + raise ValueError('Hash must be exactly 32 bytes long') + # TODO: non default sighash flag. + return ecdsa_raw_sign(hash, self.secret_bytes + b"\x01", rawmsg=True) + + + def verify(self, hash, sig): + return self.pub.verify(hash, sig) + + def verify_nonstrict(self, hash, sig): + return self.pub.verify_nonstrict(hash, sig) + + @classmethod + def from_secret_bytes(cls, secret, compressed=True): + return cls(secret, compressed=compressed) + + @classmethod + def from_bytes(cls, data): + raise NotImplementedError('subclasses must override from_bytes()') diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index 0b73d68..180e5a1 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -42,7 +42,7 @@ from .schedule import (get_schedule, get_tumble_schedule, schedule_to_text, from .commitment_utils import get_utxo_info, validate_utxo_data, quit from .taker_utils import (tumbler_taker_finished_update, restart_waiter, restart_wait, get_tumble_log, direct_send, - tumbler_filter_orders_callback) + tumbler_filter_orders_callback, direct_send) from .cli_options import (add_base_options, add_common_options, get_tumbler_parser, get_max_cj_fee_values, check_regtest, get_sendpayment_parser, diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index 4f32544..dc4a9b8 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -21,7 +21,8 @@ Currently re-used by CLI script tumbler.py and joinmarket-qt """ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, - accept_callback=None, info_callback=None): + accept_callback=None, info_callback=None, + return_transaction=False): """Send coins directly from one mixdepth to one destination address; does not need IRC. Sweep as for normal sendpayment (set amount=0). If answeryes is True, callback/command line query is not performed. @@ -38,7 +39,8 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, pushed), and returns nothing. This function returns: - The txid if transaction is pushed, False otherwise + The txid if transaction is pushed, False otherwise, + or the full CMutableTransaction if return_transaction is True. """ #Sanity checks assert validate_address(destination)[0] or is_burn_destination(destination) @@ -134,8 +136,8 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, log.info("Using a fee of : " + amount_to_str(fee_est) + ".") if amount != 0: log.info("Using a change value of: " + amount_to_str(changeval) + ".") - tx = make_shuffled_tx(list(utxos.keys()), outs, 2, compute_tx_locktime()) - list(utxos.keys()), outs, 2, tx_locktime), utxos) + tx = make_shuffled_tx(list(utxos.keys()), outs, 2, tx_locktime) + inscripts = {} for i, txinp in enumerate(tx.vin): u = (txinp.prevout.hash[::-1], txinp.prevout.n) @@ -165,7 +167,8 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, successmsg = "Transaction sent: " + txid cb = log.info if not info_callback else info_callback cb(successmsg) - return txid + txinfo = txid if not return_transaction else tx + return txinfo def sign_tx(wallet_service, tx, utxos): diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index 2a62a28..a1b80eb 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -985,6 +985,118 @@ class BaseWallet(object): def __del__(self): self.close() +class DummyKeyStore(btc.KeyStore): + @classmethod + def from_iterable(cls, iterable, **kwargs): + kstore = cls(**kwargs) + for k in iterable: + kstore.add_key(k) + return kstore + + def add_key(self, k): + if isinstance(k, btc.CKeyBase): + if k.pub.key_id in self._privkeys: + assert self._privkeys[k.pub.key_id] == k + else: + self._privkeys[k.pub.key_id] = k + else: + raise ValueError('object supplied to add_key is of unrecognized type') + +class PSBTWalletMixin(object): + """ + Mixin for BaseWallet to provide BIP174 + functions. + """ + def __init__(self, storage, **kwargs): + super(PSBTWalletMixin, self).__init__(storage, **kwargs) + + @staticmethod + def witness_utxos_to_psbt_utxos(utxos): + """ Given a dict of utxos as returned from select_utxos, + convert them to the format required to populate PSBT inputs, + namely CTxOut. Note that the non-segwit case is different, there + you should provide an entire CMutableTransaction object instead. + """ + return [btc.CMutableTxOut(v["value"], + v["script"]) for _, v in utxos.items()] + + def create_psbt_from_tx(self, tx, spent_outs=None): + """ Given a CMutableTransaction object, which should not currently + contain signatures, we create and return a new PSBT object of type + btc.PartiallySignedTransaction. + Optionally the information about the spent outputs that is stored + in PSBT_IN_NONWITNESS_UTXO, PSBT_IN_WITNESS_UTXO and PSBT_IN_REDEEM_SCRIPT + can also be provided, one item per input, in the tuple (spent_outs). + These objects should be either CMutableTransaction, CTxOut or None, + Note that redeem script information cannot be provided for inputs which + we don't own. + """ + # TODO: verify tx contains no signatures as a sanity check? + new_psbt = btc.PartiallySignedTransaction(unsigned_tx=tx) + if spent_outs is None: + # user has not provided input script information; psbt + # will not yet be usable for signing. + return new_psbt + for i, txinput in enumerate(new_psbt.inputs): + if spent_outs[i] is None: + # as above, will not be signable in this case + continue + if isinstance(spent_outs[i], (btc.CMutableTransaction, btc.CMutableTxOut)): + # note that we trust the caller to choose Tx vs TxOut as according + # to non-witness/witness: + txinput.utxo = spent_outs[i] + else: + assert False, "invalid spent output type passed into PSBT creator" + # we now insert redeemscripts where that is possible and necessary: + for i, txinput in enumerate(new_psbt.inputs): + if isinstance(txinput.utxo, btc.CMutableTxOut): + # witness + if txinput.utxo.scriptPubKey.is_witness_scriptpubkey(): + # nothing needs inserting; the scriptSig is empty. + continue + elif txinput.utxo.scriptPubKey.is_p2sh(): + try: + path = self.script_to_path(txinput.utxo.scriptPubKey) + except AssertionError: + # this happens when an input is provided but it's not in + # this wallet; in this case, we cannot set the redeem script. + continue + privkey, _ = self._get_priv_from_path(path) + txinput.redeem_script = btc.pubkey_to_p2wpkh_script( + btc.privkey_to_pubkey(privkey)) + return new_psbt + + def sign_psbt(self, in_psbt, with_sign_result=False): + """ Given a serialized PSBT in raw binary format, + iterate over the inputs and sign all that we can sign with this wallet. + NB IT IS UP TO CALLERS TO ENSURE THAT THEY ACTUALLY WANT TO SIGN + THIS TRANSACTION! + The above is important especially in coinjoin scenarios. + Return: (psbt, msg) + msg: error message or None + if not `with_sign_result`: + psbt: signed psbt in binary serialization, or None if error. + if `with_sign_result` True: + psbt: (PSBT_SignResult object, psbt (deserialized) object) + """ + try: + new_psbt = btc.PartiallySignedTransaction.from_binary(in_psbt) + except: + return None, "Unable to deserialize the PSBT object, invalid format." + privkeys = [] + for k, v in self._utxos._utxo.items(): + for k2, v2 in v.items(): + privkeys.append(self._get_priv_from_path(v2[0])) + jmckeys = list(btc.JMCKey(x[0][:-1]) for x in privkeys) + new_keystore = DummyKeyStore.from_iterable(jmckeys) + try: + signresult = new_psbt.sign(new_keystore) + except Exception as e: + return None, repr(e) + if not with_sign_result: + return new_psbt.serialize(), None + else: + return (signresult, new_psbt), None class ImportWalletMixin(object): """ @@ -1543,7 +1655,7 @@ class BIP32Wallet(BaseWallet): return self._get_mixdepth_from_path(path), path[-2], path[-1] -class LegacyWallet(ImportWalletMixin, BIP32Wallet): +class LegacyWallet(ImportWalletMixin, PSBTWalletMixin, BIP32Wallet): TYPE = TYPE_P2PKH _ENGINE = ENGINES[TYPE_P2PKH] @@ -1804,10 +1916,10 @@ class BIP84Wallet(BIP32PurposedWallet): _PURPOSE = 2**31 + 84 _ENGINE = ENGINES[TYPE_P2WPKH] -class SegwitLegacyWallet(ImportWalletMixin, BIP39WalletMixin, BIP49Wallet): +class SegwitLegacyWallet(ImportWalletMixin, BIP39WalletMixin, PSBTWalletMixin, BIP49Wallet): TYPE = TYPE_P2SH_P2WPKH -class SegwitWallet(ImportWalletMixin, BIP39WalletMixin, BIP84Wallet): +class SegwitWallet(ImportWalletMixin, BIP39WalletMixin, PSBTWalletMixin, BIP84Wallet): TYPE = TYPE_P2WPKH class SegwitLegacyWalletFidelityBonds(FidelityBondMixin, SegwitLegacyWallet): diff --git a/jmclient/test/test_psbt_wallet.py b/jmclient/test/test_psbt_wallet.py new file mode 100644 index 0000000..a2c4cd6 --- /dev/null +++ b/jmclient/test/test_psbt_wallet.py @@ -0,0 +1,176 @@ +#! /usr/bin/env python +'''Test of psbt creation, update, signing and finalizing + using the functionality of the PSBT Wallet Mixin. + Note that Joinmarket's PSBT code is a wrapper around + bitcointx.core.psbt, and the basic test vectors for + BIP174 are tested there, not here. + ''' + +import time +import binascii +import struct +import copy +from commontest import make_wallets + +import jmbitcoin as bitcoin +import pytest +from jmbase import get_log, bintohex, hextobin +from jmclient import (load_test_config, jm_single, direct_send, + SegwitLegacyWallet, SegwitWallet, LegacyWallet) + +log = get_log() + +def dummy_accept_callback(tx, destaddr, actual_amount, fee_est): + return True +def dummy_info_callback(msg): + pass + +def test_create_and_sign_psbt_with_legacy(setup_psbt_wallet): + """ The purpose of this test is to check that we can create and + then partially sign a PSBT where we own one input and the other input + is of legacy p2pkh type. + """ + wallet_service = make_wallets(1, [[1,0,0,0,0]], 1)[0]['wallet'] + wallet_service.sync_wallet(fast=True) + utxos = wallet_service.select_utxos(0, bitcoin.coins_to_satoshi(0.5)) + assert len(utxos) == 1 + # create a legacy address and make a payment into it + legacy_addr = bitcoin.CCoinAddress.from_scriptPubKey( + bitcoin.pubkey_to_p2pkh_script( + bitcoin.privkey_to_pubkey(b"\x01"*33))) + tx = direct_send(wallet_service, bitcoin.coins_to_satoshi(0.3), 0, + str(legacy_addr), accept_callback=dummy_accept_callback, + info_callback=dummy_info_callback, + return_transaction=True) + assert tx + # this time we will have one utxo worth <~ 0.7 + my_utxos = wallet_service.select_utxos(0, bitcoin.coins_to_satoshi(0.5)) + assert len(my_utxos) == 1 + # find the outpoint for the legacy address we're spending + n = -1 + for i, t in enumerate(tx.vout): + if bitcoin.CCoinAddress.from_scriptPubKey(t.scriptPubKey) == legacy_addr: + n = i + assert n > -1 + utxos = copy.deepcopy(my_utxos) + utxos[(tx.GetTxid()[::-1], n)] ={"script": legacy_addr.to_scriptPubKey(), + "value": bitcoin.coins_to_satoshi(0.3)} + outs = [{"value": bitcoin.coins_to_satoshi(0.998), + "address": wallet_service.get_addr(0,0,0)}] + tx2 = bitcoin.mktx(list(utxos.keys()), outs) + spent_outs = wallet_service.witness_utxos_to_psbt_utxos(my_utxos) + spent_outs.append(tx) + new_psbt = wallet_service.create_psbt_from_tx(tx2, spent_outs) + signed_psbt_and_signresult, err = wallet_service.sign_psbt( + new_psbt.serialize(), with_sign_result=True) + assert err is None + signresult, signed_psbt = signed_psbt_and_signresult + assert signresult.num_inputs_signed == 1 + assert signresult.num_inputs_final == 1 + assert not signresult.is_final + +@pytest.mark.parametrize('unowned_utxo, wallet_cls', [ + (True, SegwitLegacyWallet), + (False, SegwitLegacyWallet), + (True, SegwitWallet), + (False, SegwitWallet), + (True, LegacyWallet), + (False, LegacyWallet), +]) +def test_create_psbt_and_sign(setup_psbt_wallet, unowned_utxo, wallet_cls): + """ Plan of test: + 1. Create a wallet and source 3 destination addresses. + 2. Make, and confirm, transactions that fund the 3 addrs. + 3. Create a new tx spending 2 of those 3 utxos and spending + another utxo we don't own (extra is optional per `unowned_utxo`). + 4. Create a psbt using the above transaction and corresponding + `spent_outs` field to fill in the redeem script. + 5. Compare resulting PSBT with expected structure. + 6. Use the wallet's sign_psbt method to sign the whole psbt, which + means signing each input we own. + 7. Check that each input is finalized as per expected. Check that the whole + PSBT is or is not finalized as per whether there is an unowned utxo. + 8. In case where whole psbt is finalized, attempt to broadcast the tx. + """ + # steps 1 and 2: + wallet_service = make_wallets(1, [[3,0,0,0,0]], 1, + wallet_cls=wallet_cls)[0]['wallet'] + wallet_service.sync_wallet(fast=True) + utxos = wallet_service.select_utxos(0, bitcoin.coins_to_satoshi(1.5)) + # for legacy wallets, psbt creation requires querying for the spending + # transaction: + if wallet_cls == LegacyWallet: + fulltxs = [] + for utxo, v in utxos.items(): + fulltxs.append(jm_single().bc_interface.get_deser_from_gettransaction( + jm_single().bc_interface.get_transaction(utxo[0]))) + + assert len(utxos) == 2 + u_utxos = {} + if unowned_utxo: + # note: tx creation uses the key only; psbt creation uses the value, + # which can be fake here; we do not intend to attempt to fully + # finalize a psbt with an unowned input. See + # https://github.com/Simplexum/python-bitcointx/issues/30 + # the redeem script creation (which is artificial) will be + # avoided in future. + priv = b"\xaa"*32 + b"\x01" + pub = bitcoin.privkey_to_pubkey(priv) + script = bitcoin.pubkey_to_p2sh_p2wpkh_script(pub) + redeem_script = bitcoin.pubkey_to_p2wpkh_script(pub) + u_utxos[(b"\xaa"*32, 12)] = {"value": 1000, "script": script} + utxos.update(u_utxos) + # outputs aren't interesting for this test (we selected 1.5 but will get 2): + outs = [{"value": bitcoin.coins_to_satoshi(1.999), + "address": wallet_service.get_addr(0,0,0)}] + tx = bitcoin.mktx(list(utxos.keys()), outs) + + if wallet_cls != LegacyWallet: + spent_outs = wallet_service.witness_utxos_to_psbt_utxos(utxos) + else: + spent_outs = fulltxs + # the extra input is segwit: + if unowned_utxo: + spent_outs.extend( + wallet_service.witness_utxos_to_psbt_utxos(u_utxos)) + newpsbt = wallet_service.create_psbt_from_tx(tx, spent_outs) + # see note above + if unowned_utxo: + newpsbt.inputs[-1].redeem_script = redeem_script + print(bintohex(newpsbt.serialize())) + # we cannot compare with a fixed expected result due to wallet randomization, but we can + # check psbt structure: + expected_inputs_length = 3 if unowned_utxo else 2 + assert len(newpsbt.inputs) == expected_inputs_length + assert len(newpsbt.outputs) == 1 + # note: redeem_script field is a CScript which is a bytes instance, + # so checking length is best way to check for existence (comparison + # with None does not work): + if wallet_cls == SegwitLegacyWallet: + assert len(newpsbt.inputs[0].redeem_script) != 0 + assert len(newpsbt.inputs[1].redeem_script) != 0 + if unowned_utxo: + assert newpsbt.inputs[2].redeem_script == redeem_script + + signed_psbt_and_signresult, err = wallet_service.sign_psbt( + newpsbt.serialize(), with_sign_result=True) + assert err is None + signresult, signed_psbt = signed_psbt_and_signresult + expected_signed_inputs = len(utxos) if not unowned_utxo else len(utxos)-1 + assert signresult.num_inputs_signed == expected_signed_inputs + assert signresult.num_inputs_final == expected_signed_inputs + + if not unowned_utxo: + assert signresult.is_final + # only in case all signed do we try to broadcast: + extracted_tx = signed_psbt.extract_transaction().serialize() + assert jm_single().bc_interface.pushtx(extracted_tx) + else: + # transaction extraction must fail for not-fully-signed psbts: + with pytest.raises(ValueError) as e: + extracted_tx = signed_psbt.extract_transaction() + + +@pytest.fixture(scope="module") +def setup_psbt_wallet(): + load_test_config()