From f0607812434c139e39276f80a09e7a7dfabff436 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Thu, 23 Apr 2020 16:44:43 +0100 Subject: [PATCH] Add SNICKER support to wallets. This commit uses the now created PSBTWalletMixin and additionally creates a SNICKERWalletMixin, and adds a SNICKERReceiver object to jmclient. A test of the end to end workflow of create and then co-sign a SNICKER coinjoin as per the draft BIP is in test_snicker. Additional changes: updated python-bitcointx dependency to >=1.0.5 Minor refactoring of callbacks in tests and additional redeem script checks to PSBTWalletMixin.sign_psbt. Note that this work replaces #403 . --- jmbitcoin/jmbitcoin/__init__.py | 1 + jmbitcoin/jmbitcoin/secp256k1_ecies.py | 91 ++++++++ jmbitcoin/jmbitcoin/secp256k1_main.py | 14 +- jmbitcoin/jmbitcoin/snicker.py | 58 +++++ jmbitcoin/setup.py | 4 +- jmbitcoin/test/test_ecdh.py | 60 ++++++ jmbitcoin/test/test_ecies.py | 43 ++++ jmclient/jmclient/__init__.py | 1 + jmclient/jmclient/cryptoengine.py | 2 +- jmclient/jmclient/snicker_receiver.py | 224 ++++++++++++++++++++ jmclient/jmclient/wallet.py | 280 ++++++++++++++++++++++++- jmclient/test/commontest.py | 6 + jmclient/test/test_psbt_wallet.py | 7 +- jmclient/test/test_snicker.py | 118 +++++++++++ 14 files changed, 897 insertions(+), 12 deletions(-) create mode 100644 jmbitcoin/jmbitcoin/secp256k1_ecies.py create mode 100644 jmbitcoin/jmbitcoin/snicker.py create mode 100644 jmbitcoin/test/test_ecdh.py create mode 100644 jmbitcoin/test/test_ecies.py create mode 100644 jmclient/jmclient/snicker_receiver.py create mode 100644 jmclient/test/test_snicker.py diff --git a/jmbitcoin/jmbitcoin/__init__.py b/jmbitcoin/jmbitcoin/__init__.py index 2a7f481..bfb3f45 100644 --- a/jmbitcoin/jmbitcoin/__init__.py +++ b/jmbitcoin/jmbitcoin/__init__.py @@ -2,6 +2,7 @@ import coincurve as secp256k1 from jmbitcoin.secp256k1_main import * from jmbitcoin.secp256k1_transaction import * from jmbitcoin.secp256k1_deterministic import * +from jmbitcoin.snicker import * from jmbitcoin.amount import * from jmbitcoin.bip21 import * from bitcointx import select_chain_params diff --git a/jmbitcoin/jmbitcoin/secp256k1_ecies.py b/jmbitcoin/jmbitcoin/secp256k1_ecies.py new file mode 100644 index 0000000..9d928ba --- /dev/null +++ b/jmbitcoin/jmbitcoin/secp256k1_ecies.py @@ -0,0 +1,91 @@ +#!/usr/bin/python +from __future__ import (absolute_import, division, + print_function, unicode_literals) +from builtins import * # noqa: F401 +from future.utils import native +import coincurve as secp256k1 +import base64 +import hmac +import hashlib +import pyaes +import os +import jmbitcoin as btc + +ECIES_MAGIC_BYTES = b'BIE1' + +class ECIESDecryptionError(Exception): + pass + +# AES primitives. See BIP-SNICKER for specification. +def aes_encrypt(key, data, iv): + encrypter = pyaes.Encrypter( + pyaes.AESModeOfOperationCBC(key, iv=native(iv))) + enc_data = encrypter.feed(data) + enc_data += encrypter.feed() + + return enc_data + +def aes_decrypt(key, data, iv): + decrypter = pyaes.Decrypter( + pyaes.AESModeOfOperationCBC(key, iv=native(iv))) + try: + dec_data = decrypter.feed(data) + dec_data += decrypter.feed() + except ValueError: + # note decryption errors can come from PKCS7 padding errors + raise ECIESDecryptionError() + return dec_data + +def ecies_encrypt(message, pubkey): + """ Take a privkey in raw byte serialization, + and a pubkey serialized in compressed, binary format (33 bytes), + and output the shared secret as a 32 byte hash digest output. + The exact calculation is: + shared_secret = SHA256(privkey * pubkey) + .. where * is elliptic curve scalar multiplication. + See https://github.com/bitcoin/bitcoin/blob/master/src/secp256k1/src/modules/ecdh/main_impl.h + for implementation details. + """ + # create an ephemeral pubkey for this encryption: + while True: + r = os.urandom(32) + # use compressed serialization of the pubkey R: + try: + R = btc.privkey_to_pubkey(r + b"\x01") + break + except: + # accounts for improbable overflow: + continue + # note that this is *not* ECDH as in the secp256k1_ecdh module, + # since it uses sha512: + ecdh_key = btc.multiply(r, pubkey) + key = hashlib.sha512(ecdh_key).digest() + iv, key_e, key_m = key[0:16], key[16:32], key[32:] + ciphertext = aes_encrypt(key_e, message, iv=iv) + encrypted = ECIES_MAGIC_BYTES + R + ciphertext + mac = hmac.new(key_m, encrypted, hashlib.sha256).digest() + return base64.b64encode(encrypted + mac) + +def ecies_decrypt(privkey, encrypted): + if len(privkey) == 33 and privkey[-1] == 1: + privkey = privkey[:32] + encrypted = base64.b64decode(encrypted) + if len(encrypted) < 85: + raise Exception('invalid ciphertext: length') + magic = encrypted[:4] + if magic != ECIES_MAGIC_BYTES: + raise ECIESDecryptionError() + ephemeral_pubkey = encrypted[4:37] + try: + testR = secp256k1.PublicKey(ephemeral_pubkey) + except: + raise ECIESDecryptionError() + ciphertext = encrypted[37:-32] + mac = encrypted[-32:] + ecdh_key = btc.multiply(privkey, ephemeral_pubkey) + key = hashlib.sha512(ecdh_key).digest() + iv, key_e, key_m = key[0:16], key[16:32], key[32:] + if mac != hmac.new(key_m, encrypted[:-32], hashlib.sha256).digest(): + raise ECIESDecryptionError() + return aes_decrypt(key_e, ciphertext, iv=iv) + diff --git a/jmbitcoin/jmbitcoin/secp256k1_main.py b/jmbitcoin/jmbitcoin/secp256k1_main.py index 45a5bd6..197809f 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_main.py +++ b/jmbitcoin/jmbitcoin/secp256k1_main.py @@ -8,7 +8,7 @@ import struct import coincurve as secp256k1 from bitcointx import base58 -from bitcointx.core import Hash +from bitcointx.core import Hash, CBitcoinTransaction from bitcointx.core.key import CKeyBase, CPubKey from bitcointx.signmessage import BitcoinMessage @@ -166,6 +166,18 @@ def add_privkeys(priv1, priv2): res += b'\x01' return res +def ecdh(privkey, pubkey): + """ Take a privkey in raw byte serialization, + and a pubkey serialized in compressed, binary format (33 bytes), + and output the shared secret as a 32 byte hash digest output. + The exact calculation is: + shared_secret = SHA256(privkey * pubkey) + .. where * is elliptic curve scalar multiplication. + See https://github.com/bitcoin/bitcoin/blob/master/src/secp256k1/src/modules/ecdh/main_impl.h + for implementation details. + """ + secp_privkey = secp256k1.PrivateKey(privkey) + return secp_privkey.ecdh(pubkey) def ecdsa_raw_sign(msg, priv, diff --git a/jmbitcoin/jmbitcoin/snicker.py b/jmbitcoin/jmbitcoin/snicker.py new file mode 100644 index 0000000..5c6be94 --- /dev/null +++ b/jmbitcoin/jmbitcoin/snicker.py @@ -0,0 +1,58 @@ +from __future__ import (absolute_import, division, + print_function, unicode_literals) +from builtins import * # noqa: F401 + +# Implementation of proposal as per +# https://gist.github.com/AdamISZ/2c13fb5819bd469ca318156e2cf25d79 +# (BIP SNICKER) +# TODO: BIP69 is removed in this implementation, will update BIP draft. + +from jmbitcoin.secp256k1_ecies import * +from jmbitcoin.secp256k1_main import * +from jmbitcoin.secp256k1_transaction import * + +SNICKER_MAGIC_BYTES = b'SNICKER' + +# Flags may be added in future versions +SNICKER_FLAG_NONE = b"\x00" + +def snicker_pubkey_tweak(pub, tweak): + """ use secp256k1 library to perform tweak. + Both `pub` and `tweak` are expected as byte strings + (33 and 32 bytes respectively). + Return value is also a 33 byte string serialization + of the resulting pubkey (compressed). + """ + base_pub = secp256k1.PublicKey(pub) + return base_pub.add(tweak).format() + +def snicker_privkey_tweak(priv, tweak): + """ use secp256k1 library to perform tweak. + Both `priv` and `tweak` are expected as byte strings + (32 or 33 and 32 bytes respectively). + Return value isa 33 byte string serialization + of the resulting private key/secret (with compression flag). + """ + if len(priv) == 33 and priv[-1] == 1: + priv = priv[:-1] + base_priv = secp256k1.PrivateKey(priv) + return base_priv.add(tweak).secret + b'\x01' + +def verify_snicker_output(tx, pub, tweak, spk_type='p2sh-p2wpkh'): + """ A convenience function to check that one output address in a transaction + is a SNICKER-type tweak of an existing key. Returns the index of the output + for which this is True (and there must be only 1), and the derived spk, + or -1 and None if it is not found exactly once. + TODO Add support for other scriptPubKey types. + """ + assert isinstance(tx, btc.CBitcoinTransaction) + expected_destination_pub = snicker_pubkey_tweak(pub, tweak) + expected_destination_spk = pubkey_to_p2sh_p2wpkh_script(expected_destination_pub) + found = 0 + for i, o in enumerate(tx.vout): + if o.scriptPubKey == expected_destination_spk: + found += 1 + found_index = i + if found != 1: + return -1, None + return found_index, expected_destination_spk diff --git a/jmbitcoin/setup.py b/jmbitcoin/setup.py index 018025f..75ab2c8 100644 --- a/jmbitcoin/setup.py +++ b/jmbitcoin/setup.py @@ -9,6 +9,6 @@ setup(name='joinmarketbitcoin', author_email='', license='GPL', packages=['jmbitcoin'], - install_requires=['future', 'coincurve', 'python-bitcointx', 'urldecode'], - python_requires='>=3.5', + install_requires=['future', 'coincurve', 'urldecode', + 'python-bitcointx>=1.0.5', 'pyaes'], zip_safe=False) diff --git a/jmbitcoin/test/test_ecdh.py b/jmbitcoin/test/test_ecdh.py new file mode 100644 index 0000000..c417c46 --- /dev/null +++ b/jmbitcoin/test/test_ecdh.py @@ -0,0 +1,60 @@ +#! /usr/bin/env python +from __future__ import (absolute_import, division, + print_function, unicode_literals) +from builtins import * # noqa: F401 +'''Tests coincurve binding to libsecp256k1 ecdh module code''' + +import hashlib +import jmbitcoin as btc +from jmbase import hextobin +import pytest +import os +import json +testdir = os.path.dirname(os.path.realpath(__file__)) + +def test_ecdh(): + """Using private key test vectors from Bitcoin Core. + 1. Import a set of private keys from the json file. + 2. Calculate the corresponding public keys. + 3. Do ECDH on the cartesian product (x, Y), with x private + and Y public keys, for all combinations. + 4. Compare the result from CoinCurve with the manual + multiplication xY following by hash (sha256). Note that + sha256(xY) is the default hashing function used for ECDH + in libsecp256k1. + + Since there are about 20 private keys in the json file, this + creates around 400 test cases (note xX is still valid). + """ + with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f: + json_data = f.read() + valid_keys_list = json.loads(json_data) + extracted_privkeys = [] + for a in valid_keys_list: + key, hex_key, prop_dict = a + if prop_dict["isPrivkey"]: + c, k = btc.read_privkey(hextobin(hex_key)) + extracted_privkeys.append(k) + extracted_pubkeys = [btc.privkey_to_pubkey(x) for x in extracted_privkeys] + for p in extracted_privkeys: + for P in extracted_pubkeys: + c, k = btc.read_privkey(p) + shared_secret = btc.ecdh(k, P) + assert len(shared_secret) == 32 + # try recreating the shared secret manually: + pre_secret = btc.multiply(p, P) + derived_secret = hashlib.sha256(pre_secret).digest() + assert derived_secret == shared_secret + + # test some important failure cases; null key, overflow case + privkeys_invalid = [b'\x00'*32, hextobin( + 'fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141')] + for p in privkeys_invalid: + with pytest.raises(Exception) as e_info: + shared_secret = btc.ecdh(p, extracted_pubkeys[0]) + pubkeys_invalid = [b'0xff' + extracted_pubkeys[0][1:], b'0x00'*12] + for p in extracted_privkeys: + with pytest.raises(Exception) as e_info: + shared_secret = btc.ecdh(p, pubkeys_invalid[0]) + with pytest.raises(Exception) as e_info: + shared_secret = btc.ecdh(p, pubkeys_invalid[1]) diff --git a/jmbitcoin/test/test_ecies.py b/jmbitcoin/test/test_ecies.py new file mode 100644 index 0000000..4a529c5 --- /dev/null +++ b/jmbitcoin/test/test_ecies.py @@ -0,0 +1,43 @@ +#! /usr/bin/env python +from __future__ import (absolute_import, division, + print_function, unicode_literals) +from builtins import * # noqa: F401 +'''Tests ECIES implementation as defined in BIP-SNICKER +(and will be updated if that is).''' + +from jmbase import hextobin +import jmbitcoin as btc +import base64 +import os +import json +testdir = os.path.dirname(os.path.realpath(__file__)) + +def test_ecies(): + """Using private key test vectors from Bitcoin Core. + 1. Import a set of private keys from the json file. + 2. Calculate the corresponding public keys. + 3. Do ECDH on the cartesian product (x, Y), with x private + and Y public keys, for all combinations. + 4. Compare the result from CoinCurve with the manual + multiplication xY following by hash (sha256). Note that + sha256(xY) is the default hashing function used for ECDH + in libsecp256k1. + + Since there are about 20 private keys in the json file, this + creates around 400 test cases (note xX is still valid). + """ + with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f: + json_data = f.read() + valid_keys_list = json.loads(json_data) + print("got valid keys list") + extracted_privkeys = [] + for a in valid_keys_list: + key, hex_key, prop_dict = a + if prop_dict["isPrivkey"]: + c, k = btc.read_privkey(hextobin(hex_key)) + + extracted_privkeys.append(k) + extracted_pubkeys = [btc.privkey_to_pubkey(x) for x in extracted_privkeys] + for (priv, pub) in zip(extracted_privkeys, extracted_pubkeys): + test_message = base64.b64encode(os.urandom(15)*20) + assert btc.ecies_decrypt(priv, btc.ecies_encrypt(test_message, pub)) == test_message diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index 180e5a1..f088b94 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -55,6 +55,7 @@ from .wallet_utils import ( from .wallet_service import WalletService from .maker import Maker, P2EPMaker from .yieldgenerator import YieldGenerator, YieldGeneratorBasic, ygmain +from .snicker_receiver import SNICKERError, SNICKERReceiver # Set default logging handler to avoid "No handler found" warnings. diff --git a/jmclient/jmclient/cryptoengine.py b/jmclient/jmclient/cryptoengine.py index c24d108..d713191 100644 --- a/jmclient/jmclient/cryptoengine.py +++ b/jmclient/jmclient/cryptoengine.py @@ -191,7 +191,7 @@ class BTCEngine(object): @classmethod def pubkey_to_address(cls, pubkey): script = cls.pubkey_to_script(pubkey) - return str(btc.script_to_address(script, cls.VBYTE)) + return str(btc.CCoinAddress.from_scriptPubKey(script)) @classmethod def pubkey_has_address(cls, pubkey, addr): diff --git a/jmclient/jmclient/snicker_receiver.py b/jmclient/jmclient/snicker_receiver.py new file mode 100644 index 0000000..8d91067 --- /dev/null +++ b/jmclient/jmclient/snicker_receiver.py @@ -0,0 +1,224 @@ +#! /usr/bin/env python +from __future__ import (absolute_import, division, + print_function, unicode_literals) +from builtins import * # noqa: F401 + +import sys +import binascii + +import jmbitcoin as btc +from jmclient.configure import get_p2pk_vbyte, jm_single +from jmbase import (get_log, EXIT_FAILURE, utxo_to_utxostr, + bintohex, hextobin) + +jlog = get_log() + +class SNICKERError(Exception): + pass + +class SNICKERReceiver(object): + supported_flags = [] + import_branch = 0 + # TODO implement http api or similar + # for polling, here just a file: + proposals_source = "proposals.txt" + + def __init__(self, wallet_service, income_threshold=0, + acceptance_callback=None): + """ + Class to manage processing of SNICKER proposals and + co-signs and broadcasts in case the application level + configuration permits. + + `acceptance_callback`, if specified, must have arguments + and return type as for the default_acceptance_callback + in this class. + """ + + # This is a Joinmarket WalletService object. + self.wallet_service = wallet_service + + # The simplest filter on accepting SNICKER joins: + # that they pay a minimum of this value in satoshis, + # which can be negative (to account for fees). + # TODO this will be a config variable. + self.income_threshold = income_threshold + + # The acceptance callback which defines if we accept + # a valid proposal and sign it, or not. + if acceptance_callback is None: + self.acceptance_callback = self.default_acceptance_callback + else: + self.acceptance_callback = acceptance_callback + + # A list of currently viable key candidates; these must + # all be (pub)keys for which the privkey is accessible, + # i.e. they must be in-wallet keys. + # This list will be continuously updated by polling the + # wallet. + self.candidate_keys = [] + + # A list of already processed proposals + self.processed_proposals = [] + + # maintain a list of all successfully broadcast + # SNICKER transactions in the current run. + self.successful_txs = [] + + def poll_for_proposals(self): + """ Intended to be invoked in a LoopingCall or other + event loop. + Retrieves any entries in the proposals_source, then + compares with existing, + and invokes parse_proposal on all new entries. + # TODO considerable thought should go into how to store + proposals cross-runs, and also handling of keys, which + must be optional. + """ + new_proposals = [] + with open(self.proposals_source, "rb") as f: + current_entries = f.readlines() + for entry in current_entries: + if entry in self.processed_proposals: + continue + new_proposals.append(entry) + if not self.process_proposals(new_proposals): + jlog.error("Critical logic error, shutting down.") + sys.exit(EXIT_FAILURE) + self.processed_proposals.extend(new_proposals) + + def default_acceptance_callback(self, our_ins, their_ins, + our_outs, their_outs): + """ Accepts lists of inputs as CTXIns, + a single output (belonging to us) as a CTxOut, + and a list of other outputs (belonging not to us) in same + format, and must return only True or False representing acceptance. + + Note that this code is relying on the calling function to give + accurate information about the outputs. + """ + # we must find the utxo in our wallet to get its amount. + # this serves as a sanity check that the input is indeed + # ours. + # we use get_all* because for these purposes mixdepth + # is irrelevant. + utxos = self.wallet_service.get_all_utxos() + print("gau returned these utxos: ", utxos) + our_in_amts = [] + our_out_amts = [] + for i in our_ins: + utxo_for_i = (i.prevout.hash[::-1], i.prevout.n) + if utxo_for_i not in utxos.keys(): + success, utxostr =utxo_to_utxostr(utxo_for_i) + if not success: + jlog.error("Code error: input utxo in wrong format.") + jlog.debug("The input utxo was not found: " + utxostr) + return False + our_in_amts.append(utxos[utxo_for_i]["value"]) + for o in our_outs: + our_out_amts.append(o.nValue) + if sum(our_out_amts) - sum(our_in_amts) < self.income_threshold: + return False + return True + + def process_proposals(self, proposals): + """ Each entry in `proposals` is of form: + encrypted_proposal - base64 string + key - hex encoded compressed pubkey, or '' + if the key is not null, we attempt to decrypt and + process according to that key, else cycles over all keys. + + If all SNICKER validations succeed, the decision to spend is + entirely dependent on self.acceptance_callback. + If the callback returns True, we co-sign and broadcast the + transaction and also update the wallet with the new + imported key (TODO: future versions will enable searching + for these keys using history + HD tree; note the jmbitcoin + snicker.py module DOES insist on ECDH being correctly used, + so this will always be possible for transactions created here. + + Returned is a list of txids of any transactions which + were broadcast, unless a critical error occurs, in which case + False is returned (to minimize this function's trust in other + parts of the code being executed, if something appears to be + inconsistent, we trigger immediate halt with this return). + """ + + for kp in proposals: + try: + p, k = kp.split(b',') + except: + jlog.error("Invalid proposal string, ignoring: " + kp) + if k is not None: + # note that this operation will succeed as long as + # the key is in the wallet._script_map, which will + # be true if the key is at an HD index lower than + # the current wallet.index_cache + k = hextobin(k.decode('utf-8')) + addr = self.wallet_service.pubkey_to_addr(k) + if not self.wallet_service.is_known_addr(addr): + jlog.debug("Key not recognized as part of our " + "wallet, ignoring.") + continue + # TODO: interface/API of SNICKERWalletMixin would better take + # address as argument here, not privkey: + priv = self.wallet_service.get_key_from_addr(addr) + result = self.wallet_service.parse_proposal_to_signed_tx( + priv, p, self.acceptance_callback) + if result[0] is not None: + tx, tweak, out_spk = result + + # We will: rederive the key as a sanity check, + # and see if it matches the claimed spk. + # Then, we import the key into the wallet + # (even though it's re-derivable from history, this + # is the easiest for a first implementation). + # Finally, we co-sign, then push. + # (Again, simplest function: checks already passed, + # so do it automatically). + # TODO: the more sophisticated actions. + tweaked_key = btc.snicker_pubkey_tweak(k, tweak) + tweaked_spk = btc.pubkey_to_p2sh_p2wpkh_script(tweaked_key) + if not tweaked_spk == out_spk: + jlog.error("The spk derived from the pubkey does " + "not match the scriptPubkey returned from " + "the snicker module - code error.") + return False + # before import, we should derive the tweaked *private* key + # from the tweak, also: + tweaked_privkey = btc.snicker_privkey_tweak(priv, tweak) + if not btc.privkey_to_pubkey(tweaked_privkey) == tweaked_key: + jlog.error("Was not able to recover tweaked pubkey " + "from tweaked privkey - code error.") + jlog.error("Expected: " + bintohex(tweaked_key)) + jlog.error("Got: " + bintohex(btc.privkey_to_pubkey( + tweaked_privkey))) + return False + # the recreated private key matches, so we import to the wallet, + # note that type = None here is because we use the same + # scriptPubKey type as the wallet, this has been implicitly + # checked above by deriving the scriptPubKey. + self.wallet_service.import_private_key(self.import_branch, + self.wallet_service._ENGINE.privkey_to_wif(tweaked_privkey), + key_type=self.wallet_service.TYPE) + + + # TODO condition on automatic brdcst or not + if not jm_single().bc_interface.pushtx(tx.serialize()): + jlog.error("Failed to broadcast SNICKER CJ.") + return False + self.successful_txs.append(tx) + return True + else: + jlog.debug('Failed to parse proposal: ' + result[1]) + continue + else: + # Some extra work to implement checking all possible + # keys. + raise NotImplementedError() + + # Completed processing all proposals without any logic + # errors (whether the proposals were valid or accepted + # or not). + return True + diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index a1b80eb..2a9dc1e 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -5,6 +5,7 @@ import functools import collections import numbers import random +import base64 from binascii import hexlify, unhexlify from datetime import datetime from calendar import timegm @@ -751,6 +752,18 @@ class BaseWallet(object): script_utxos[md][utxo]['height'] = height return script_utxos + + def get_all_utxos(self, include_disabled=False): + """ Get all utxos in the wallet, format of return + is as for get_utxos_by_mixdepth for each mixdepth. + """ + mix_utxos = self.get_utxos_by_mixdepth( + include_disabled=include_disabled) + all_utxos = {} + for d in mix_utxos.values(): + all_utxos.update(d) + return all_utxos + @classmethod def _get_merge_algorithm(cls, algorithm_name=None): if not algorithm_name: @@ -1089,6 +1102,31 @@ class PSBTWalletMixin(object): privkeys.append(self._get_priv_from_path(v2[0])) jmckeys = list(btc.JMCKey(x[0][:-1]) for x in privkeys) new_keystore = DummyKeyStore.from_iterable(jmckeys) + + # for p2sh inputs that we want to sign, the redeem_script + # field must be populated by us, as the counterparty did not + # know it. If this was set in an earlier create-psbt role, + # then overwriting it is harmless (preimage resistance). + if isinstance(self, SegwitLegacyWallet): + for i, txinput in enumerate(new_psbt.inputs): + tu = txinput.utxo + if isinstance(tu, btc.CTxOut): + # witness + if tu.scriptPubKey.is_witness_scriptpubkey(): + # native segwit; no insertion needed. + continue + elif tu.scriptPubKey.is_p2sh(): + try: + path = self.script_to_path(tu.scriptPubKey) + except AssertionError: + # this happens when an input is provided but it's not in + # this wallet; in this case, we cannot set the redeem script. + continue + privkey, _ = self._get_priv_from_path(path) + txinput.redeem_script = btc.pubkey_to_p2wpkh_script( + btc.privkey_to_pubkey(privkey)) + # no else branch; any other form of scriptPubKey will just be + # ignored. try: signresult = new_psbt.sign(new_keystore) except Exception as e: @@ -1098,6 +1136,244 @@ class PSBTWalletMixin(object): else: return (signresult, new_psbt), None +class SNICKERWalletMixin(object): + + SUPPORTED_SNICKER_VERSIONS = bytes([0, 1]) + + def __init__(self, storage, **kwargs): + super(SNICKERWalletMixin, self).__init__(storage, **kwargs) + + def create_snicker_proposal(self, our_input, their_input, our_input_utxo, + their_input_utxo, net_transfer, network_fee, + our_priv, their_pub, our_spk, change_spk, + encrypted=True, version_byte=1): + """ Creates a SNICKER proposal from the given transaction data. + This only applies to existing specification, i.e. SNICKER v 00 or 01. + This is only to be used for Joinmarket and only segwit wallets. + `our_input`, `their_input` - utxo format used in JM wallets, + keyed by (tixd, n), as dicts (currently of single entry). + `our_input_utxo`, `their..` - type CTxOut (contains value, scriptPubKey) + net_transfer - amount, after bitcoin transaction fee, transferred from + Proposer (our) to Receiver (their). May be negative. + network_fee - total bitcoin network transaction fee to be paid (so estimates + must occur before this function). + `our_priv`, `their_pub` - these are the keys to be used in ECDH to derive + the tweak as per the BIP. Note `their_pub` may or may not be associated with + the input of the receiver, so is specified here separately. Note also that + according to the BIP the privkey we use *must* be the one corresponding to + the input we provided, else (properly coded) Receivers will reject our + proposal. + `our_spk` - a scriptPubKey for the Proposer coinjoin output + `change_spk` - a change scriptPubkey for the proposer as per BIP + `encrypted` - whether or not to return the ECIES encrypted version of the + proposal. + `version_byte` - which of currently specified Snicker versions is being + used, (0 for reused address, 1 for inferred key). + returns: + if encrypted is True: + base 64 encoded encrypted transaction proposal as a string + else: + binary serialized plaintext SNICKER message. + """ + assert isinstance(self, PSBTWalletMixin) + # before constructing the bitcoin transaction we must calculate the output + # amounts + # TODO investigate arithmetic for negative transfer + if our_input_utxo.nValue - their_input_utxo.nValue - network_fee <= 0: + raise Exception( + "Cannot create SNICKER proposal, Proposer input too small") + total_input_amount = our_input_utxo.nValue + their_input_utxo.nValue + total_output_amount = total_input_amount - network_fee + receiver_output_amount = their_input_utxo.nValue + net_transfer + proposer_output_amount = total_output_amount - receiver_output_amount + + # we must also use ecdh to calculate the output scriptpubkey for the + # receiver + # First, check that `our_priv` corresponds to scriptPubKey in + # `our_input_utxo` to prevent callers from making useless proposals. + expected_pub = btc.privkey_to_pubkey(our_priv) + expected_spk = self.pubkey_to_script(expected_pub) + assert our_input_utxo.scriptPubKey == expected_spk + # now we create the ecdh based tweak: + tweak_bytes = btc.ecdh(our_priv[:-1], their_pub) + tweaked_pub = btc.snicker_pubkey_tweak(their_pub, tweak_bytes) + # TODO: remove restriction to one scriptpubkey type + tweaked_spk = btc.pubkey_to_p2sh_p2wpkh_script(tweaked_pub) + tweaked_addr, our_addr, change_addr = [str( + btc.CCoinAddress.from_scriptPubKey(x)) for x in ( + tweaked_spk, expected_spk, change_spk)] + # now we must construct the three outputs with correct output amounts. + outputs = [{"address": tweaked_addr, "value": receiver_output_amount}] + outputs.append({"address": our_addr, "value": receiver_output_amount}) + outputs.append({"address": change_addr, + "value": total_output_amount - 2 * receiver_output_amount}) + assert all([x["value"] > 0 for x in outputs]) + + # version and locktime as currently specified in the BIP + # for 0/1 version SNICKER. + tx = btc.make_shuffled_tx([our_input, their_input], outputs, + version=2, locktime=0) + # we need to know which randomized input is ours: + our_index = -1 + for i, inp in enumerate(tx.vin): + if our_input == (inp.prevout.hash[::-1], inp.prevout.n): + our_index = i + assert our_index in [0, 1], "code error: our input not in tx" + spent_outs = [our_input_utxo, their_input_utxo] + if our_index == 1: + spent_outs = spent_outs[::-1] + # create the psbt and then sign our input. + snicker_psbt = self.create_psbt_from_tx(tx, spent_outs=spent_outs) + + # having created the PSBT, sign our input + # TODO this requires bitcointx updated minor version else fails + signed_psbt_and_signresult, err = self.sign_psbt( + snicker_psbt.serialize(), with_sign_result=True) + assert err is None + signresult, partially_signed_psbt = signed_psbt_and_signresult + assert signresult.num_inputs_signed == 1 + assert signresult.num_inputs_final == 1 + assert not signresult.is_final + snicker_serialized_message = btc.SNICKER_MAGIC_BYTES + bytes( + [version_byte]) + btc.SNICKER_FLAG_NONE + tweak_bytes + \ + partially_signed_psbt.serialize() + + if not encrypted: + return snicker_serialized_message + + # encryption has been requested; + # we apply ECIES in the form given by the BIP. + return btc.ecies_encrypt(snicker_serialized_message, their_pub) + + def parse_proposal_to_signed_tx(self, privkey, proposal, + acceptance_callback): + """ Given a candidate privkey (binary and compressed format), + and a candidate encrypted SNICKER proposal, attempt to decrypt + and validate it in all aspects. If validation fails the first + return value is None and the second is the reason as a string. + + If all validation checks pass, the next step is checking + acceptance according to financial rules: the acceptance + callback must be a function that accepts four arguments: + (our_ins, their_ins, our_outs, their_outs), where *ins values + are lists of CTxIns and *outs are lists of CTxOuts, + and must return only True/False where True means that the + transaction should be signed. + + If True is returned from the callback, the following are returned + from this function: + (raw transaction for broadcasting (serialized), + tweak value as bytes, derived output spk belonging to receiver) + + Note: flags is currently always None as version is only 0 or 1. + """ + assert isinstance(self, PSBTWalletMixin) + + our_pub = btc.privkey_to_pubkey(privkey) + + if len(proposal) < 5: + return None, "Invalid proposal, too short." + + if base64.b64decode(proposal)[:4] == btc.ECIES_MAGIC_BYTES: + # attempt decryption and reject if fails: + try: + snicker_message = btc.ecies_decrypt(privkey, proposal) + except Exception as e: + return None, "Failed to decrypt." + repr(e) + else: + snicker_message = proposal + + # magic + version,flag + tweak + psbt: + # TODO replace '20' with the minimum feasible PSBT. + if len(snicker_message) < 7 + 2 + 32 + 20: + return None, "Invalid proposal, too short." + + if snicker_message[:7] != btc.SNICKER_MAGIC_BYTES: + return None, "Invalid SNICKER magic bytes." + + version_byte = bytes([snicker_message[7]]) + flag_byte = bytes([snicker_message[8]]) + if version_byte not in self.SUPPORTED_SNICKER_VERSIONS: + return None, "Unrecognized SNICKER version: " + version_byte + if flag_byte != btc.SNICKER_FLAG_NONE: + return None, "Invalid flag byte for version 0,1: " + flag_byte + + tweak_bytes = snicker_message[9:41] + candidate_psbt_serialized = snicker_message[41:] + # attempt to validate the PSBT's format: + try: + cpsbt = btc.PartiallySignedTransaction.from_base64_or_binary( + candidate_psbt_serialized) + except: + return None, "Invalid PSBT format." + + utx = cpsbt.unsigned_tx + # validate that it contains one signature, and two inputs. + # else the proposal is invalid. To achieve this, we call + # PartiallySignedTransaction.sign() with an empty KeyStore, + # which populates the 'is_signed' info fields for us. Note that + # we do not use the PSBTWalletMixin.sign_psbt() which automatically + # signs with our keys. + if not len(utx.vin) == 2: + return None, "PSBT proposal does not contain 2 inputs." + testsignresult = cpsbt.sign(btc.KeyStore(), finalize=False) + print("got sign result: ", testsignresult) + # Note: "num_inputs_signed" refers to how many *we* signed, + # which is obviously none here as we provided no keys. + if not (testsignresult.num_inputs_signed == 0 and \ + testsignresult.num_inputs_final == 1 and \ + not testsignresult.is_final): + return None, "PSBT proposal does not contain 1 signature." + + # Validate that we own one SNICKER style output: + spk = btc.verify_snicker_output(utx, our_pub, tweak_bytes) + + if spk[0] == -1: + return None, "Tweaked destination not found exactly once." + our_output_index = spk[0] + our_output_amount = utx.vout[our_output_index].nValue + + # At least one other output must have an amount equal to that at + # `our_output_index`, according to the spec. + found = 0 + for i, o in enumerate(utx.vout): + if i == our_output_index: + continue + if o.nValue == our_output_amount: + found += 1 + if found != 1: + return None, "Invalid SNICKER, there are not two equal outputs." + + # To allow the acceptance callback to assess validity, we must identify + # which input is ours and which is(are) not. + # TODO This check may (will) change if we allow non-p2sh-pwpkh inputs: + unsigned_index = -1 + for i, psbtinputsigninfo in enumerate(testsignresult.inputs_info): + if psbtinputsigninfo is None: + unsigned_index = i + break + assert unsigned_index != -1 + # All validation checks passed. We now check whether the + #transaction is acceptable according to the caller: + if not acceptance_callback([utx.vin[unsigned_index]], + [x for i, x in enumerate(utx.vin) if i != unsigned_index], + [utx.vout[our_output_index]], + [x for i, x in enumerate(utx.vout) if i != our_output_index]): + return None, "Caller rejected transaction for signing." + + # Acceptance passed, prepare the deserialized tx for signing by us: + signresult_and_signedpsbt, err = self.sign_psbt(cpsbt.serialize(), + with_sign_result=True) + if err: + return None, "Unable to sign proposed PSBT, reason: " + err + signresult, signed_psbt = signresult_and_signedpsbt + assert signresult.num_inputs_signed == 1 + assert signresult.num_inputs_final == 2 + assert signresult.is_final + # we now know the transaction is valid and fully signed; return to caller, + # along with supporting data for this tx: + return (signed_psbt.extract_transaction(), tweak_bytes, spk[1]) + class ImportWalletMixin(object): """ Mixin for BaseWallet to support importing keys. @@ -1916,10 +2192,10 @@ class BIP84Wallet(BIP32PurposedWallet): _PURPOSE = 2**31 + 84 _ENGINE = ENGINES[TYPE_P2WPKH] -class SegwitLegacyWallet(ImportWalletMixin, BIP39WalletMixin, PSBTWalletMixin, BIP49Wallet): +class SegwitLegacyWallet(ImportWalletMixin, BIP39WalletMixin, PSBTWalletMixin, SNICKERWalletMixin, BIP49Wallet): TYPE = TYPE_P2SH_P2WPKH -class SegwitWallet(ImportWalletMixin, BIP39WalletMixin, PSBTWalletMixin, BIP84Wallet): +class SegwitWallet(ImportWalletMixin, BIP39WalletMixin, PSBTWalletMixin, SNICKERWalletMixin, BIP84Wallet): TYPE = TYPE_P2WPKH class SegwitLegacyWalletFidelityBonds(FidelityBondMixin, SegwitLegacyWallet): diff --git a/jmclient/test/commontest.py b/jmclient/test/commontest.py index 36ed7c4..6295b17 100644 --- a/jmclient/test/commontest.py +++ b/jmclient/test/commontest.py @@ -27,6 +27,12 @@ PINL = '\r\n' if OS == 'Windows' else '\n' default_max_cj_fee = (1, float('inf')) +# callbacks for making transfers in-script with direct_send: +def dummy_accept_callback(tx, destaddr, actual_amount, fee_est): + return True +def dummy_info_callback(msg): + pass + class DummyBlockchainInterface(BlockchainInterface): def __init__(self): self.fake_query_results = None diff --git a/jmclient/test/test_psbt_wallet.py b/jmclient/test/test_psbt_wallet.py index a2c4cd6..93ac335 100644 --- a/jmclient/test/test_psbt_wallet.py +++ b/jmclient/test/test_psbt_wallet.py @@ -10,7 +10,7 @@ import time import binascii import struct import copy -from commontest import make_wallets +from commontest import make_wallets, dummy_accept_callback, dummy_info_callback import jmbitcoin as bitcoin import pytest @@ -20,11 +20,6 @@ from jmclient import (load_test_config, jm_single, direct_send, log = get_log() -def dummy_accept_callback(tx, destaddr, actual_amount, fee_est): - return True -def dummy_info_callback(msg): - pass - def test_create_and_sign_psbt_with_legacy(setup_psbt_wallet): """ The purpose of this test is to check that we can create and then partially sign a PSBT where we own one input and the other input diff --git a/jmclient/test/test_snicker.py b/jmclient/test/test_snicker.py new file mode 100644 index 0000000..8417a78 --- /dev/null +++ b/jmclient/test/test_snicker.py @@ -0,0 +1,118 @@ +#! /usr/bin/env python +from __future__ import (absolute_import, division, + print_function, unicode_literals) +from builtins import * # noqa: F401 +'''Test of unusual transaction types creation and push to +network to check validity.''' + +import binascii +from commontest import make_wallets, dummy_accept_callback, dummy_info_callback + +import jmbitcoin as btc +import pytest +from jmbase import get_log, bintohex, hextobin +from jmclient import (load_test_config, jm_single, + estimate_tx_fee, SNICKERReceiver, direct_send) + +log = get_log() + +@pytest.mark.parametrize( + "nw, wallet_structures, mean_amt, sdev_amt, amt, net_transfer", [ + (2, [[1, 0, 0, 0, 0]] * 2, 4, 0, 20000000, 1000), + ]) +def test_snicker_e2e(setup_snicker, nw, wallet_structures, + mean_amt, sdev_amt, amt, net_transfer): + """ Test strategy: + 1. create two wallets. + 2. with wallet 1 (Receiver), create a single transaction + tx1, from mixdepth 0 to 1. + 3. with wallet 2 (Proposer), take pubkey of all inputs from tx1, and use + them to create snicker proposals to the non-change out of tx1, + in base64 and place in proposals.txt. + 4. Receiver polls for proposals in the file manually (instead of twisted + LoopingCall) and processes them. + 5. Check for valid final transaction with broadcast. + """ + wallets = make_wallets(nw, wallet_structures, mean_amt, sdev_amt) + for w in wallets.values(): + w['wallet'].sync_wallet(fast=True) + print(wallets) + wallet_r = wallets[0]['wallet'] + wallet_p = wallets[1]['wallet'] + # next, create a tx from the receiver wallet + our_destn_script = wallet_r.get_new_script(1, 1) + tx = direct_send(wallet_r, btc.coins_to_satoshi(0.3), 0, + wallet_r.script_to_addr(our_destn_script), + accept_callback=dummy_accept_callback, + info_callback=dummy_info_callback, + return_transaction=True) + + assert tx, "Failed to spend from receiver wallet" + print("Parent transaction OK. It was: ") + print(tx) + wallet_r.process_new_tx(tx) + # we must identify the receiver's output we're going to use; + # it can be destination or change, that's up to the proposer + # to guess successfully; here we'll just choose index 0. + txid1 = tx.GetTxid()[::-1] + txid1_index = 0 + + receiver_start_bal = sum([x['value'] for x in wallet_r.get_all_utxos( + ).values()]) + + # Now create a proposal for every input index in tx1 + # (version 1 proposals mean we source keys from the/an + # ancestor transaction) + propose_keys = [] + for i in range(len(tx.vin)): + # todo check access to pubkey + sig, pub = [a for a in iter(tx.wit.vtxinwit[i].scriptWitness)] + propose_keys.append(pub) + # the proposer wallet needs to choose a single + # utxo that is bigger than the output amount of tx1 + prop_m_utxos = wallet_p.get_utxos_by_mixdepth()[0] + prop_utxo = prop_m_utxos[list(prop_m_utxos)[0]] + # get the private key for that utxo + priv = wallet_p.get_key_from_addr( + wallet_p.script_to_addr(prop_utxo['script'])) + prop_input_amt = prop_utxo['value'] + # construct the arguments for the snicker proposal: + our_input = list(prop_m_utxos)[0] # should be (txid, index) + their_input = (txid1, txid1_index) + our_input_utxo = btc.CMutableTxOut(prop_utxo['value'], + prop_utxo['script']) + fee_est = estimate_tx_fee(len(tx.vin), 2) + change_spk = wallet_p.get_new_script(0, 1) + + encrypted_proposals = [] + + for p in propose_keys: + # TODO: this can be a loop over all outputs, + # not just one guessed output, if desired. + encrypted_proposals.append( + wallet_p.create_snicker_proposal( + our_input, their_input, + our_input_utxo, + tx.vout[txid1_index], + net_transfer, + fee_est, + priv, + p, + prop_utxo['script'], + change_spk, + version_byte=1) + b"," + bintohex(p).encode('utf-8')) + with open("test_proposals.txt", "wb") as f: + f.write(b"\n".join(encrypted_proposals)) + sR = SNICKERReceiver(wallet_r) + sR.proposals_source = "test_proposals.txt" # avoid clashing with mainnet + sR.poll_for_proposals() + assert len(sR.successful_txs) == 1 + wallet_r.process_new_tx(sR.successful_txs[0]) + end_utxos = wallet_r.get_all_utxos() + print("At end the receiver has these utxos: ", end_utxos) + receiver_end_bal = sum([x['value'] for x in end_utxos.values()]) + assert receiver_end_bal == receiver_start_bal + net_transfer + +@pytest.fixture(scope="module") +def setup_snicker(): + load_test_config()