Add SNICKER support to wallets.

This commit uses the now created PSBTWalletMixin and additionally
creates a SNICKERWalletMixin, and adds a SNICKERReceiver object
to jmclient. A test of the end to end workflow of create and then
co-sign a SNICKER coinjoin as per the draft BIP is in test_snicker.
Additional changes:
updated python-bitcointx dependency to >=1.0.5
Minor refactoring of callbacks in tests and additional redeem
script checks to PSBTWalletMixin.sign_psbt.

Note that this work replaces #403 .

jmbitcoin/jmbitcoin/__init__.py

@@ -2,6 +2,7 @@ import coincurve as secp256k1
 from jmbitcoin.secp256k1_main import *
 from jmbitcoin.secp256k1_transaction import *
 from jmbitcoin.secp256k1_deterministic import *
+from jmbitcoin.snicker import *
 from jmbitcoin.amount import *
 from jmbitcoin.bip21 import *
 from bitcointx import select_chain_params

jmbitcoin/jmbitcoin/secp256k1_ecies.py

@@ -0,0 +1,91 @@
+#!/usr/bin/python
+from __future__ import (absolute_import, division,
+                        print_function, unicode_literals)
+from builtins import * # noqa: F401
+from future.utils import native
+import coincurve as secp256k1
+import base64
+import hmac
+import hashlib
+import pyaes
+import os
+import jmbitcoin as btc
+
+ECIES_MAGIC_BYTES = b'BIE1'
+
+class ECIESDecryptionError(Exception):
+    pass
+
+# AES primitives. See BIP-SNICKER for specification.
+def aes_encrypt(key, data, iv):
+    encrypter = pyaes.Encrypter(
+        pyaes.AESModeOfOperationCBC(key, iv=native(iv)))
+    enc_data = encrypter.feed(data)
+    enc_data += encrypter.feed()
+
+    return enc_data
+
+def aes_decrypt(key, data, iv):
+    decrypter = pyaes.Decrypter(
+        pyaes.AESModeOfOperationCBC(key, iv=native(iv)))
+    try:
+        dec_data = decrypter.feed(data)
+        dec_data += decrypter.feed()
+    except ValueError:
+        # note decryption errors can come from PKCS7 padding errors
+        raise ECIESDecryptionError()
+    return dec_data
+
+def ecies_encrypt(message, pubkey):
+    """ Take a privkey in raw byte serialization,
+    and a pubkey serialized in compressed, binary format (33 bytes),
+    and output the shared secret as a 32 byte hash digest output.
+    The exact calculation is:
+    shared_secret = SHA256(privkey * pubkey)
+    .. where * is elliptic curve scalar multiplication.
+    See https://github.com/bitcoin/bitcoin/blob/master/src/secp256k1/src/modules/ecdh/main_impl.h
+    for implementation details.
+    """
+    # create an ephemeral pubkey for this encryption:
+    while True:
+        r = os.urandom(32)
+        # use compressed serialization of the pubkey R:
+        try:
+            R = btc.privkey_to_pubkey(r + b"\x01")
+            break
+        except:
+            # accounts for improbable overflow:
+            continue
+    # note that this is *not* ECDH as in the secp256k1_ecdh module,
+    # since it uses sha512:
+    ecdh_key = btc.multiply(r, pubkey)
+    key = hashlib.sha512(ecdh_key).digest()
+    iv, key_e, key_m = key[0:16], key[16:32], key[32:]
+    ciphertext = aes_encrypt(key_e, message, iv=iv)
+    encrypted = ECIES_MAGIC_BYTES + R + ciphertext
+    mac = hmac.new(key_m, encrypted, hashlib.sha256).digest()
+    return base64.b64encode(encrypted + mac)    
+
+def ecies_decrypt(privkey, encrypted):
+    if len(privkey) == 33 and privkey[-1] == 1:
+        privkey = privkey[:32]
+    encrypted = base64.b64decode(encrypted)
+    if len(encrypted) < 85:
+        raise Exception('invalid ciphertext: length')
+    magic = encrypted[:4]
+    if magic != ECIES_MAGIC_BYTES:
+        raise ECIESDecryptionError()
+    ephemeral_pubkey = encrypted[4:37]
+    try:
+        testR = secp256k1.PublicKey(ephemeral_pubkey)
+    except:
+        raise ECIESDecryptionError()
+    ciphertext = encrypted[37:-32]
+    mac = encrypted[-32:]
+    ecdh_key = btc.multiply(privkey, ephemeral_pubkey)
+    key = hashlib.sha512(ecdh_key).digest()
+    iv, key_e, key_m = key[0:16], key[16:32], key[32:]
+    if mac != hmac.new(key_m, encrypted[:-32], hashlib.sha256).digest():
+        raise ECIESDecryptionError()
+    return aes_decrypt(key_e, ciphertext, iv=iv)    
+

jmbitcoin/jmbitcoin/secp256k1_main.py

@@ -8,7 +8,7 @@ import struct
 import coincurve as secp256k1
 
 from bitcointx import base58
-from bitcointx.core import Hash
+from bitcointx.core import Hash, CBitcoinTransaction
 from bitcointx.core.key import CKeyBase, CPubKey
 from bitcointx.signmessage import BitcoinMessage
 
@@ -166,6 +166,18 @@ def add_privkeys(priv1, priv2):
         res += b'\x01'
     return res
 
+def ecdh(privkey, pubkey):
+    """ Take a privkey in raw byte serialization,
+    and a pubkey serialized in compressed, binary format (33 bytes),
+    and output the shared secret as a 32 byte hash digest output.
+    The exact calculation is:
+    shared_secret = SHA256(privkey * pubkey)
+    .. where * is elliptic curve scalar multiplication.
+    See https://github.com/bitcoin/bitcoin/blob/master/src/secp256k1/src/modules/ecdh/main_impl.h
+    for implementation details.
+    """
+    secp_privkey = secp256k1.PrivateKey(privkey)
+    return secp_privkey.ecdh(pubkey)
 
 def ecdsa_raw_sign(msg,
                    priv,

jmbitcoin/jmbitcoin/snicker.py

@@ -0,0 +1,58 @@
+from __future__ import (absolute_import, division,
+                        print_function, unicode_literals)
+from builtins import * # noqa: F401
+
+# Implementation of proposal as per
+# https://gist.github.com/AdamISZ/2c13fb5819bd469ca318156e2cf25d79
+# (BIP SNICKER)
+# TODO: BIP69 is removed in this implementation, will update BIP draft.
+
+from jmbitcoin.secp256k1_ecies import *
+from jmbitcoin.secp256k1_main import *
+from jmbitcoin.secp256k1_transaction import *
+
+SNICKER_MAGIC_BYTES = b'SNICKER'
+
+# Flags may be added in future versions
+SNICKER_FLAG_NONE = b"\x00"
+
+def snicker_pubkey_tweak(pub, tweak):
+    """ use secp256k1 library to perform tweak.
+    Both `pub` and `tweak` are expected as byte strings
+    (33 and 32 bytes respectively).
+    Return value is also a 33 byte string serialization
+    of the resulting pubkey (compressed).
+    """
+    base_pub = secp256k1.PublicKey(pub)
+    return base_pub.add(tweak).format()
+
+def snicker_privkey_tweak(priv, tweak):
+    """ use secp256k1 library to perform tweak.
+    Both `priv` and `tweak` are expected as byte strings
+    (32 or 33 and 32 bytes respectively).
+    Return value isa 33 byte string serialization
+    of the resulting private key/secret (with compression flag).
+    """
+    if len(priv) == 33 and priv[-1] == 1:
+        priv = priv[:-1]
+    base_priv = secp256k1.PrivateKey(priv)
+    return base_priv.add(tweak).secret + b'\x01'
+
+def verify_snicker_output(tx, pub, tweak, spk_type='p2sh-p2wpkh'):
+    """ A convenience function to check that one output address in a transaction
+    is a SNICKER-type tweak of an existing key. Returns the index of the output
+    for which this is True (and there must be only 1), and the derived spk,
+    or -1 and None if it is not found exactly once.
+    TODO Add support for other scriptPubKey types.
+    """
+    assert isinstance(tx, btc.CBitcoinTransaction)
+    expected_destination_pub = snicker_pubkey_tweak(pub, tweak)
+    expected_destination_spk = pubkey_to_p2sh_p2wpkh_script(expected_destination_pub)
+    found = 0
+    for i, o in enumerate(tx.vout):
+        if o.scriptPubKey == expected_destination_spk:
+            found += 1
+            found_index = i
+    if found != 1:
+        return -1, None
+    return found_index, expected_destination_spk

jmbitcoin/setup.py

@@ -9,6 +9,6 @@ setup(name='joinmarketbitcoin',
       author_email='',
       license='GPL',
       packages=['jmbitcoin'],
-      install_requires=['future', 'coincurve', 'python-bitcointx', 'urldecode'],
-      python_requires='>=3.5',
+      install_requires=['future', 'coincurve', 'urldecode',
+                        'python-bitcointx>=1.0.5', 'pyaes'],
       zip_safe=False)

jmbitcoin/test/test_ecdh.py

@@ -0,0 +1,60 @@
+#! /usr/bin/env python
+from __future__ import (absolute_import, division,
+                        print_function, unicode_literals)
+from builtins import * # noqa: F401
+'''Tests coincurve binding to libsecp256k1 ecdh module code'''
+
+import hashlib
+import jmbitcoin as btc
+from jmbase import hextobin
+import pytest
+import os
+import json
+testdir = os.path.dirname(os.path.realpath(__file__))
+
+def test_ecdh():
+    """Using private key test vectors from Bitcoin Core.
+    1. Import a set of private keys from the json file.
+    2. Calculate the corresponding public keys.
+    3. Do ECDH on the cartesian product (x, Y), with x private
+    and Y public keys, for all combinations.
+    4. Compare the result from CoinCurve with the manual
+    multiplication xY following by hash (sha256). Note that
+    sha256(xY) is the default hashing function used for ECDH
+    in libsecp256k1.
+
+    Since there are about 20 private keys in the json file, this
+    creates around 400 test cases (note xX is still valid).
+    """
+    with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f:
+        json_data = f.read()
+        valid_keys_list = json.loads(json_data)
+        extracted_privkeys = []
+        for a in valid_keys_list:
+            key, hex_key, prop_dict = a
+            if prop_dict["isPrivkey"]:
+                c, k = btc.read_privkey(hextobin(hex_key))
+                extracted_privkeys.append(k)
+    extracted_pubkeys = [btc.privkey_to_pubkey(x) for x in extracted_privkeys]
+    for p in extracted_privkeys:
+        for P in extracted_pubkeys:
+            c, k = btc.read_privkey(p)
+            shared_secret = btc.ecdh(k, P)
+            assert len(shared_secret) == 32
+            # try recreating the shared secret manually:
+            pre_secret = btc.multiply(p, P)
+            derived_secret = hashlib.sha256(pre_secret).digest()
+            assert derived_secret == shared_secret
+
+    # test some important failure cases; null key, overflow case
+    privkeys_invalid = [b'\x00'*32, hextobin(
+        'fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141')]
+    for p in privkeys_invalid:
+        with pytest.raises(Exception) as e_info:
+            shared_secret = btc.ecdh(p, extracted_pubkeys[0])
+    pubkeys_invalid = [b'0xff' + extracted_pubkeys[0][1:], b'0x00'*12]
+    for p in extracted_privkeys:
+        with pytest.raises(Exception) as e_info:
+            shared_secret = btc.ecdh(p, pubkeys_invalid[0])
+        with pytest.raises(Exception) as e_info:
+            shared_secret = btc.ecdh(p, pubkeys_invalid[1])

jmbitcoin/test/test_ecies.py

@@ -0,0 +1,43 @@
+#! /usr/bin/env python
+from __future__ import (absolute_import, division,
+                        print_function, unicode_literals)
+from builtins import * # noqa: F401
+'''Tests ECIES implementation as defined in BIP-SNICKER
+(and will be updated if that is).'''
+
+from jmbase import hextobin
+import jmbitcoin as btc
+import base64
+import os
+import json
+testdir = os.path.dirname(os.path.realpath(__file__))
+
+def test_ecies():
+    """Using private key test vectors from Bitcoin Core.
+    1. Import a set of private keys from the json file.
+    2. Calculate the corresponding public keys.
+    3. Do ECDH on the cartesian product (x, Y), with x private
+    and Y public keys, for all combinations.
+    4. Compare the result from CoinCurve with the manual
+    multiplication xY following by hash (sha256). Note that
+    sha256(xY) is the default hashing function used for ECDH
+    in libsecp256k1.
+
+    Since there are about 20 private keys in the json file, this
+    creates around 400 test cases (note xX is still valid).
+    """
+    with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f:
+        json_data = f.read()
+        valid_keys_list = json.loads(json_data)
+        print("got valid keys list")
+        extracted_privkeys = []
+        for a in valid_keys_list:
+            key, hex_key, prop_dict = a
+            if prop_dict["isPrivkey"]:
+                c, k = btc.read_privkey(hextobin(hex_key))
+
+                extracted_privkeys.append(k)
+    extracted_pubkeys = [btc.privkey_to_pubkey(x) for x in extracted_privkeys]
+    for (priv, pub) in zip(extracted_privkeys, extracted_pubkeys):
+        test_message = base64.b64encode(os.urandom(15)*20)
+        assert btc.ecies_decrypt(priv, btc.ecies_encrypt(test_message, pub)) == test_message

jmclient/jmclient/__init__.py

@@ -55,6 +55,7 @@ from .wallet_utils import (
 from .wallet_service import WalletService
 from .maker import Maker, P2EPMaker
 from .yieldgenerator import YieldGenerator, YieldGeneratorBasic, ygmain
+from .snicker_receiver import SNICKERError, SNICKERReceiver
 
 # Set default logging handler to avoid "No handler found" warnings.
 

jmclient/jmclient/cryptoengine.py

@@ -191,7 +191,7 @@ class BTCEngine(object):
     @classmethod
     def pubkey_to_address(cls, pubkey):
         script = cls.pubkey_to_script(pubkey)
-        return str(btc.script_to_address(script, cls.VBYTE))
+        return str(btc.CCoinAddress.from_scriptPubKey(script))
 
     @classmethod
     def pubkey_has_address(cls, pubkey, addr):

jmclient/jmclient/snicker_receiver.py

@@ -0,0 +1,224 @@
+#! /usr/bin/env python
+from __future__ import (absolute_import, division,
+                        print_function, unicode_literals)
+from builtins import * # noqa: F401
+
+import sys
+import binascii
+
+import jmbitcoin as btc
+from jmclient.configure import get_p2pk_vbyte, jm_single
+from jmbase import (get_log, EXIT_FAILURE, utxo_to_utxostr,
+                    bintohex, hextobin)
+
+jlog = get_log()
+
+class SNICKERError(Exception):
+    pass
+
+class SNICKERReceiver(object):
+    supported_flags = []
+    import_branch = 0
+    # TODO implement http api or similar
+    # for polling, here just a file:
+    proposals_source = "proposals.txt"
+
+    def __init__(self, wallet_service, income_threshold=0,
+                 acceptance_callback=None):
+        """
+        Class to manage processing of SNICKER proposals and
+        co-signs and broadcasts in case the application level
+        configuration permits.
+
+        `acceptance_callback`, if specified, must have arguments
+        and return type as for the default_acceptance_callback
+        in this class.
+        """
+
+        # This is a Joinmarket WalletService object.
+        self.wallet_service = wallet_service
+
+        # The simplest filter on accepting SNICKER joins:
+        # that they pay a minimum of this value in satoshis,
+        # which can be negative (to account for fees).
+        # TODO this will be a config variable.
+        self.income_threshold = income_threshold
+
+        # The acceptance callback which defines if we accept
+        # a valid proposal and sign it, or not.
+        if acceptance_callback is None:
+            self.acceptance_callback = self.default_acceptance_callback
+        else:
+            self.acceptance_callback = acceptance_callback
+
+        # A list of currently viable key candidates; these must
+        # all be (pub)keys for which the privkey is accessible,
+        # i.e. they must be in-wallet keys.
+        # This list will be continuously updated by polling the
+        # wallet.
+        self.candidate_keys = []
+
+        # A list of already processed proposals
+        self.processed_proposals = []
+
+        # maintain a list of all successfully broadcast
+        # SNICKER transactions in the current run.
+        self.successful_txs = []
+
+    def poll_for_proposals(self):
+        """ Intended to be invoked in a LoopingCall or other
+        event loop.
+        Retrieves any entries in the proposals_source, then
+        compares with existing,
+        and invokes parse_proposal on all new entries.
+        # TODO considerable thought should go into how to store
+        proposals cross-runs, and also handling of keys, which
+        must be optional.
+        """
+        new_proposals = []
+        with open(self.proposals_source, "rb") as f:
+            current_entries = f.readlines()
+        for entry in current_entries:
+            if entry in self.processed_proposals:
+                continue
+            new_proposals.append(entry)
+        if not self.process_proposals(new_proposals):
+            jlog.error("Critical logic error, shutting down.")
+            sys.exit(EXIT_FAILURE)
+        self.processed_proposals.extend(new_proposals)
+
+    def default_acceptance_callback(self, our_ins, their_ins,
+                                    our_outs, their_outs):
+        """ Accepts lists of inputs as CTXIns,
+        a single output (belonging to us) as a CTxOut,
+        and a list of other outputs (belonging not to us) in same
+        format, and must return only True or False representing acceptance.
+
+        Note that this code is relying on the calling function to give
+        accurate information about the outputs.
+        """
+        # we must find the utxo in our wallet to get its amount.
+        # this serves as a sanity check that the input is indeed
+        # ours.
+        # we use get_all* because for these purposes mixdepth
+        # is irrelevant.
+        utxos = self.wallet_service.get_all_utxos()
+        print("gau returned these utxos: ", utxos)
+        our_in_amts = []
+        our_out_amts = []
+        for i in our_ins:
+            utxo_for_i = (i.prevout.hash[::-1], i.prevout.n)
+            if  utxo_for_i not in utxos.keys():
+                success, utxostr =utxo_to_utxostr(utxo_for_i)
+                if not success:
+                    jlog.error("Code error: input utxo in wrong format.")
+                jlog.debug("The input utxo was not found: " + utxostr)
+                return False
+            our_in_amts.append(utxos[utxo_for_i]["value"])
+        for o in our_outs:
+            our_out_amts.append(o.nValue)
+        if sum(our_out_amts) - sum(our_in_amts) < self.income_threshold:
+            return False
+        return True
+
+    def process_proposals(self, proposals):
+        """ Each entry in `proposals` is of form:
+        encrypted_proposal - base64 string
+        key - hex encoded compressed pubkey, or ''
+        if the key is not null, we attempt to decrypt and
+        process according to that key, else cycles over all keys.
+
+        If all SNICKER validations succeed, the decision to spend is
+        entirely dependent on self.acceptance_callback.
+        If the callback returns True, we co-sign and broadcast the
+        transaction and also update the wallet with the new
+        imported key (TODO: future versions will enable searching
+        for these keys using history + HD tree; note the jmbitcoin
+        snicker.py module DOES insist on ECDH being correctly used,
+        so this will always be possible for transactions created here.
+
+        Returned is a list of txids of any transactions which
+        were broadcast, unless a critical error occurs, in which case
+        False is returned (to minimize this function's trust in other
+        parts of the code being executed, if something appears to be
+        inconsistent, we trigger immediate halt with this return).
+        """
+
+        for kp in proposals:
+            try:
+                p, k = kp.split(b',')
+            except:
+                jlog.error("Invalid proposal string, ignoring: " + kp)
+            if k is not None:
+                # note that this operation will succeed as long as
+                # the key is in the wallet._script_map, which will
+                # be true if the key is at an HD index lower than
+                # the current wallet.index_cache
+                k = hextobin(k.decode('utf-8'))
+                addr = self.wallet_service.pubkey_to_addr(k)
+                if not self.wallet_service.is_known_addr(addr):
+                    jlog.debug("Key not recognized as part of our "
+                               "wallet, ignoring.")
+                    continue
+                # TODO: interface/API of SNICKERWalletMixin would better take
+                # address as argument here, not privkey:
+                priv = self.wallet_service.get_key_from_addr(addr)
+                result = self.wallet_service.parse_proposal_to_signed_tx(
+                    priv, p, self.acceptance_callback)
+                if result[0] is not None:
+                    tx, tweak, out_spk = result
+
+                    # We will: rederive the key as a sanity check,
+                    # and see if it matches the claimed spk.
+                    # Then, we import the key into the wallet
+                    # (even though it's re-derivable from history, this
+                    # is the easiest for a first implementation).
+                    # Finally, we co-sign, then push.
+                    # (Again, simplest function: checks already passed,
+                    # so do it automatically).
+                    # TODO: the more sophisticated actions.
+                    tweaked_key = btc.snicker_pubkey_tweak(k, tweak)
+                    tweaked_spk = btc.pubkey_to_p2sh_p2wpkh_script(tweaked_key)
+                    if not tweaked_spk == out_spk:
+                        jlog.error("The spk derived from the pubkey does "
+                                   "not match the scriptPubkey returned from "
+                                   "the snicker module - code error.")
+                        return False
+                    # before import, we should derive the tweaked *private* key
+                    # from the tweak, also:
+                    tweaked_privkey = btc.snicker_privkey_tweak(priv, tweak)
+                    if not btc.privkey_to_pubkey(tweaked_privkey) == tweaked_key:
+                        jlog.error("Was not able to recover tweaked pubkey "
+                                   "from tweaked privkey - code error.")
+                        jlog.error("Expected: " + bintohex(tweaked_key))
+                        jlog.error("Got: " + bintohex(btc.privkey_to_pubkey(
+                            tweaked_privkey)))
+                        return False
+                    # the recreated private key matches, so we import to the wallet,
+                    # note that type = None here is because we use the same
+                    # scriptPubKey type as the wallet, this has been implicitly
+                    # checked above by deriving the scriptPubKey.
+                    self.wallet_service.import_private_key(self.import_branch,
+                            self.wallet_service._ENGINE.privkey_to_wif(tweaked_privkey),
+                            key_type=self.wallet_service.TYPE)
+
+
+                    # TODO condition on automatic brdcst or not
+                    if not jm_single().bc_interface.pushtx(tx.serialize()):
+                        jlog.error("Failed to broadcast SNICKER CJ.")
+                        return False
+                    self.successful_txs.append(tx)
+                    return True
+                else:
+                    jlog.debug('Failed to parse proposal: ' + result[1])
+                    continue
+            else:
+                # Some extra work to implement checking all possible
+                # keys.
+                raise NotImplementedError()
+
+        # Completed processing all proposals without any logic
+        # errors (whether the proposals were valid or accepted
+        # or not).
+        return True
+

jmclient/jmclient/wallet.py

@@ -5,6 +5,7 @@ import functools
 import collections
 import numbers
 import random
+import base64
 from binascii import hexlify, unhexlify
 from datetime import datetime
 from calendar import timegm
@@ -751,6 +752,18 @@ class BaseWallet(object):
                     script_utxos[md][utxo]['height'] = height
         return script_utxos
 
+
+    def get_all_utxos(self, include_disabled=False):
+        """ Get all utxos in the wallet, format of return
+        is as for get_utxos_by_mixdepth for each mixdepth.
+        """
+        mix_utxos = self.get_utxos_by_mixdepth(
+            include_disabled=include_disabled)
+        all_utxos = {}
+        for d in mix_utxos.values():
+            all_utxos.update(d)
+        return all_utxos
+
     @classmethod
     def _get_merge_algorithm(cls, algorithm_name=None):
         if not algorithm_name:
@@ -1089,6 +1102,31 @@ class PSBTWalletMixin(object):
                 privkeys.append(self._get_priv_from_path(v2[0]))
         jmckeys = list(btc.JMCKey(x[0][:-1]) for x in privkeys)
         new_keystore = DummyKeyStore.from_iterable(jmckeys)
+
+        # for p2sh inputs that we want to sign, the redeem_script
+        # field must be populated by us, as the counterparty did not
+        # know it. If this was set in an earlier create-psbt role,
+        # then overwriting it is harmless (preimage resistance).
+        if isinstance(self, SegwitLegacyWallet):
+            for i, txinput in enumerate(new_psbt.inputs):
+                tu = txinput.utxo
+                if isinstance(tu, btc.CTxOut):
+                    # witness
+                    if tu.scriptPubKey.is_witness_scriptpubkey():
+                        # native segwit; no insertion needed.
+                        continue
+                    elif tu.scriptPubKey.is_p2sh():
+                        try:
+                            path = self.script_to_path(tu.scriptPubKey)
+                        except AssertionError:
+                            # this happens when an input is provided but it's not in
+                            # this wallet; in this case, we cannot set the redeem script.
+                            continue
+                        privkey, _ = self._get_priv_from_path(path)
+                        txinput.redeem_script = btc.pubkey_to_p2wpkh_script(
+                            btc.privkey_to_pubkey(privkey))
+                    # no else branch; any other form of scriptPubKey will just be
+                    # ignored.
         try:
             signresult = new_psbt.sign(new_keystore)
         except Exception as e:
@@ -1098,6 +1136,244 @@ class PSBTWalletMixin(object):
         else:
             return (signresult, new_psbt), None
 
+class SNICKERWalletMixin(object):
+
+    SUPPORTED_SNICKER_VERSIONS = bytes([0, 1])
+
+    def __init__(self, storage, **kwargs):
+        super(SNICKERWalletMixin, self).__init__(storage, **kwargs)
+
+    def create_snicker_proposal(self, our_input, their_input, our_input_utxo,
+                                their_input_utxo, net_transfer, network_fee,
+                                our_priv, their_pub, our_spk, change_spk,
+                                encrypted=True, version_byte=1):
+        """ Creates a SNICKER proposal from the given transaction data.
+        This only applies to existing specification, i.e. SNICKER v 00 or 01.
+        This is only to be used for Joinmarket and only segwit wallets.
+        `our_input`, `their_input` - utxo format used in JM wallets,
+        keyed by (tixd, n), as dicts (currently of single entry).
+        `our_input_utxo`, `their..` - type CTxOut (contains value, scriptPubKey)
+        net_transfer - amount, after bitcoin transaction fee, transferred from
+        Proposer (our) to Receiver (their). May be negative.
+        network_fee - total bitcoin network transaction fee to be paid (so estimates
+        must occur before this function).
+        `our_priv`, `their_pub` - these are the keys to be used in ECDH to derive
+        the tweak as per the BIP. Note `their_pub` may or may not be associated with
+        the input of the receiver, so is specified here separately. Note also that
+        according to the BIP the privkey we use *must* be the one corresponding to
+        the input we provided, else (properly coded) Receivers will reject our
+        proposal.
+        `our_spk` - a scriptPubKey for the Proposer coinjoin output
+        `change_spk` - a change scriptPubkey for the proposer as per BIP
+        `encrypted` - whether or not to return the ECIES encrypted version of the
+        proposal.
+        `version_byte` - which of currently specified Snicker versions is being
+        used, (0 for reused address, 1 for inferred key).
+        returns:
+        if encrypted is True:
+          base 64 encoded encrypted transaction proposal as a string
+        else:
+          binary serialized plaintext SNICKER message.
+        """
+        assert isinstance(self, PSBTWalletMixin)
+        # before constructing the bitcoin transaction we must calculate the output
+        # amounts
+        # TODO investigate arithmetic for negative transfer
+        if our_input_utxo.nValue - their_input_utxo.nValue - network_fee <= 0:
+            raise Exception(
+                "Cannot create SNICKER proposal, Proposer input too small")
+        total_input_amount = our_input_utxo.nValue + their_input_utxo.nValue
+        total_output_amount = total_input_amount - network_fee
+        receiver_output_amount = their_input_utxo.nValue + net_transfer
+        proposer_output_amount = total_output_amount - receiver_output_amount
+
+        # we must also use ecdh to calculate the output scriptpubkey for the
+        # receiver
+        # First, check that `our_priv` corresponds to scriptPubKey in
+        # `our_input_utxo` to prevent callers from making useless proposals.
+        expected_pub = btc.privkey_to_pubkey(our_priv)
+        expected_spk = self.pubkey_to_script(expected_pub)
+        assert our_input_utxo.scriptPubKey == expected_spk
+        # now we create the ecdh based tweak:
+        tweak_bytes = btc.ecdh(our_priv[:-1], their_pub)
+        tweaked_pub = btc.snicker_pubkey_tweak(their_pub, tweak_bytes)
+        # TODO: remove restriction to one scriptpubkey type
+        tweaked_spk = btc.pubkey_to_p2sh_p2wpkh_script(tweaked_pub)
+        tweaked_addr, our_addr, change_addr = [str(
+            btc.CCoinAddress.from_scriptPubKey(x)) for x in (
+                tweaked_spk, expected_spk, change_spk)]
+        # now we must construct the three outputs with correct output amounts.
+        outputs = [{"address": tweaked_addr, "value": receiver_output_amount}]
+        outputs.append({"address": our_addr, "value": receiver_output_amount})
+        outputs.append({"address": change_addr,
+                "value": total_output_amount - 2 * receiver_output_amount})
+        assert all([x["value"] > 0 for x in outputs])
+
+        # version and locktime as currently specified in the BIP
+        # for 0/1 version SNICKER.
+        tx = btc.make_shuffled_tx([our_input, their_input], outputs,
+                              version=2, locktime=0)
+        # we need to know which randomized input is ours:
+        our_index = -1
+        for i, inp in enumerate(tx.vin):
+            if our_input == (inp.prevout.hash[::-1], inp.prevout.n):
+                our_index = i
+        assert our_index in [0, 1], "code error: our input not in tx"
+        spent_outs = [our_input_utxo, their_input_utxo]
+        if our_index == 1:
+            spent_outs = spent_outs[::-1]
+        # create the psbt and then sign our input.
+        snicker_psbt = self.create_psbt_from_tx(tx, spent_outs=spent_outs)
+
+        # having created the PSBT, sign our input
+        # TODO this requires bitcointx updated minor version else fails
+        signed_psbt_and_signresult, err = self.sign_psbt(
+        snicker_psbt.serialize(), with_sign_result=True)
+        assert err is None
+        signresult, partially_signed_psbt = signed_psbt_and_signresult
+        assert signresult.num_inputs_signed == 1
+        assert signresult.num_inputs_final == 1
+        assert not signresult.is_final
+        snicker_serialized_message = btc.SNICKER_MAGIC_BYTES + bytes(
+            [version_byte]) + btc.SNICKER_FLAG_NONE + tweak_bytes + \
+            partially_signed_psbt.serialize()
+
+        if not encrypted:
+            return snicker_serialized_message
+
+        # encryption has been requested;
+        # we apply ECIES in the form given by the BIP.
+        return btc.ecies_encrypt(snicker_serialized_message, their_pub)
+
+    def parse_proposal_to_signed_tx(self, privkey, proposal,
+                                    acceptance_callback):
+        """ Given a candidate privkey (binary and compressed format),
+        and a candidate encrypted SNICKER proposal, attempt to decrypt
+        and validate it in all aspects. If validation fails the first
+        return value is None and the second is the reason as a string.
+
+        If all validation checks pass, the next step is checking
+        acceptance according to financial rules: the acceptance
+        callback must be a function that accepts four arguments:
+        (our_ins, their_ins, our_outs, their_outs), where *ins values
+        are lists of CTxIns and *outs are lists of CTxOuts,
+        and must return only True/False where True means that the
+        transaction should be signed.
+
+        If True is returned from the callback, the following are returned
+        from this function:
+        (raw transaction for broadcasting (serialized),
+        tweak value as bytes,  derived output spk belonging to receiver)
+
+        Note: flags is currently always None as version is only 0 or 1.
+        """
+        assert isinstance(self, PSBTWalletMixin)
+
+        our_pub = btc.privkey_to_pubkey(privkey)
+
+        if len(proposal) < 5:
+            return None, "Invalid proposal, too short."
+
+        if base64.b64decode(proposal)[:4] == btc.ECIES_MAGIC_BYTES:
+            # attempt decryption and reject if fails:
+            try:
+                snicker_message = btc.ecies_decrypt(privkey, proposal)
+            except Exception as e:
+                return None, "Failed to decrypt." + repr(e)
+        else:
+            snicker_message = proposal
+
+        # magic + version,flag + tweak + psbt:
+        # TODO replace '20' with the minimum feasible PSBT.
+        if len(snicker_message) < 7 + 2 + 32 + 20:
+            return None, "Invalid proposal, too short."
+
+        if snicker_message[:7] != btc.SNICKER_MAGIC_BYTES:
+            return None, "Invalid SNICKER magic bytes."
+
+        version_byte = bytes([snicker_message[7]])
+        flag_byte = bytes([snicker_message[8]])
+        if version_byte not in self.SUPPORTED_SNICKER_VERSIONS:
+            return None, "Unrecognized SNICKER version: " + version_byte
+        if flag_byte != btc.SNICKER_FLAG_NONE:
+            return None, "Invalid flag byte for version 0,1: " + flag_byte
+
+        tweak_bytes = snicker_message[9:41]
+        candidate_psbt_serialized = snicker_message[41:]
+        # attempt to validate the PSBT's format:
+        try:
+            cpsbt = btc.PartiallySignedTransaction.from_base64_or_binary(
+                candidate_psbt_serialized)
+        except:
+            return None, "Invalid PSBT format."
+
+        utx = cpsbt.unsigned_tx
+        # validate that it contains one signature, and two inputs.
+        # else the proposal is invalid. To achieve this, we call
+        # PartiallySignedTransaction.sign() with an empty KeyStore,
+        # which populates the 'is_signed' info fields for us. Note that
+        # we do not use the PSBTWalletMixin.sign_psbt() which automatically
+        # signs with our keys.
+        if not len(utx.vin) == 2:
+            return None, "PSBT proposal does not contain 2 inputs."
+        testsignresult = cpsbt.sign(btc.KeyStore(), finalize=False)
+        print("got sign result: ", testsignresult)
+        # Note: "num_inputs_signed" refers to how many *we* signed,
+        # which is obviously none here as we provided no keys.
+        if not (testsignresult.num_inputs_signed == 0 and \
+                testsignresult.num_inputs_final == 1 and \
+                not testsignresult.is_final):
+            return None, "PSBT proposal does not contain 1 signature."
+
+        # Validate that we own one SNICKER style output:
+        spk = btc.verify_snicker_output(utx, our_pub, tweak_bytes)
+
+        if spk[0] == -1:
+            return None, "Tweaked destination not found exactly once."
+        our_output_index = spk[0]
+        our_output_amount = utx.vout[our_output_index].nValue
+
+        # At least one other output must have an amount equal to that at
+        # `our_output_index`, according to the spec.
+        found = 0
+        for i, o in enumerate(utx.vout):
+            if i == our_output_index:
+                continue
+            if o.nValue == our_output_amount:
+                found += 1
+        if found != 1:
+            return None, "Invalid SNICKER, there are not two equal outputs."
+
+        # To allow the acceptance callback to assess validity, we must identify
+        # which input is ours and which is(are) not.
+        # TODO This check may (will) change if we allow non-p2sh-pwpkh inputs:
+        unsigned_index = -1
+        for i, psbtinputsigninfo in enumerate(testsignresult.inputs_info):
+            if psbtinputsigninfo is None:
+                unsigned_index = i
+                break
+        assert unsigned_index != -1
+        # All validation checks passed. We now check whether the
+        #transaction is acceptable according to the caller:
+        if not acceptance_callback([utx.vin[unsigned_index]],
+                [x for i, x in enumerate(utx.vin) if i != unsigned_index],
+                [utx.vout[our_output_index]],
+                [x for i, x in enumerate(utx.vout) if i != our_output_index]):
+            return None, "Caller rejected transaction for signing."
+
+        # Acceptance passed, prepare the deserialized tx for signing by us:
+        signresult_and_signedpsbt, err = self.sign_psbt(cpsbt.serialize(),
+                                                        with_sign_result=True)
+        if err:
+            return None, "Unable to sign proposed PSBT, reason: " + err
+        signresult, signed_psbt = signresult_and_signedpsbt
+        assert signresult.num_inputs_signed == 1
+        assert signresult.num_inputs_final == 2
+        assert signresult.is_final
+        # we now know the transaction is valid and fully signed; return to caller,
+        # along with supporting data for this tx:
+        return (signed_psbt.extract_transaction(), tweak_bytes, spk[1])
+
 class ImportWalletMixin(object):
     """
     Mixin for BaseWallet to support importing keys.
@@ -1916,10 +2192,10 @@ class BIP84Wallet(BIP32PurposedWallet):
     _PURPOSE = 2**31 + 84
     _ENGINE = ENGINES[TYPE_P2WPKH]
 
-class SegwitLegacyWallet(ImportWalletMixin, BIP39WalletMixin, PSBTWalletMixin, BIP49Wallet):
+class SegwitLegacyWallet(ImportWalletMixin, BIP39WalletMixin, PSBTWalletMixin, SNICKERWalletMixin, BIP49Wallet):
     TYPE = TYPE_P2SH_P2WPKH
 
-class SegwitWallet(ImportWalletMixin, BIP39WalletMixin, PSBTWalletMixin, BIP84Wallet):
+class SegwitWallet(ImportWalletMixin, BIP39WalletMixin, PSBTWalletMixin, SNICKERWalletMixin, BIP84Wallet):
     TYPE = TYPE_P2WPKH
 
 class SegwitLegacyWalletFidelityBonds(FidelityBondMixin, SegwitLegacyWallet):

jmclient/test/commontest.py

@@ -27,6 +27,12 @@ PINL = '\r\n' if OS == 'Windows' else '\n'
 
 default_max_cj_fee = (1, float('inf'))
 
+# callbacks for making transfers in-script with direct_send:
+def dummy_accept_callback(tx, destaddr, actual_amount, fee_est):
+    return True
+def dummy_info_callback(msg):
+    pass
+
 class DummyBlockchainInterface(BlockchainInterface):
     def __init__(self):
         self.fake_query_results = None

jmclient/test/test_psbt_wallet.py

@@ -10,7 +10,7 @@ import time
 import binascii
 import struct
 import copy
-from commontest import make_wallets
+from commontest import make_wallets, dummy_accept_callback, dummy_info_callback
 
 import jmbitcoin as bitcoin
 import pytest
@@ -20,11 +20,6 @@ from jmclient import (load_test_config, jm_single, direct_send,
 
 log = get_log()
 
-def dummy_accept_callback(tx, destaddr, actual_amount, fee_est):
-    return True
-def dummy_info_callback(msg):
-    pass
-
 def test_create_and_sign_psbt_with_legacy(setup_psbt_wallet):
     """ The purpose of this test is to check that we can create and
     then partially sign a PSBT where we own one input and the other input

jmclient/test/test_snicker.py

@@ -0,0 +1,118 @@
+#! /usr/bin/env python
+from __future__ import (absolute_import, division,
+                        print_function, unicode_literals)
+from builtins import * # noqa: F401
+'''Test of unusual transaction types creation and push to
+network to check validity.'''
+
+import binascii
+from commontest import make_wallets, dummy_accept_callback, dummy_info_callback
+
+import jmbitcoin as btc
+import pytest
+from jmbase import get_log, bintohex, hextobin
+from jmclient import (load_test_config, jm_single,
+     estimate_tx_fee, SNICKERReceiver, direct_send)
+
+log = get_log()
+
+@pytest.mark.parametrize(
+    "nw, wallet_structures, mean_amt, sdev_amt, amt, net_transfer", [
+        (2, [[1, 0, 0, 0, 0]] * 2, 4, 0, 20000000, 1000),
+    ])
+def test_snicker_e2e(setup_snicker, nw, wallet_structures,
+                     mean_amt, sdev_amt, amt, net_transfer):
+    """ Test strategy:
+    1. create two wallets.
+    2. with wallet 1 (Receiver), create a single transaction
+    tx1, from mixdepth 0 to 1.
+    3. with wallet 2 (Proposer), take pubkey of all inputs from tx1, and use
+    them to create snicker proposals to the non-change out of tx1,
+    in base64 and place in proposals.txt.
+    4. Receiver polls for proposals in the file manually (instead of twisted
+    LoopingCall) and processes them.
+    5. Check for valid final transaction with broadcast.
+    """
+    wallets = make_wallets(nw, wallet_structures, mean_amt, sdev_amt)
+    for w in wallets.values():
+        w['wallet'].sync_wallet(fast=True)
+    print(wallets)
+    wallet_r = wallets[0]['wallet']
+    wallet_p = wallets[1]['wallet']
+    # next, create a tx from the receiver wallet
+    our_destn_script = wallet_r.get_new_script(1, 1)
+    tx = direct_send(wallet_r, btc.coins_to_satoshi(0.3), 0,
+                     wallet_r.script_to_addr(our_destn_script),
+                     accept_callback=dummy_accept_callback,
+                     info_callback=dummy_info_callback,
+                     return_transaction=True)    
+    
+    assert tx, "Failed to spend from receiver wallet"
+    print("Parent transaction OK. It was: ")
+    print(tx)
+    wallet_r.process_new_tx(tx)
+    # we must identify the receiver's output we're going to use;
+    # it can be destination or change, that's up to the proposer
+    # to guess successfully; here we'll just choose index 0.
+    txid1 = tx.GetTxid()[::-1]
+    txid1_index = 0
+
+    receiver_start_bal = sum([x['value'] for x in wallet_r.get_all_utxos(
+        ).values()])
+
+    # Now create a proposal for every input index in tx1
+    # (version 1 proposals mean we source keys from the/an
+    # ancestor transaction)
+    propose_keys = []
+    for i in range(len(tx.vin)):
+        # todo check access to pubkey
+        sig, pub = [a for a in iter(tx.wit.vtxinwit[i].scriptWitness)]
+        propose_keys.append(pub)
+    # the proposer wallet needs to choose a single
+    # utxo that is bigger than the output amount of tx1
+    prop_m_utxos = wallet_p.get_utxos_by_mixdepth()[0]
+    prop_utxo = prop_m_utxos[list(prop_m_utxos)[0]]
+    # get the private key for that utxo
+    priv = wallet_p.get_key_from_addr(
+        wallet_p.script_to_addr(prop_utxo['script']))
+    prop_input_amt = prop_utxo['value']
+    # construct the arguments for the snicker proposal:
+    our_input = list(prop_m_utxos)[0] # should be (txid, index)
+    their_input = (txid1, txid1_index)
+    our_input_utxo = btc.CMutableTxOut(prop_utxo['value'],
+                                       prop_utxo['script'])
+    fee_est = estimate_tx_fee(len(tx.vin), 2)
+    change_spk = wallet_p.get_new_script(0, 1)
+
+    encrypted_proposals = []
+
+    for p in propose_keys:
+        # TODO: this can be a loop over all outputs,
+        # not just one guessed output, if desired.
+        encrypted_proposals.append(
+            wallet_p.create_snicker_proposal(
+            our_input, their_input,
+            our_input_utxo,
+            tx.vout[txid1_index],
+            net_transfer,
+            fee_est,
+            priv,
+            p,
+            prop_utxo['script'],
+            change_spk,
+            version_byte=1) + b"," + bintohex(p).encode('utf-8'))
+    with open("test_proposals.txt", "wb") as f:
+        f.write(b"\n".join(encrypted_proposals))
+    sR = SNICKERReceiver(wallet_r)
+    sR.proposals_source = "test_proposals.txt" # avoid clashing with mainnet
+    sR.poll_for_proposals()
+    assert len(sR.successful_txs) == 1
+    wallet_r.process_new_tx(sR.successful_txs[0])
+    end_utxos = wallet_r.get_all_utxos()
+    print("At end the receiver has these utxos: ", end_utxos)
+    receiver_end_bal = sum([x['value'] for x in end_utxos.values()])
+    assert receiver_end_bal == receiver_start_bal + net_transfer
+
+@pytest.fixture(scope="module")
+def setup_snicker():
+    load_test_config()