Browse Source

Merge #921: added opt-in replace-by-fee support for direct sends

635f3f1 Adds test case for fee bumping a tx using PSBT (Adam Gibson)
b19888e add opt-in rbf support for direct sends (Tim Akinbo)
master
Adam Gibson 4 years ago
parent
commit
50f5228498
No known key found for this signature in database
GPG Key ID: 141001A1AF77F20B
  1. 78
      jmbitcoin/jmbitcoin/secp256k1_transaction.py
  2. 5
      jmclient/jmclient/cli_options.py
  3. 9
      jmclient/jmclient/cryptoengine.py
  4. 15
      jmclient/jmclient/taker.py
  5. 7
      jmclient/jmclient/taker_utils.py
  6. 45
      jmclient/jmclient/wallet.py
  7. 8
      jmclient/test/test_taker.py
  8. 94
      jmclient/test/test_tx_creation.py
  9. 2
      scripts/sendpayment.py
  10. 4
      scripts/yg-privacyenhanced.py

78
jmbitcoin/jmbitcoin/secp256k1_transaction.py

@ -18,6 +18,31 @@ from bitcointx.core.scripteval import (VerifyScript, SCRIPT_VERIFY_WITNESS,
SCRIPT_VERIFY_STRICTENC,
SIGVERSION_WITNESS_V0)
# for each transaction type, different output script pubkeys may result in
# a difference in the number of bytes accounted for while estimating the
# transaction size, this variable stores the difference and is factored in
# when calculating the correct transaction size. For example, for a p2pkh
# transaction, if one of the outputs is a p2wsh pubkey, then the transaction
# would need 9 extra bytes to account for the difference in script pubkey
# sizes
OUTPUT_EXTRA_BYTES = {
'p2pkh': {
'p2wpkh': -3,
'p2sh-p2wpkh': -2,
'p2wsh': 9
},
'p2wpkh': {
'p2pkh': 3,
'p2sh-p2wpkh': 1,
'p2wsh': 12
},
'p2sh-p2wpkh': {
'p2pkh': 2,
'p2wpkh': -1,
'p2wsh': 11
}
}
def human_readable_transaction(tx, jsonified=True):
""" Given a CTransaction object, output a human
readable json-formatted string (suitable for terminal
@ -81,52 +106,67 @@ def human_readable_output(txoutput):
pass # non standard script
return outdict
def estimate_tx_size(ins, outs, txtype='p2pkh'):
def estimate_tx_size(ins, outs, txtype='p2pkh', outtype=None):
'''Estimate transaction size.
The txtype field as detailed below is used to distinguish
the type, but there is at least one source of meaningful roughness:
we assume the output types are the same as the input (to be fair,
outputs only contribute a little to the overall total). This combined
with a few bytes variation in signature sizes means we will expect,
say, 10% inaccuracy here.
we assume that the scriptPubKey type of all the outputs are the same as
the input, unless `outtype` is specified, in which case *one* of
the outputs is assumed to be that other type, with all of the other
outputs being of the same type as before.
This, combined with a few bytes variation in signature sizes means
we will sometimes see small inaccuracies in this estimate.
Assuming p2pkh:
out: 8+1+3+2+20=34, in: 1+32+4+1+1+~73+1+1+33=147,
ver:4,seq:4, +2 (len in,out)
total ~= 34*len_out + 147*len_in + 10 (sig sizes vary slightly)
out: 8+1+3+20+2=34, in: 32+4+1+1+~72+1+33+4=148,
ver: 4, locktime:4, +2 (len in,out)
total = 34*len_out + 148*len_in + 10 (sig sizes vary slightly)
Assuming p2sh M of N multisig:
"ins" must contain M, N so ins= (numins, M, N) (crude assuming all same)
74*M + 34*N + 45 per input, so total ins ~ len_ins * (45+74M+34N)
so total ~ 34*len_out + (45+74M+34N)*len_in + 10
73*M + 34*N + 45 per input, so total ins ~ len_ins * (45+73M+34N)
so total ~ 32*len_out + (45+73M+34N)*len_in + 10
Assuming p2sh-p2wpkh:
witness are roughly 3+~73+33 for each input
witness are roughly 1+1+~72+1+33 for each input
(txid, vin, 4+20 for witness program encoded as scriptsig, 4 for sequence)
non-witness input fields are roughly 32+4+4+20+4=64, so total becomes
n_in * 64 + 4(ver) + 4(locktime) + n_out*34
n_in * 64 + 4(ver) + 2(marker, flag) + 2(n_in, n_out) + 4(locktime) + n_out*32
Assuming p2wpkh native:
witness as previous case
non-witness loses the 24 witnessprogram, replaced with 1 zero,
in the scriptSig, so becomes:
n_in * 41 + 4(ver) + 4(locktime) +2 (len in, out) + n_out*34
4 + 1 + 1 + (n_in) + (vin) + (n_out) + (vout) + (witness) + (locktime)
non-witness: 4(ver) +2 (marker, flag) + n_in*41 + 4(locktime) +2 (len in, out) + n_out*31
witness: 1 + 1 + 72 + 1 + 33
'''
if txtype == 'p2pkh':
return 10 + ins * 147 + 34 * outs
return 4 + 4 + 2 + ins*148 + 34*outs + (
OUTPUT_EXTRA_BYTES[txtype][outtype]
if outtype and outtype in OUTPUT_EXTRA_BYTES[txtype] else 0)
elif txtype == 'p2sh-p2wpkh':
#return the estimate for the witness and non-witness
#portions of the transaction, assuming that all the inputs
#are of segwit type p2sh-p2wpkh
# Note as of Jan19: this misses 2 bytes (trivial) for len in, out
# and also overestimates output size by 2 bytes.
witness_estimate = ins*109
non_witness_estimate = 4 + 4 + outs*34 + ins*64
witness_estimate = ins*108
non_witness_estimate = 4 + 4 + 4 + outs*32 + ins*64 + (
OUTPUT_EXTRA_BYTES[txtype][outtype]
if outtype and outtype in OUTPUT_EXTRA_BYTES[txtype] else 0)
return (witness_estimate, non_witness_estimate)
elif txtype == 'p2wpkh':
witness_estimate = ins*109
non_witness_estimate = 4 + 4 + 2 + outs*31 + ins*41
witness_estimate = ins*108
non_witness_estimate = 4 + 4 + 4 + outs*31 + ins*41 + (
OUTPUT_EXTRA_BYTES[txtype][outtype]
if outtype and outtype in OUTPUT_EXTRA_BYTES[txtype] else 0)
return (witness_estimate, non_witness_estimate)
elif txtype == 'p2shMofN':
ins, M, N = ins
return 10 + (45 + 74*M + 34*N) * ins + 34 * outs
return 4 + 4 + 2 + (45 + 73*M + 34*N)*ins + outs*32 + (
OUTPUT_EXTRA_BYTES['p2sh-p2wpkh'][outtype]
if outtype and outtype in OUTPUT_EXTRA_BYTES['p2sh-p2wpkh'] else 0)
else:
raise NotImplementedError("Transaction size estimation not" +
"yet implemented for type: " + txtype)

5
jmclient/jmclient/cli_options.py

@ -520,6 +520,11 @@ def get_sendpayment_parser():
'broadcasting the transaction. '
'Currently only works with direct '
'send (-N 0).')
parser.add_option('--rbf',
action='store_true',
dest='rbf',
default=False,
help='enable opt-in rbf')
parser.add_option('-u',
'--custom-change',
type="str",

9
jmclient/jmclient/cryptoengine.py

@ -10,10 +10,13 @@ from .configure import get_network, jm_single
#NOTE: before fidelity bonds and watchonly wallet, each of these types corresponded
# to one wallet type and one engine, not anymore
#with fidelity bond wallets and watchonly fidelity bond wallet, the wallet class
# can have two engines, one for single-sig addresses and the other for timelocked addresses
# can have two engines, one for single-sig addresses and the other for timelocked addresses.
# It is also necessary to preserve the order of the wallet types when making modifications
# as they are mapped to a different Engine when using wallets. Failure to do this would
# make existing wallets unsable.
TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH, TYPE_P2SH_M_N, TYPE_TIMELOCK_P2WSH, \
TYPE_SEGWIT_WALLET_FIDELITY_BONDS, TYPE_WATCHONLY_FIDELITY_BONDS, \
TYPE_WATCHONLY_TIMELOCK_P2WSH, TYPE_WATCHONLY_P2WPKH = range(9)
TYPE_WATCHONLY_TIMELOCK_P2WSH, TYPE_WATCHONLY_P2WPKH, TYPE_P2WSH = range(10)
NET_MAINNET, NET_TESTNET, NET_SIGNET = range(3)
NET_MAP = {'mainnet': NET_MAINNET, 'testnet': NET_TESTNET,
'signet': NET_SIGNET}
@ -42,6 +45,8 @@ def detect_script_type(script_str):
return TYPE_P2SH_P2WPKH
elif script.is_witness_v0_keyhash():
return TYPE_P2WPKH
elif script.is_witness_v0_scripthash():
return TYPE_P2WSH
raise EngineError("Unknown script type for script '{}'"
.format(bintohex(script_str)))

15
jmclient/jmclient/taker.py

@ -328,7 +328,9 @@ class Taker(object):
#find sufficient utxos extremely rare. Indeed, a doubling of 'normal'
#txfee indicates undesirable behaviour on maker side anyway.
self.total_txfee = estimate_tx_fee(3, 2,
txtype=self.wallet_service.get_txtype()) * self.n_counterparties
txtype=self.wallet_service.get_txtype()) * (self.n_counterparties - 1) + \
estimate_tx_fee(3, 2, txtype=self.wallet_service.get_txtype(),
outtype=self.wallet_service.get_outtype(self.coinjoin_address()))
total_amount = self.cjamount + self.total_cj_fee + self.total_txfee
jlog.info('total estimated amount spent = ' + btc.amount_to_str(total_amount))
try:
@ -351,8 +353,9 @@ class Taker(object):
jlog.debug("Estimated ins: "+str(est_ins))
est_outs = 2*self.n_counterparties + 1
jlog.debug("Estimated outs: "+str(est_outs))
self.total_txfee = estimate_tx_fee(est_ins, est_outs,
txtype=self.wallet_service.get_txtype())
self.total_txfee = estimate_tx_fee(
est_ins, est_outs, txtype=self.wallet_service.get_txtype(),
outtype=self.wallet_service.get_outtype(self.coinjoin_address()))
jlog.debug("We have a fee estimate: "+str(self.total_txfee))
total_value = sum([va['value'] for va in self.input_utxos.values()])
if self.wallet_service.get_txtype() == "p2pkh":
@ -439,7 +442,8 @@ class Taker(object):
#Estimate fee per choice of next/3/6 blocks targetting.
estimated_fee = estimate_tx_fee(
len(sum(self.utxos.values(), [])), len(self.outputs) + 2,
txtype=self.wallet_service.get_txtype())
txtype=self.wallet_service.get_txtype(),
outtype=self.wallet_service.get_outtype(self.coinjoin_address()))
jlog.info("Based on initial guess: " +
btc.amount_to_str(self.total_txfee) +
", we estimated a miner fee of: " +
@ -483,7 +487,8 @@ class Taker(object):
num_ins = len([u for u in sum(self.utxos.values(), [])])
num_outs = len(self.outputs) + 1
new_total_fee = estimate_tx_fee(num_ins, num_outs,
txtype=self.wallet_service.get_txtype())
txtype=self.wallet_service.get_txtype(),
outtype=self.wallet_service.get_outtype(self.coinjoin_address()))
feeratio = new_total_fee/self.total_txfee
jlog.debug("Ratio of actual to estimated sweep fee: {}".format(
feeratio))

7
jmclient/jmclient/taker_utils.py

@ -80,6 +80,7 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False,
return
txtype = wallet_service.get_txtype()
outtype = wallet_service.get_outtype(destination)
if amount == 0:
#doing a sweep
utxos = wallet_service.get_utxos_by_mixdepth()[mixdepth]
@ -111,15 +112,15 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False,
+ "\n\nWARNING: This transaction if broadcasted will PERMANENTLY DESTROY your bitcoins\n"
else:
#regular sweep (non-burn)
fee_est = estimate_tx_fee(len(utxos), 1, txtype=txtype)
fee_est = estimate_tx_fee(len(utxos), 1, txtype=txtype, outtype=outtype)
outs = [{"address": destination, "value": total_inputs_val - fee_est}]
else:
#not doing a sweep; we will have change
#8 inputs to be conservative
initial_fee_est = estimate_tx_fee(8,2, txtype=txtype)
initial_fee_est = estimate_tx_fee(8,2, txtype=txtype, outtype=outtype)
utxos = wallet_service.select_utxos(mixdepth, amount + initial_fee_est)
if len(utxos) < 8:
fee_est = estimate_tx_fee(len(utxos), 2, txtype=txtype)
fee_est = estimate_tx_fee(len(utxos), 2, txtype=txtype, outtype=outtype)
else:
fee_est = initial_fee_est
total_inputs_val = sum([va['value'] for u, va in utxos.items()])

45
jmclient/jmclient/wallet.py

@ -24,10 +24,10 @@ from .configure import jm_single
from .blockchaininterface import INF_HEIGHT
from .support import select_gradual, select_greedy, select_greediest, \
select, NotEnoughFundsException
from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH,\
from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WSH,\
TYPE_P2WPKH, TYPE_TIMELOCK_P2WSH, TYPE_SEGWIT_WALLET_FIDELITY_BONDS,\
TYPE_WATCHONLY_FIDELITY_BONDS, TYPE_WATCHONLY_TIMELOCK_P2WSH, TYPE_WATCHONLY_P2WPKH,\
ENGINES
ENGINES, detect_script_type
from .support import get_random_bytes
from . import mn_encode, mn_decode
import jmbitcoin as btc
@ -47,7 +47,7 @@ class Mnemonic(MnemonicParent):
def detect_language(cls, code):
return "english"
def estimate_tx_fee(ins, outs, txtype='p2pkh', extra_bytes=0):
def estimate_tx_fee(ins, outs, txtype='p2pkh', outtype=None, extra_bytes=0):
'''Returns an estimate of the number of satoshis required
for a transaction with the given number of inputs and outputs,
based on information from the blockchain interface.
@ -68,11 +68,11 @@ def estimate_tx_fee(ins, outs, txtype='p2pkh', extra_bytes=0):
" greater than absurd value " +
btc.fee_per_kb_to_str(absurd_fee) + ", quitting.")
if txtype in ['p2pkh', 'p2shMofN']:
tx_estimated_bytes = btc.estimate_tx_size(ins, outs, txtype) + extra_bytes
tx_estimated_bytes = btc.estimate_tx_size(ins, outs, txtype, outtype) + extra_bytes
return int((tx_estimated_bytes * fee_per_kb)/Decimal(1000.0))
elif txtype in ['p2wpkh', 'p2sh-p2wpkh']:
witness_estimate, non_witness_estimate = btc.estimate_tx_size(
ins, outs, txtype)
ins, outs, txtype, outtype)
non_witness_estimate += extra_bytes
return int(int((
non_witness_estimate + 0.25*witness_estimate)*fee_per_kb)/Decimal(1000.0))
@ -412,6 +412,19 @@ class BaseWallet(object):
return 'p2wpkh'
assert False
def get_outtype(self, addr):
script_type = detect_script_type(
btc.CCoinAddress(addr).to_scriptPubKey())
if script_type == TYPE_P2PKH:
return 'p2pkh'
elif script_type == TYPE_P2WPKH:
return 'p2wpkh'
elif script_type == TYPE_P2SH_P2WPKH:
return 'p2sh-p2wpkh'
elif script_type == TYPE_P2WSH:
return 'p2wsh'
assert False
def sign_tx(self, tx, scripts, **kwargs):
"""
Add signatures to transaction for inputs referenced by scripts.
@ -635,7 +648,7 @@ class BaseWallet(object):
wallet, and returns [(index, CTxIn),..] for each.
"""
retval = []
for i, txin in len(tx.vin):
for i, txin in enumerate(tx.vin):
pub, msg = btc.extract_pubkey_from_witness(tx, i)
if not pub:
# this can certainly occur since other inputs
@ -1292,6 +1305,26 @@ class PSBTWalletMixin(object):
#key is ((privkey, locktime), engine) for timelocked addrs
key = (key[0][0], key[1])
privkeys.append(key)
# in the rare situation that we want to sign a psbt using private keys
# to utxos that we've stopped tracking, let's also find inputs that
# belong to us and add those private keys as well
for vin in new_psbt.inputs:
try:
path = self.script_to_path(vin.utxo.scriptPubKey)
key = self._get_key_from_path(path)
if key not in privkeys:
privkeys.append(key)
except AssertionError:
# we can safely assume that an exception means we do not
# have the ability to sign for this input
continue
except AttributeError:
# shouldn't happen for properly constructed psbts
# however, psbts with no utxo information will raise
# an AttributeError exception. we simply ignore it.
continue
jmckeys = list(btc.JMCKey(x[0][:-1]) for x in privkeys)
new_keystore = btc.KeyStore.from_iterable(jmckeys)

8
jmclient/test/test_taker.py

@ -511,17 +511,17 @@ def test_custom_change(setup_taker):
# the address we intended with the right amount:
custom_change_found = False
for out in taker.latest_tx.vout:
# input utxo is 20M; amount is 2M; as per logs:
# input utxo is 200M; amount is 20M; as per logs:
# totalin=200000000
# my_txfee=12930
# my_txfee=13050
# makers_txfee=3000
# cjfee_total=12000 => changevalue=179975070
# cjfee_total=12000 => changevalue=179974950
# note that there is a small variation in the size of
# the transaction (a few bytes) for the different scriptPubKey
# type, but this is currently ignored by the Taker, who makes
# fee estimate purely based on the number of ins and outs;
# this will never be too far off anyway.
if out.scriptPubKey == script and out.nValue == 179975070:
if out.scriptPubKey == script and out.nValue == 179974950:
# must be only one
assert not custom_change_found
custom_change_found = True

94
jmclient/test/test_tx_creation.py

@ -11,7 +11,7 @@ from commontest import make_wallets, make_sign_and_push, ensure_bip65_activated
import jmbitcoin as bitcoin
import pytest
from jmbase import get_log
from jmclient import load_test_config, jm_single
from jmclient import load_test_config, jm_single, direct_send, estimate_tx_fee, compute_tx_locktime
log = get_log()
#just a random selection of pubkeys for receiving multisigs;
@ -86,6 +86,7 @@ def test_absurd_fees(setup_tx_creation):
ins_full = wallet_service.select_utxos(0, amount)
with pytest.raises(ValueError) as e_info:
txid = make_sign_and_push(ins_full, wallet_service, amount, estimate_fee=True)
jm_single().bc_interface.absurd_fees = False
def test_create_sighash_txs(setup_tx_creation):
#non-standard hash codes:
@ -137,6 +138,97 @@ def test_spend_p2wpkh(setup_tx_creation):
txid = jm_single().bc_interface.pushtx(tx.serialize())
assert txid
def test_spend_then_rbf(setup_tx_creation):
""" Test plan: first, create a normal spend with
rbf enabled in direct_send, then broadcast but
do not mine a block. Then create a re-spend of
the same utxos with a higher fee and check
that broadcast succeeds.
"""
# First phase: broadcast with RBF enabled.
#
# set a baseline feerate:
old_feerate = jm_single().config.get("POLICY", "tx_fees")
jm_single().config.set("POLICY", "tx_fees", "20000")
# set up a single wallet with some coins:
wallet_service = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet']
wallet_service.sync_wallet(fast=True)
# ensure selection of two utxos, doesn't really matter
# but a more general case than only one:
amount = 350000000
# destination doesn't matter; this is easiest:
destn = wallet_service.get_internal_addr(1)
# While `direct_send` usually encapsulates utxo selection
# for user, here we need to know what was chosen, hence
# we return the transaction object, not directly broadcast.
tx1 = direct_send(wallet_service, amount, 0,
destn, answeryes=True,
return_transaction=True,
optin_rbf=True)
assert tx1
# record the utxos for reuse:
assert isinstance(tx1, bitcoin.CTransaction)
utxos_objs = (x.prevout for x in tx1.vin)
utxos = [(x.hash[::-1], x.n) for x in utxos_objs]
# in order to sign on those utxos, we need their script and value.
scrs = {}
vals = {}
for u, details in wallet_service.get_utxos_by_mixdepth()[0].items():
if u in utxos:
scrs[u] = details["script"]
vals[u] = details["value"]
assert len(scrs.keys()) == 2
assert len(vals.keys()) == 2
# This will go to mempool but not get mined because
# we don't call `tick_forward_chain`.
push_succeed = jm_single().bc_interface.pushtx(tx1.serialize())
if push_succeed:
# mimics real operations with transaction monitor:
wallet_service.process_new_tx(tx1)
else:
assert False
# Second phase: bump fee.
#
# we set a larger fee rate.
jm_single().config.set("POLICY", "tx_fees", "30000")
# just a different destination to avoid confusion:
destn2 = wallet_service.get_internal_addr(2)
# We reuse *both* utxos so total fees are comparable
# (modulo tiny 1 byte differences in signatures).
# Ordinary wallet operations would remove the first-spent utxos,
# so for now we build a PSBT using the code from #921 to select
# the same utxos (it could be done other ways).
# Then we broadcast the PSBT and check it is allowed
# before constructing the outputs, we need a good fee estimate,
# using the bumped feerate:
fee = estimate_tx_fee(2, 2, wallet_service.get_txtype())
# reset the feerate:
total_input_val = sum(vals.values())
jm_single().config.set("POLICY", "tx_fees", old_feerate)
outs = [{"address": destn2, "value": 1000000},
{"address": wallet_service.get_internal_addr(0),
"value": total_input_val - 1000000 - fee}]
tx2 = bitcoin.mktx(utxos, outs, version=2,
locktime=compute_tx_locktime())
spent_outs = []
for u in utxos:
spent_outs.append(bitcoin.CTxOut(nValue=vals[u],
scriptPubKey=scrs[u]))
psbt_unsigned = wallet_service.create_psbt_from_tx(tx2,
spent_outs=spent_outs)
signresultandpsbt, err = wallet_service.sign_psbt(
psbt_unsigned.serialize(), with_sign_result=True)
assert not err
signresult, psbt_signed = signresultandpsbt
tx2_signed = psbt_signed.extract_transaction()
# the following assertion is sufficient, because
# tx broadcast would fail if the replacement were
# not allowed by Core:
assert jm_single().bc_interface.pushtx(tx2_signed.serialize())
def test_spend_freeze_script(setup_tx_creation):
ensure_bip65_activated()

2
scripts/sendpayment.py

@ -243,7 +243,7 @@ def main():
if options.makercount == 0 and not bip78url:
tx = direct_send(wallet_service, amount, mixdepth, destaddr,
options.answeryes, with_final_psbt=options.with_psbt,
custom_change_addr=custom_change)
optin_rbf=options.rbf, custom_change_addr=custom_change)
if options.with_psbt:
log.info("This PSBT is fully signed and can be sent externally for "
"broadcasting:")

4
scripts/yg-privacyenhanced.py

@ -22,7 +22,7 @@ class YieldGeneratorPrivacyEnhanced(YieldGeneratorBasic):
def __init__(self, wallet_service, offerconfig):
super().__init__(wallet_service, offerconfig)
def select_input_mixdepth(self, available, offer, amount):
"""Mixdepths are in cyclic order and we select the mixdepth to
maximize the largest interval of non-available mixdepths by choosing
@ -44,7 +44,7 @@ class YieldGeneratorPrivacyEnhanced(YieldGeneratorBasic):
# available mixdepths ends. Selecting this mixdepth will send the CoinJoin
# outputs closer to the others available mixdepths which are after in cyclical order
return available[max(range(len(available)), key = intervals.__getitem__)]
def create_my_orders(self):
mix_balance = self.get_available_mixdepths()
# We publish ONLY the maximum amount and use minsize for lower bound;

Loading…
Cancel
Save