From cc78daead9b5ae539fe45bbdb7bd388639032ab7 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Fri, 13 Nov 2020 17:33:44 +0000 Subject: [PATCH] Update commitments utility scripts This commit updates sendtomany.py and add-utxo.py to reflect the new default wallet type (p2wpkh) and the new bitcoin backend code; in doing this it slightly modifies the functions in commitment_utils.py. --- jmclient/jmclient/__init__.py | 2 +- jmclient/jmclient/commitment_utils.py | 34 +++++++++---- jmclient/test/test_commitment_utils.py | 12 +++-- scripts/add-utxo.py | 37 +++++++++----- scripts/sendtomany.py | 70 ++++++++++++-------------- 5 files changed, 88 insertions(+), 67 deletions(-) diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index e6e2a93..9b2fd43 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -18,7 +18,7 @@ from .wallet import (Mnemonic, estimate_tx_fee, WalletError, BaseWallet, ImportW UTXOManager, WALLET_IMPLEMENTATIONS, compute_tx_locktime) from .storage import (Argon2Hash, Storage, StorageError, RetryableStorageError, StoragePasswordError, VolatileStorage) -from .cryptoengine import (BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, EngineError, +from .cryptoengine import (BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, BTC_P2WPKH, EngineError, TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH) from .configure import (load_test_config, process_shutdown, load_program_config, jm_single, get_network, update_persist_config, diff --git a/jmclient/jmclient/commitment_utils.py b/jmclient/jmclient/commitment_utils.py index 49233ee..730cb18 100644 --- a/jmclient/jmclient/commitment_utils.py +++ b/jmclient/jmclient/commitment_utils.py @@ -1,8 +1,8 @@ import sys -from jmbase import jmprint -from jmclient import jm_single, BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH -from jmbase.support import EXIT_FAILURE, utxostr_to_utxo +from jmbase import jmprint, utxostr_to_utxo +from jmclient import jm_single, BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, BTC_P2WPKH +from jmbase.support import EXIT_FAILURE, utxostr_to_utxo, utxo_to_utxostr def quit(parser, errmsg): #pragma: no cover @@ -32,8 +32,8 @@ def get_utxo_info(upriv): raise return u, priv -def validate_utxo_data(utxo_datas, retrieve=False, segwit=False): - """For each txid: N, privkey, first +def validate_utxo_data(utxo_datas, retrieve=False, utxo_address_type="p2wpkh"): + """For each (utxo, privkey), first convert the privkey and convert to address, then use the blockchain instance to look up the utxo and check that its address field matches. @@ -44,21 +44,33 @@ def validate_utxo_data(utxo_datas, retrieve=False, segwit=False): """ results = [] for u, priv in utxo_datas: - jmprint('validating this utxo: ' + str(u), "info") + success, utxostr = utxo_to_utxostr(u) + if not success: + jmprint("Invalid utxo format: " + str(u), "error") + sys.exit(EXIT_FAILURE) + jmprint('validating this utxo: ' + utxostr, "info") # as noted in `ImportWalletMixin` code comments, there is not # yet a functional auto-detection of key type from WIF, so the - # second argument is ignored; we assume p2sh-p2wpkh if segwit, - # else we assume p2pkh. - engine = BTC_P2SH_P2WPKH if segwit else BTC_P2PKH + # second argument is ignored; we assume p2sh-p2wpkh if segwit=True, + # p2pkh if segwit=False, and p2wpkh if segwit="native" (slightly + # ugly, just done for backwards compat.). + if utxo_address_type == "p2wpkh": + engine = BTC_P2WPKH + elif utxo_address_type == "p2sh-p2wpkh": + engine = BTC_P2SH_P2WPKH + elif utxo_address_type == "p2pkh": + engine = BTC_P2PKH + else: + raise Exception("Invalid argument: " + str(utxo_address_type)) rawpriv, _ = BTCEngine.wif_to_privkey(priv) addr = engine.privkey_to_address(rawpriv) jmprint('claimed address: ' + addr, "info") res = jm_single().bc_interface.query_utxo_set([u]) if len(res) != 1 or None in res: - jmprint("utxo not found on blockchain: " + str(u), "error") + jmprint("utxo not found on blockchain: " + utxostr, "error") return False if res[0]['address'] != addr: - jmprint("privkey corresponds to the wrong address for utxo: " + str(u), "error") + jmprint("privkey corresponds to the wrong address for utxo: " + utxostr, "error") jmprint("blockchain returned address: {}".format(res[0]['address']), "error") jmprint("your privkey gave this address: " + addr, "error") return False diff --git a/jmclient/test/test_commitment_utils.py b/jmclient/test/test_commitment_utils.py index 070e7e8..030e0f0 100644 --- a/jmclient/test/test_commitment_utils.py +++ b/jmclient/test/test_commitment_utils.py @@ -2,6 +2,7 @@ from commontest import DummyBlockchainInterface import pytest +from jmbase import utxostr_to_utxo from jmclient import (load_test_config, jm_single) from jmclient.commitment_utils import get_utxo_info, validate_utxo_data from jmbitcoin import select_chain_params @@ -15,12 +16,13 @@ def test_get_utxo_info(): dbci = DummyBlockchainInterface() privkey = "L1RrrnXkcKut5DEMwtDthjwRcTTwED36thyL1DebVrKuwvohjMNi" #to verify use from_wif_privkey and privkey_to_address - iaddr = "1LDsjB43N2NAQ1Vbc2xyHca4iBBciN8iwC" + iaddr = "bc1q6tvmnmetj8vfz98vuetpvtuplqtj4uvvwjgxxc" fakeutxo = "aa"*32+":08" - + success, fakeutxo_bin = utxostr_to_utxo(fakeutxo) + assert success fake_query_results = [{'value': 200000000, 'address': iaddr, - 'utxo': fakeutxo, + 'utxo': fakeutxo_bin, 'confirms': 20}] dbci.insert_fake_query_results(fake_query_results) jm_single().bc_interface = dbci @@ -39,12 +41,12 @@ def test_get_utxo_info(): with pytest.raises(Exception) as e_info: u, priv = get_utxo_info(fakeutxo + "," + p2) - utxodatas = [(fakeutxo, privkey)] + utxodatas = [(fakeutxo_bin, privkey)] retval = validate_utxo_data(utxodatas, False) assert retval #try to retrieve retval = validate_utxo_data(utxodatas, True) - assert retval[0] == (fakeutxo, 200000000) + assert retval[0] == (fakeutxo_bin, 200000000) fake_query_results[0]['address'] = "fakeaddress" dbci.insert_fake_query_results(fake_query_results) #validate should fail for wrong address diff --git a/scripts/add-utxo.py b/scripts/add-utxo.py index 426f273..40a19d1 100755 --- a/scripts/add-utxo.py +++ b/scripts/add-utxo.py @@ -17,7 +17,8 @@ from jmclient import load_program_config, jm_single,\ open_wallet, WalletService, add_external_commitments, update_commitments,\ PoDLE, get_podle_commitments, get_utxo_info, validate_utxo_data, quit,\ get_wallet_path, add_base_options, BTCEngine, BTC_P2SH_P2WPKH -from jmbase.support import EXIT_SUCCESS, EXIT_FAILURE, EXIT_ARGERROR, jmprint +from jmbase.support import EXIT_SUCCESS, EXIT_FAILURE, EXIT_ARGERROR, \ + jmprint, utxostr_to_utxo def add_ext_commitments(utxo_datas): @@ -47,7 +48,7 @@ def add_ext_commitments(utxo_datas): if 'P' not in ecs[u]: ecs[u]['P']=P ecs[u]['reveal'][j] = {'P2':P2, 's':s, 'e':e} - add_external_commitments(ecs) + add_external_commitments(ecs) def main(): parser = OptionParser( @@ -181,11 +182,10 @@ def main(): # minor note: adding a utxo from an external wallet for commitments, we # default to not allowing disabled utxos to avoid a privacy leak, so the # user would have to explicitly enable. - for md, utxos in wallet_service.get_utxos_by_mixdepth(hexfmt=False).items(): - for (txid, index), utxo in utxos.items(): - txhex = binascii.hexlify(txid).decode('ascii') + ':' + str(index) - wif = wallet_service.get_wif_path(utxo['path']) - utxo_data.append((txhex, wif)) + for md, utxos in wallet_service.get_utxos_by_mixdepth().items(): + for utxo, utxodata in utxos.items(): + wif = wallet_service.get_wif_path(utxodata['path']) + utxo_data.append((utxo, wif)) elif options.in_file: with open(options.in_file, "rb") as f: @@ -196,7 +196,7 @@ def main(): u, priv = get_utxo_info(ul) if not u: quit(parser, "Failed to parse utxo info: " + str(ul)) - utxo_data.append((u, priv)) + utxo_data.append((utxostr_to_utxo(u), priv)) elif options.in_json: if not os.path.isfile(options.in_json): jmprint("File: " + options.in_json + " not found.", "error") @@ -208,7 +208,7 @@ def main(): jmprint("Failed to read json from " + options.in_json, "error") sys.exit(EXIT_FAILURE) for u, pva in iteritems(utxo_json): - utxo_data.append((u, pva['privkey'])) + utxo_data.append((utxostr_to_utxo(u), pva['privkey'])) elif len(args) == 1: u = args[0] priv = input( @@ -216,15 +216,26 @@ def main(): u, priv = get_utxo_info(','.join([u, priv])) if not u: quit(parser, "Failed to parse utxo info: " + u) - utxo_data.append((u, priv)) + utxo_data.append((utxostr_to_utxo(u), priv)) else: quit(parser, 'Invalid syntax') if options.validate or options.vonly: - sw = False if jm_single().config.get("POLICY", "segwit") == "false" else True - if not validate_utxo_data(utxo_data, segwit=sw): + # if the utxos are loaded from a wallet, we use the wallet's + # txtype to determine the value of `utxo_address_type`; if not, + # we use joinmarket.cfg. + if options.loadwallet: + utxo_address_type = wallet_service.get_txtype() + else: + if jm_single().config.get("POLICY", "segwit") == "false": + utxo_address_type = "p2pkh" + elif jm_single().config.get("POLICY", "native") == "false": + utxo_address_type = "p2sh-p2wpkh" + else: + utxo_address_type = "p2wpkh" + if not validate_utxo_data(utxo_data, utxo_address_type=utxo_address_type): quit(parser, "Utxos did not validate, quitting") if options.vonly: - sys.exit(EXIT_ARGERROR) + sys.exit(EXIT_SUCCESS) #We are adding utxos to the external list assert len(utxo_data) diff --git a/scripts/sendtomany.py b/scripts/sendtomany.py index 12037bb..6d36d25 100755 --- a/scripts/sendtomany.py +++ b/scripts/sendtomany.py @@ -11,26 +11,24 @@ import jmbitcoin as btc from jmbase import get_log, jmprint, bintohex, utxostr_to_utxo from jmclient import load_program_config, estimate_tx_fee, jm_single,\ validate_address, get_utxo_info, add_base_options,\ - validate_utxo_data, quit, BTCEngine + validate_utxo_data, quit, BTCEngine, compute_tx_locktime log = get_log() -def sign(utxo, priv, destaddrs, segwit=True): +def sign(utxo, priv, destaddrs, utxo_address_type): """Sign a tx sending the amount amt, from utxo utxo, equally to each of addresses in list destaddrs, - after fees; the purpose is to create a large - number of utxos. If segwit=True the (single) utxo is assumed to - be of type segwit p2sh/p2wpkh. + after fees; the purpose is to create multiple utxos. + utxo_address_type must be one of p2sh-p2wpkh/p2wpkh/p2pkh. """ - results = validate_utxo_data([(utxo, priv)], retrieve=True, segwit=segwit) + results = validate_utxo_data([(utxo, priv)], retrieve=True, + utxo_address_type=utxo_address_type) if not results: return False assert results[0][0] == utxo amt = results[0][1] ins = [utxo] - # TODO extend to other utxo types - txtype = 'p2sh-p2wpkh' if segwit else 'p2pkh' - estfee = estimate_tx_fee(1, len(destaddrs), txtype=txtype) + estfee = estimate_tx_fee(1, len(destaddrs), txtype=utxo_address_type) outs = [] share = int((amt - estfee) / len(destaddrs)) fee = amt - share*len(destaddrs) @@ -38,10 +36,14 @@ def sign(utxo, priv, destaddrs, segwit=True): log.info("Using fee: " + str(fee)) for i, addr in enumerate(destaddrs): outs.append({'address': addr, 'value': share}) - tx = btc.mktx(ins, outs) - amtforsign = amt if segwit else None + tx = btc.make_shuffled_tx(ins, outs, version=2, locktime=compute_tx_locktime()) + amtforsign = amt if utxo_address_type != "p2pkh" else None rawpriv, _ = BTCEngine.wif_to_privkey(priv) - success, msg = btc.sign(tx, 0, rawpriv, amount=amtforsign) + if utxo_address_type == "p2wpkh": + native = utxo_address_type + else: + native = False + success, msg = btc.sign(tx, 0, rawpriv, amount=amtforsign, native=native) assert success, msg return tx @@ -69,28 +71,14 @@ def main(): " joinmarket.cfg for the former." ) parser.add_option( - '-v', - '--validate-utxos', - action='store_true', - dest='validate', - help='validate the utxos and pubkeys provided against the blockchain', - default=False - ) - parser.add_option( - '-o', - '--validate-only', - action='store_true', - dest='vonly', - help='only validate the provided utxos (file or command line), not add', - default=False - ) - parser.add_option( - '-n', - '--non-segwit-input', - action='store_true', - dest='nonsegwit', - help='input is p2pkh ("1" address), not segwit; if not used, input is assumed to be segwit type.', - default=False + '-t', + '--utxo-address-type', + action='store', + dest='utxo_address_type', + help=('type of address of coin being spent - one of "p2pkh", "p2wpkh", "p2sh-p2wpkh". ' + 'No other scriptpubkey types (e.g. multisig) are supported. If not set, we default ' + 'to what is in joinmarket.cfg.'), + default="" ) add_base_options(parser) (options, args) = parser.parse_args() @@ -110,13 +98,21 @@ def main(): success, utxo = utxostr_to_utxo(u) if not success: quit(parser, "Failed to load utxo from string: " + utxo) - txsigned = sign(utxo, priv, destaddrs, segwit = not options.nonsegwit) + if options.utxo_address_type == "": + if jm_single().config.get("POLICY", "segwit") == "false": + utxo_address_type = "p2pkh" + elif jm_single().config.get("POLICY", "native") == "false": + utxo_address_type = "p2sh-p2wpkh" + else: + utxo_address_type = "p2wpkh" + else: + utxo_address_type = options.utxo_address_type + txsigned = sign(utxo, priv, destaddrs, utxo_address_type) if not txsigned: log.info("Transaction signing operation failed, see debug messages for details.") return log.info("Got signed transaction:\n" + bintohex(txsigned.serialize())) - log.debug("Deserialized:") - log.debug(pformat(str(txsigned))) + log.info(btc.human_readable_transaction(txsigned)) if input('Would you like to push to the network? (y/n):')[0] != 'y': log.info("You chose not to broadcast the transaction, quitting.") return