Update commitments utility scripts

This commit updates sendtomany.py and add-utxo.py to reflect
the new default wallet type (p2wpkh) and the new bitcoin
backend code; in doing this it slightly modifies the functions
in commitment_utils.py.

jmclient/jmclient/__init__.py

@@ -18,7 +18,7 @@ from .wallet import (Mnemonic, estimate_tx_fee, WalletError, BaseWallet, ImportW
                      UTXOManager, WALLET_IMPLEMENTATIONS, compute_tx_locktime)
 from .storage import (Argon2Hash, Storage, StorageError, RetryableStorageError,
                       StoragePasswordError, VolatileStorage)
-from .cryptoengine import (BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, EngineError,
+from .cryptoengine import (BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, BTC_P2WPKH, EngineError,
                            TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH)
 from .configure import (load_test_config, process_shutdown,
     load_program_config, jm_single, get_network, update_persist_config,

jmclient/jmclient/commitment_utils.py

@@ -1,8 +1,8 @@
 
 import sys
-from jmbase import jmprint
-from jmclient import jm_single, BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH
-from jmbase.support import EXIT_FAILURE, utxostr_to_utxo
+from jmbase import jmprint, utxostr_to_utxo
+from jmclient import jm_single, BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, BTC_P2WPKH
+from jmbase.support import EXIT_FAILURE, utxostr_to_utxo, utxo_to_utxostr
 
 
 def quit(parser, errmsg): #pragma: no cover
@@ -32,8 +32,8 @@ def get_utxo_info(upriv):
         raise
     return u, priv
     
-def validate_utxo_data(utxo_datas, retrieve=False, segwit=False):
-    """For each txid: N, privkey, first
+def validate_utxo_data(utxo_datas, retrieve=False, utxo_address_type="p2wpkh"):
+    """For each (utxo, privkey), first
     convert the privkey and convert to address,
     then use the blockchain instance to look up
     the utxo and check that its address field matches.
@@ -44,21 +44,33 @@ def validate_utxo_data(utxo_datas, retrieve=False, segwit=False):
     """
     results = []
     for u, priv in utxo_datas:
-        jmprint('validating this utxo: ' + str(u), "info")
+        success, utxostr = utxo_to_utxostr(u)
+        if not success:
+            jmprint("Invalid utxo format: " + str(u), "error")
+            sys.exit(EXIT_FAILURE)
+        jmprint('validating this utxo: ' + utxostr, "info")
         # as noted in `ImportWalletMixin` code comments, there is not
         # yet a functional auto-detection of key type from WIF, so the
-        # second argument is ignored; we assume p2sh-p2wpkh if segwit,
-        # else we assume p2pkh.
-        engine = BTC_P2SH_P2WPKH if segwit else BTC_P2PKH
+        # second argument is ignored; we assume p2sh-p2wpkh if segwit=True,
+        # p2pkh if segwit=False, and p2wpkh if segwit="native" (slightly
+        # ugly, just done for backwards compat.).
+        if utxo_address_type == "p2wpkh":
+            engine = BTC_P2WPKH
+        elif utxo_address_type == "p2sh-p2wpkh":
+            engine = BTC_P2SH_P2WPKH
+        elif utxo_address_type == "p2pkh":
+            engine = BTC_P2PKH
+        else:
+            raise Exception("Invalid argument: " + str(utxo_address_type))
         rawpriv, _ = BTCEngine.wif_to_privkey(priv)
         addr = engine.privkey_to_address(rawpriv)
         jmprint('claimed address: ' + addr, "info")
         res = jm_single().bc_interface.query_utxo_set([u])
         if len(res) != 1 or None in res:
-            jmprint("utxo not found on blockchain: " + str(u), "error")
+            jmprint("utxo not found on blockchain: " + utxostr, "error")
             return False
         if res[0]['address'] != addr:
-            jmprint("privkey corresponds to the wrong address for utxo: " + str(u), "error")
+            jmprint("privkey corresponds to the wrong address for utxo: " + utxostr, "error")
             jmprint("blockchain returned address: {}".format(res[0]['address']), "error")
             jmprint("your privkey gave this address: " + addr, "error")
             return False

jmclient/test/test_commitment_utils.py

@@ -2,6 +2,7 @@
 from commontest import DummyBlockchainInterface
 import pytest
 
+from jmbase import utxostr_to_utxo
 from jmclient import (load_test_config, jm_single)
 from jmclient.commitment_utils import get_utxo_info, validate_utxo_data
 from jmbitcoin import select_chain_params
@@ -15,12 +16,13 @@ def test_get_utxo_info():
     dbci = DummyBlockchainInterface()
     privkey = "L1RrrnXkcKut5DEMwtDthjwRcTTwED36thyL1DebVrKuwvohjMNi"
     #to verify use from_wif_privkey and privkey_to_address
-    iaddr = "1LDsjB43N2NAQ1Vbc2xyHca4iBBciN8iwC"
+    iaddr = "bc1q6tvmnmetj8vfz98vuetpvtuplqtj4uvvwjgxxc"
     fakeutxo = "aa"*32+":08"
-    
+    success, fakeutxo_bin = utxostr_to_utxo(fakeutxo)
+    assert success
     fake_query_results = [{'value': 200000000,
                                 'address': iaddr,
-                                'utxo': fakeutxo,
+                                'utxo': fakeutxo_bin,
                                 'confirms': 20}]    
     dbci.insert_fake_query_results(fake_query_results)
     jm_single().bc_interface = dbci
@@ -39,12 +41,12 @@ def test_get_utxo_info():
     with pytest.raises(Exception) as e_info:
         u, priv = get_utxo_info(fakeutxo + "," + p2)
 
-    utxodatas = [(fakeutxo, privkey)]
+    utxodatas = [(fakeutxo_bin, privkey)]
     retval = validate_utxo_data(utxodatas, False)
     assert retval
     #try to retrieve
     retval = validate_utxo_data(utxodatas, True)
-    assert retval[0] == (fakeutxo, 200000000)
+    assert retval[0] == (fakeutxo_bin, 200000000)
     fake_query_results[0]['address'] = "fakeaddress"
     dbci.insert_fake_query_results(fake_query_results)
     #validate should fail for wrong address

scripts/add-utxo.py

@@ -17,7 +17,8 @@ from jmclient import load_program_config, jm_single,\
     open_wallet, WalletService, add_external_commitments, update_commitments,\
     PoDLE, get_podle_commitments, get_utxo_info, validate_utxo_data, quit,\
     get_wallet_path, add_base_options, BTCEngine, BTC_P2SH_P2WPKH
-from jmbase.support import EXIT_SUCCESS, EXIT_FAILURE, EXIT_ARGERROR, jmprint
+from jmbase.support import EXIT_SUCCESS, EXIT_FAILURE, EXIT_ARGERROR, \
+     jmprint, utxostr_to_utxo
 
 
 def add_ext_commitments(utxo_datas):
@@ -47,7 +48,7 @@ def add_ext_commitments(utxo_datas):
             if 'P' not in ecs[u]:
                 ecs[u]['P']=P
             ecs[u]['reveal'][j] = {'P2':P2, 's':s, 'e':e}
-        add_external_commitments(ecs)
+    add_external_commitments(ecs)
 
 def main():
     parser = OptionParser(
@@ -181,11 +182,10 @@ def main():
         # minor note: adding a utxo from an external wallet for commitments, we
         # default to not allowing disabled utxos to avoid a privacy leak, so the
         # user would have to explicitly enable.
-        for md, utxos in wallet_service.get_utxos_by_mixdepth(hexfmt=False).items():
-            for (txid, index), utxo in utxos.items():
-                txhex = binascii.hexlify(txid).decode('ascii') + ':' + str(index)
-                wif = wallet_service.get_wif_path(utxo['path'])
-                utxo_data.append((txhex, wif))
+        for md, utxos in wallet_service.get_utxos_by_mixdepth().items():
+            for utxo, utxodata in utxos.items():
+                wif = wallet_service.get_wif_path(utxodata['path'])
+                utxo_data.append((utxo, wif))
 
     elif options.in_file:
         with open(options.in_file, "rb") as f:
@@ -196,7 +196,7 @@ def main():
                 u, priv = get_utxo_info(ul)
                 if not u:
                     quit(parser, "Failed to parse utxo info: " + str(ul))
-                utxo_data.append((u, priv))
+                utxo_data.append((utxostr_to_utxo(u), priv))
     elif options.in_json:
         if not os.path.isfile(options.in_json):
             jmprint("File: " + options.in_json + " not found.", "error")
@@ -208,7 +208,7 @@ def main():
                 jmprint("Failed to read json from " + options.in_json, "error")
                 sys.exit(EXIT_FAILURE)
         for u, pva in iteritems(utxo_json):
-            utxo_data.append((u, pva['privkey']))
+            utxo_data.append((utxostr_to_utxo(u), pva['privkey']))
     elif len(args) == 1:
         u = args[0]
         priv = input(
@@ -216,15 +216,26 @@ def main():
         u, priv = get_utxo_info(','.join([u, priv]))
         if not u:
             quit(parser, "Failed to parse utxo info: " + u)
-        utxo_data.append((u, priv))
+        utxo_data.append((utxostr_to_utxo(u), priv))
     else:
         quit(parser, 'Invalid syntax')
     if options.validate or options.vonly:
-        sw = False if jm_single().config.get("POLICY", "segwit") == "false" else True
-        if not validate_utxo_data(utxo_data, segwit=sw):
+        # if the utxos are loaded from a wallet, we use the wallet's
+        # txtype to determine the value of `utxo_address_type`; if not,
+        # we use joinmarket.cfg.
+        if options.loadwallet:
+            utxo_address_type = wallet_service.get_txtype()
+        else:
+            if jm_single().config.get("POLICY", "segwit") == "false":
+                utxo_address_type = "p2pkh"
+            elif jm_single().config.get("POLICY", "native") == "false":
+                utxo_address_type = "p2sh-p2wpkh"
+            else:
+                utxo_address_type = "p2wpkh"
+        if not validate_utxo_data(utxo_data, utxo_address_type=utxo_address_type):
             quit(parser, "Utxos did not validate, quitting")
     if options.vonly:
-        sys.exit(EXIT_ARGERROR)
+        sys.exit(EXIT_SUCCESS)
     
     #We are adding utxos to the external list
     assert len(utxo_data)

scripts/sendtomany.py

@@ -11,26 +11,24 @@ import jmbitcoin as btc
 from jmbase import get_log, jmprint, bintohex, utxostr_to_utxo
 from jmclient import load_program_config, estimate_tx_fee, jm_single,\
     validate_address, get_utxo_info, add_base_options,\
-    validate_utxo_data, quit, BTCEngine
+    validate_utxo_data, quit, BTCEngine, compute_tx_locktime
 
 log = get_log()
 
-def sign(utxo, priv, destaddrs, segwit=True):
+def sign(utxo, priv, destaddrs, utxo_address_type):
     """Sign a tx sending the amount amt, from utxo utxo,
     equally to each of addresses in list destaddrs,
-    after fees; the purpose is to create a large
-    number of utxos. If segwit=True the (single) utxo is assumed to
-    be of type segwit p2sh/p2wpkh.
+    after fees; the purpose is to create multiple utxos.
+    utxo_address_type must be one of p2sh-p2wpkh/p2wpkh/p2pkh.
     """
-    results = validate_utxo_data([(utxo, priv)], retrieve=True, segwit=segwit)
+    results = validate_utxo_data([(utxo, priv)], retrieve=True,
+                                 utxo_address_type=utxo_address_type)
     if not results:
         return False
     assert results[0][0] == utxo
     amt = results[0][1]
     ins = [utxo]
-    # TODO extend to other utxo types
-    txtype = 'p2sh-p2wpkh' if segwit else 'p2pkh'
-    estfee = estimate_tx_fee(1, len(destaddrs), txtype=txtype)
+    estfee = estimate_tx_fee(1, len(destaddrs), txtype=utxo_address_type)
     outs = []
     share = int((amt - estfee) / len(destaddrs))
     fee = amt - share*len(destaddrs)
@@ -38,10 +36,14 @@ def sign(utxo, priv, destaddrs, segwit=True):
     log.info("Using fee: " + str(fee))
     for i, addr in enumerate(destaddrs):
         outs.append({'address': addr, 'value': share})
-    tx = btc.mktx(ins, outs)
-    amtforsign = amt if segwit else None
+    tx = btc.make_shuffled_tx(ins, outs, version=2, locktime=compute_tx_locktime())
+    amtforsign = amt if utxo_address_type != "p2pkh" else None
     rawpriv, _ = BTCEngine.wif_to_privkey(priv)
-    success, msg = btc.sign(tx, 0, rawpriv, amount=amtforsign)
+    if utxo_address_type == "p2wpkh":
+        native = utxo_address_type
+    else:
+        native = False
+    success, msg = btc.sign(tx, 0, rawpriv, amount=amtforsign, native=native)
     assert success, msg
     return tx
     
@@ -69,28 +71,14 @@ def main():
                     " joinmarket.cfg for the former."
     )
     parser.add_option(
-        '-v',
-        '--validate-utxos',
-        action='store_true',
-        dest='validate',
-        help='validate the utxos and pubkeys provided against the blockchain',
-        default=False
-    )
-    parser.add_option(
-        '-o',
-        '--validate-only',
-        action='store_true',
-        dest='vonly',
-        help='only validate the provided utxos (file or command line), not add',
-        default=False
-    )
-    parser.add_option(
-        '-n',
-        '--non-segwit-input',
-        action='store_true',
-        dest='nonsegwit',
-        help='input is p2pkh ("1" address), not segwit; if not used, input is assumed to be segwit type.',
-        default=False
+        '-t',
+        '--utxo-address-type',
+        action='store',
+        dest='utxo_address_type',
+        help=('type of address of coin being spent - one of "p2pkh", "p2wpkh", "p2sh-p2wpkh". '
+        'No other scriptpubkey types (e.g. multisig) are supported. If not set, we default '
+        'to what is in joinmarket.cfg.'),
+        default=""
     )
     add_base_options(parser)
     (options, args) = parser.parse_args()
@@ -110,13 +98,21 @@ def main():
     success, utxo = utxostr_to_utxo(u)
     if not success:
         quit(parser, "Failed to load utxo from string: " + utxo)
-    txsigned = sign(utxo, priv, destaddrs, segwit = not options.nonsegwit)
+    if options.utxo_address_type == "":
+        if jm_single().config.get("POLICY", "segwit") == "false":
+            utxo_address_type = "p2pkh"
+        elif jm_single().config.get("POLICY", "native") == "false":
+            utxo_address_type = "p2sh-p2wpkh"
+        else:
+            utxo_address_type = "p2wpkh"
+    else:
+        utxo_address_type = options.utxo_address_type
+    txsigned = sign(utxo, priv, destaddrs, utxo_address_type)
     if not txsigned:
         log.info("Transaction signing operation failed, see debug messages for details.")
         return
     log.info("Got signed transaction:\n" + bintohex(txsigned.serialize()))
-    log.debug("Deserialized:")
-    log.debug(pformat(str(txsigned)))
+    log.info(btc.human_readable_transaction(txsigned))
     if input('Would you like to push to the network? (y/n):')[0] != 'y':
         log.info("You chose not to broadcast the transaction, quitting.")
         return