Add commitments tools

add-utxo.py and sendtomany.py; cmttools renamed to cmtdata, only
stores commitments.json files.

jmclient/__init__.py

@@ -24,9 +24,12 @@ from .blockchaininterface import (BlockrInterface, BlockchainInterface, sync_wal
                                   RegtestBitcoinCoreInterface, BitcoinCoreInterface)
 from .client_protocol import JMTakerClientProtocolFactory, start_reactor
 from .podle import (set_commitment_file, get_commitment_file,
-                    generate_podle_error_string)
+                    generate_podle_error_string, add_external_commitments,
+                    PoDLE, generate_podle, get_podle_commitments,
+                    update_commitments)
 from .commands import *
 from .schedule import get_schedule
+from .commitment_utils import get_utxo_info, validate_utxo_data, quit
 # Set default logging handler to avoid "No handler found" warnings.
 
 try:

jmclient/commitment_utils.py

@@ -0,0 +1,61 @@
+from __future__ import print_function
+
+import sys, os
+import jmclient.btc as btc
+from jmclient import jm_single, get_p2pk_vbyte
+
+def quit(parser, errmsg):
+    parser.error(errmsg)
+    sys.exit(0)
+
+def get_utxo_info(upriv):
+    """Verify that the input string parses correctly as (utxo, priv)
+    and return that.
+    """
+    try:
+        u, priv = upriv.split(',')
+        u = u.strip()
+        priv = priv.strip()
+        txid, n = u.split(':')
+        assert len(txid)==64
+        assert len(n) in range(1, 4)
+        n = int(n)
+        assert n in range(256)
+    except:
+        #not sending data to stdout in case privkey info
+        print("Failed to parse utxo information for utxo")
+    try:
+        hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte())
+    except:
+        print("failed to parse privkey, make sure it's WIF compressed format.")
+    return u, priv
+    
+def validate_utxo_data(utxo_datas, retrieve=False):
+    """For each txid: N, privkey, first
+    convert the privkey and convert to address,
+    then use the blockchain instance to look up
+    the utxo and check that its address field matches.
+    If retrieve is True, return the set of utxos and their values.
+    """
+    results = []
+    for u, priv in utxo_datas:
+        print('validating this utxo: ' + str(u))
+        hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte())
+        addr = btc.privkey_to_address(hexpriv, magicbyte=get_p2pk_vbyte())
+        print('claimed address: ' + addr)
+        res = jm_single().bc_interface.query_utxo_set([u])
+        print('blockchain shows this data: ' + str(res))
+        if len(res) != 1:
+            print("utxo not found on blockchain: " + str(u))
+            return False
+        if res[0]['address'] != addr:
+            print("privkey corresponds to the wrong address for utxo: " + str(u))
+            print("blockchain returned address: " + res[0]['address'])
+            print("your privkey gave this address: " + addr)
+            return False
+        if retrieve:
+            results.append((u, res[0]['value']))
+    print('all utxos validated OK')
+    if retrieve:
+        return results
+    return True

jmclient/configure.py

@@ -74,7 +74,7 @@ global_singleton.config = SafeConfigParser()
 #This is reset to a full path after load_program_config call
 global_singleton.config_location = 'joinmarket.cfg'
 #as above
-global_singleton.commit_file_location = 'cmttools/commitments.json'
+global_singleton.commit_file_location = 'cmtdata/commitments.json'
 global_singleton.wait_for_commitments = 0
 
 
@@ -186,8 +186,8 @@ taker_utxo_amtpercent = 20
 accept_commitment_broadcasts = 1
 
 #Location of your commitments.json file (stores commitments you've used
-#and those you want to use in future), relative to root joinmarket directory.
-commit_file_location = cmttools/commitments.json
+#and those you want to use in future), relative to the scripts directory.
+commit_file_location = cmtdata/commitments.json
 """
 
 

scripts/README.md

@@ -0,0 +1,53 @@
+# Command line scripts for Joinmarket
+
+
+All user level scripts here.
+
+(The phrase "normal Joinmarket" in the below refers to the [existing repo](https://github.com/Joinmarket-Org/joinmarket).
+
+The subdirectories `logs` and `wallets` have the same role as in normal Joinmarket.
+The subdirectory `cmtdata` contains only your `commitments.json` storage of your used
+commitments (ignored by github of course!). The filename is set in joinmarket.cfg.
+
+The `joinmarket.cfg` will be created and maintained in this directory.
+
+Brief explanation of the function of each of the scripts:
+
+###sendpayment.py
+
+Either use the same syntax as for normal Joinmarket:
+
+    `python sendpayment.py --fast -N 3 -m 1 -P wallet.json 50000000 <address>`
+
+or use the new schedule approach. For an example, see the [sample schedule file](https://github.com/AdamISZ/joinmarket-clientserver/blob/master/scripts/sample-schedule-for-testnet).
+Do:
+
+    `python sendpayment.py --fast -S sample-schedule-for-testnet wallet.json`
+
+Note that the magic string `INTERNAL` in the file creates a payment to a new address
+in the next mixdepth (wrapping around to zero if you reach the maximum mixdepth).
+
+The schedule file can have any name, and is a comma separated value file, the lists
+must follow that format (length 4 items).
+
+###wallet-tool.py
+
+This is the same as in normal Joinmarket.
+
+###joinmarketd.py
+
+This file's role is explained in the main README in the top level directory. It only
+takes one argument, the port it serves on:
+
+    `python joinmarketd.py 12345`
+
+###add-utxo.py
+
+This works exactly as in normal Joinmarket, with the exception of the location
+of the `commitments.json` file, explained above.
+
+###sendtomany.py
+
+As above.
+
+More details above, and probably more scripts, will be added later.

scripts/add-utxo.py

@@ -0,0 +1,233 @@
+#! /usr/bin/env python
+from __future__ import absolute_import
+"""A very simple command line tool to import utxos to be used
+as commitments into joinmarket's commitments.json file, allowing
+users to retry transactions more often without getting banned by
+the anti-snooping feature employed by makers.
+"""
+
+import binascii
+import sys
+import os
+import json
+from pprint import pformat
+
+from optparse import OptionParser
+import jmclient.btc as btc
+from jmclient import (load_program_config, jm_single, get_p2pk_vbyte,
+                      Wallet, sync_wallet, add_external_commitments,
+                      generate_podle, update_commitments, PoDLE,
+                      set_commitment_file, get_podle_commitments,
+                      get_utxo_info, validate_utxo_data, quit)
+
+def add_ext_commitments(utxo_datas):
+    """Persist the PoDLE commitments for this utxo
+    to the commitments.json file. The number of separate
+    entries is dependent on the taker_utxo_retries entry, by
+    default 3.
+    """
+    def generate_single_podle_sig(u, priv, i):
+        """Make a podle entry for key priv at index i, using a dummy utxo value.
+        This calls the underlying 'raw' code based on the class PoDLE, not the
+        library 'generate_podle' which intelligently searches and updates commitments.
+        """
+        #Convert priv to hex
+        hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte())
+        podle = PoDLE(u, hexpriv)
+        r = podle.generate_podle(i)
+        return (r['P'], r['P2'], r['sig'],
+                r['e'], r['commit'])
+    ecs = {}
+    for u, priv in utxo_datas:
+        ecs[u] = {}
+        ecs[u]['reveal']={}
+        for j in range(jm_single().config.getint("POLICY", "taker_utxo_retries")):
+            P, P2, s, e, commit = generate_single_podle_sig(u, priv, j)
+            if 'P' not in ecs[u]:
+                ecs[u]['P']=P
+            ecs[u]['reveal'][j] = {'P2':P2, 's':s, 'e':e}
+        add_external_commitments(ecs)
+
+def main():
+    parser = OptionParser(
+        usage=
+        'usage: %prog [options] [txid:n]',
+        description="Adds one or more utxos to the list that can be used to make "
+                    "commitments for anti-snooping. Note that this utxo, and its "
+                    "PUBkey, will be revealed to makers, so consider the privacy "
+                    "implication. "
+                    
+                    "It may be useful to those who are having trouble making "
+                    "coinjoins due to several unsuccessful attempts (especially "
+                    "if your joinmarket wallet is new). "
+                    
+                    "'Utxo' means unspent transaction output, it must not "
+                    "already be spent. "
+                    "The options -w, -r and -R offer ways to load these utxos "
+                    "from a file or wallet. "
+                    "If you enter a single utxo without these options, you will be "
+                    "prompted to enter the private key here - it must be in "
+                    "WIF compressed format. "
+
+                    "BE CAREFUL about handling private keys! "
+                    "Don't do this in insecure environments. "
+                    
+                    "Also note this ONLY works for standard (p2pkh) utxos."
+    )
+    parser.add_option(
+        '-r',
+        '--read-from-file',
+        action='store',
+        type='str',
+        dest='in_file',
+        help='name of plain text csv file containing utxos, one per line, format: '
+        'txid:N, WIF-compressed-privkey'
+    )
+    parser.add_option(
+        '-R',
+        '--read-from-json',
+        action='store',
+        type='str',
+        dest='in_json',
+        help='name of json formatted file containing utxos with private keys, as '
+        'output from "python wallet-tool.py -u -p walletname showutxos"'
+        )
+    parser.add_option(
+        '-w',
+        '--load-wallet',
+        action='store',
+        type='str',
+        dest='loadwallet',
+        help='name of wallet from which to load utxos and use as commitments.'
+        )
+    parser.add_option(
+        '-g',
+        '--gap-limit',
+        action='store',
+        type='int',
+        dest='gaplimit',
+        default = 6,
+        help='Only to be used with -w; gap limit for Joinmarket wallet, default 6.'
+    )
+    parser.add_option(
+        '-M',
+        '--max-mixdepth',
+        action='store',
+        type='int',
+        dest='maxmixdepth',
+        default=5,
+        help='Only to be used with -w; number of mixdepths for wallet, default 5.'
+    )
+    parser.add_option(
+        '-d',
+        '--delete-external',
+        action='store_true',
+        dest='delete_ext',
+        help='deletes the current list of external commitment utxos',
+        default=False
+        )
+    parser.add_option(
+        '-v',
+        '--validate-utxos',
+        action='store_true',
+        dest='validate',
+        help='validate the utxos and pubkeys provided against the blockchain',
+        default=False
+    )
+    parser.add_option(
+        '-o',
+        '--validate-only',
+        action='store_true',
+        dest='vonly',
+        help='only validate the provided utxos (file or command line), not add',
+        default=False
+    )
+    parser.add_option('--fast',
+                      action='store_true',
+                      dest='fastsync',
+                      default=False,
+                      help=('choose to do fast wallet sync, only for Core and '
+                      'only for previously synced wallet'))
+    (options, args) = parser.parse_args()
+    load_program_config()
+    #TODO; sort out "commit file location" global so this script can
+    #run without this hardcoding:
+    utxo_data = []
+    if options.delete_ext:
+        other = options.in_file or options.in_json or options.loadwallet
+        if len(args) > 0 or other:
+            if raw_input("You have chosen to delete commitments, other arguments "
+                         "will be ignored; continue? (y/n)") != 'y':
+                print "Quitting"
+                sys.exit(0)
+        c, e = get_podle_commitments()
+        print pformat(e)
+        if raw_input(
+            "You will remove the above commitments; are you sure? (y/n): ") != 'y':
+            print "Quitting"
+            sys.exit(0)
+        update_commitments(external_to_remove=e)
+        print "Commitments deleted."
+        sys.exit(0)
+
+    #Three options (-w, -r, -R) for loading utxo and privkey pairs from a wallet,
+    #csv file or json file.
+    if options.loadwallet:
+        wallet = Wallet(options.loadwallet,
+                            options.maxmixdepth,
+                            options.gaplimit)
+        sync_wallet(wallet, fast=options.fastsync)
+        unsp = {}
+        for u, av in wallet.unspent.iteritems():
+                    addr = av['address']
+                    key = wallet.get_key_from_addr(addr)
+                    wifkey = btc.wif_compressed_privkey(key, vbyte=get_p2pk_vbyte())
+                    unsp[u] = {'address': av['address'],
+                               'value': av['value'], 'privkey': wifkey}
+        for u, pva  in unsp.iteritems():
+            utxo_data.append((u, pva['privkey']))
+    elif options.in_file:
+        with open(options.in_file, "rb") as f:
+            utxo_info = f.readlines()
+        for ul in utxo_info:
+            ul = ul.rstrip()
+            if ul:
+                u, priv = get_utxo_info(ul)
+                if not u:
+                    quit(parser, "Failed to parse utxo info: " + str(ul))
+                utxo_data.append((u, priv))
+    elif options.in_json:
+        if not os.path.isfile(options.in_json):
+            print "File: " + options.in_json + " not found."
+            sys.exit(0)
+        with open(options.in_json, "rb") as f:
+            try:
+                utxo_json = json.loads(f.read())
+            except:
+                print "Failed to read json from " + options.in_json
+                sys.exit(0)
+        for u, pva in utxo_json.iteritems():
+            utxo_data.append((u, pva['privkey']))
+    elif len(args) == 1:
+        u = args[0]
+        priv = raw_input(
+            'input private key for ' + u + ', in WIF compressed format : ')
+        u, priv = get_utxo_info(','.join([u, priv]))
+        if not u:
+            quit(parser, "Failed to parse utxo info: " + u)
+        utxo_data.append((u, priv))
+    else:
+        quit(parser, 'Invalid syntax')
+    if options.validate or options.vonly:
+        if not validate_utxo_data(utxo_data):
+            quit(parser, "Utxos did not validate, quitting")
+    if options.vonly:
+        sys.exit(0)
+    
+    #We are adding utxos to the external list
+    assert len(utxo_data)
+    add_ext_commitments(utxo_data)
+
+if __name__ == "__main__":
+    main()
+    print('done')

scripts/cmtdata/.gitignore

@@ -0,0 +1,4 @@
+# Ignore all 
+*
+# Except this file
+!.gitignore

scripts/sendtomany.py

@@ -0,0 +1,105 @@
+#! /usr/bin/env python
+from __future__ import absolute_import, print_function
+"""A simple command line tool to create a bunch
+of utxos from one (thus giving more potential commitments
+for a Joinmarket user, although of course it may be useful
+for other reasons).
+"""
+
+import binascii
+import sys, os
+from pprint import pformat
+from optparse import OptionParser
+import jmclient.btc as btc
+from jmclient import (load_program_config, estimate_tx_fee, jm_single,
+                      get_p2pk_vbyte, validate_address, get_log,
+                      get_utxo_info, validate_utxo_data, quit)
+log = get_log()
+
+def sign(utxo, priv, destaddrs):
+    """Sign a tx sending the amount amt, from utxo utxo,
+    equally to each of addresses in list destaddrs,
+    after fees; the purpose is to create a large
+    number of utxos.
+    """
+    results = validate_utxo_data([(utxo, priv)], retrieve=True)
+    if not results:
+        return False
+    assert results[0][0] == utxo
+    amt = results[0][1]
+    ins = [utxo]
+    estfee = estimate_tx_fee(1, len(destaddrs))
+    outs = []
+    share = int((amt - estfee) / len(destaddrs))
+    fee = amt - share*len(destaddrs)
+    assert fee >= estfee
+    log.info("Using fee: " + str(fee))
+    for i, addr in enumerate(destaddrs):
+        outs.append({'address': addr, 'value': share})
+    unsigned_tx = btc.mktx(ins, outs)
+    return btc.sign(unsigned_tx, 0, btc.from_wif_privkey(
+        priv, vbyte=get_p2pk_vbyte()))
+    
+def main():
+    parser = OptionParser(
+        usage=
+        'usage: %prog [options] utxo destaddr1 destaddr2 ..',
+        description="For creating multiple utxos from one (for commitments in JM)."
+                    "Provide a utxo in form txid:N that has some unspent coins;"
+                    "Specify a list of destination addresses and the coins will"
+                    "be split equally between them (after bitcoin fees)."
+
+                    "You'll be prompted to enter the private key for the utxo"
+                    "during the run; it must be in WIF compressed format."
+                    "After the transaction is completed, the utxo strings for"
+
+                    "the new outputs will be shown."
+                    "Note that these utxos will not be ready for use as external"
+
+                    "commitments in Joinmarket until 5 confirmations have passed."
+                    " BE CAREFUL about handling private keys!"
+                    " Don't do this in insecure environments."
+                    " Also note this ONLY works for standard (p2pkh) utxos."
+    )
+    parser.add_option(
+        '-v',
+        '--validate-utxos',
+        action='store_true',
+        dest='validate',
+        help='validate the utxos and pubkeys provided against the blockchain',
+        default=False
+    )
+    parser.add_option(
+        '-o',
+        '--validate-only',
+        action='store_true',
+        dest='vonly',
+        help='only validate the provided utxos (file or command line), not add',
+        default=False
+    )
+    (options, args) = parser.parse_args()
+    load_program_config()
+    if len(args) < 2:
+        quit(parser, 'Invalid syntax')
+    u = args[0]
+    priv = raw_input(
+        'input private key for ' + u + ', in WIF compressed format : ')
+    u, priv = get_utxo_info(','.join([u, priv]))
+    if not u:
+        quit(parser, "Failed to parse utxo info: " + u)
+    destaddrs = args[1:]
+    for d in destaddrs:
+        if not validate_address(d):
+            quit(parser, "Address was not valid; wrong network?: " + d)
+    txsigned = sign(u, priv, destaddrs)
+    log.debug("Got signed transaction:\n" + txsigned)
+    log.debug("Deserialized:")
+    log.debug(pformat(btc.deserialize(txsigned)))
+    if raw_input('Would you like to push to the network? (y/n):')[0] != 'y':
+        log.info("You chose not to broadcast the transaction, quitting.")
+        return
+    jm_single().bc_interface.pushtx(txsigned)
+
+if __name__ == "__main__":
+    main()
+    print('done')