diff --git a/jmclient/__init__.py b/jmclient/__init__.py index bf8fce1..6d39662 100644 --- a/jmclient/__init__.py +++ b/jmclient/__init__.py @@ -24,9 +24,12 @@ from .blockchaininterface import (BlockrInterface, BlockchainInterface, sync_wal RegtestBitcoinCoreInterface, BitcoinCoreInterface) from .client_protocol import JMTakerClientProtocolFactory, start_reactor from .podle import (set_commitment_file, get_commitment_file, - generate_podle_error_string) + generate_podle_error_string, add_external_commitments, + PoDLE, generate_podle, get_podle_commitments, + update_commitments) from .commands import * from .schedule import get_schedule +from .commitment_utils import get_utxo_info, validate_utxo_data, quit # Set default logging handler to avoid "No handler found" warnings. try: diff --git a/jmclient/commitment_utils.py b/jmclient/commitment_utils.py new file mode 100644 index 0000000..95d1d6a --- /dev/null +++ b/jmclient/commitment_utils.py @@ -0,0 +1,61 @@ +from __future__ import print_function + +import sys, os +import jmclient.btc as btc +from jmclient import jm_single, get_p2pk_vbyte + +def quit(parser, errmsg): + parser.error(errmsg) + sys.exit(0) + +def get_utxo_info(upriv): + """Verify that the input string parses correctly as (utxo, priv) + and return that. + """ + try: + u, priv = upriv.split(',') + u = u.strip() + priv = priv.strip() + txid, n = u.split(':') + assert len(txid)==64 + assert len(n) in range(1, 4) + n = int(n) + assert n in range(256) + except: + #not sending data to stdout in case privkey info + print("Failed to parse utxo information for utxo") + try: + hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte()) + except: + print("failed to parse privkey, make sure it's WIF compressed format.") + return u, priv + +def validate_utxo_data(utxo_datas, retrieve=False): + """For each txid: N, privkey, first + convert the privkey and convert to address, + then use the blockchain instance to look up + the utxo and check that its address field matches. + If retrieve is True, return the set of utxos and their values. + """ + results = [] + for u, priv in utxo_datas: + print('validating this utxo: ' + str(u)) + hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte()) + addr = btc.privkey_to_address(hexpriv, magicbyte=get_p2pk_vbyte()) + print('claimed address: ' + addr) + res = jm_single().bc_interface.query_utxo_set([u]) + print('blockchain shows this data: ' + str(res)) + if len(res) != 1: + print("utxo not found on blockchain: " + str(u)) + return False + if res[0]['address'] != addr: + print("privkey corresponds to the wrong address for utxo: " + str(u)) + print("blockchain returned address: " + res[0]['address']) + print("your privkey gave this address: " + addr) + return False + if retrieve: + results.append((u, res[0]['value'])) + print('all utxos validated OK') + if retrieve: + return results + return True \ No newline at end of file diff --git a/jmclient/configure.py b/jmclient/configure.py index c673e96..e7edea1 100644 --- a/jmclient/configure.py +++ b/jmclient/configure.py @@ -74,7 +74,7 @@ global_singleton.config = SafeConfigParser() #This is reset to a full path after load_program_config call global_singleton.config_location = 'joinmarket.cfg' #as above -global_singleton.commit_file_location = 'cmttools/commitments.json' +global_singleton.commit_file_location = 'cmtdata/commitments.json' global_singleton.wait_for_commitments = 0 @@ -186,8 +186,8 @@ taker_utxo_amtpercent = 20 accept_commitment_broadcasts = 1 #Location of your commitments.json file (stores commitments you've used -#and those you want to use in future), relative to root joinmarket directory. -commit_file_location = cmttools/commitments.json +#and those you want to use in future), relative to the scripts directory. +commit_file_location = cmtdata/commitments.json """ diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..5427b3e --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,53 @@ +# Command line scripts for Joinmarket + + +All user level scripts here. + +(The phrase "normal Joinmarket" in the below refers to the [existing repo](https://github.com/Joinmarket-Org/joinmarket). + +The subdirectories `logs` and `wallets` have the same role as in normal Joinmarket. +The subdirectory `cmtdata` contains only your `commitments.json` storage of your used +commitments (ignored by github of course!). The filename is set in joinmarket.cfg. + +The `joinmarket.cfg` will be created and maintained in this directory. + +Brief explanation of the function of each of the scripts: + +###sendpayment.py + +Either use the same syntax as for normal Joinmarket: + + `python sendpayment.py --fast -N 3 -m 1 -P wallet.json 50000000
` + +or use the new schedule approach. For an example, see the [sample schedule file](https://github.com/AdamISZ/joinmarket-clientserver/blob/master/scripts/sample-schedule-for-testnet). +Do: + + `python sendpayment.py --fast -S sample-schedule-for-testnet wallet.json` + +Note that the magic string `INTERNAL` in the file creates a payment to a new address +in the next mixdepth (wrapping around to zero if you reach the maximum mixdepth). + +The schedule file can have any name, and is a comma separated value file, the lists +must follow that format (length 4 items). + +###wallet-tool.py + +This is the same as in normal Joinmarket. + +###joinmarketd.py + +This file's role is explained in the main README in the top level directory. It only +takes one argument, the port it serves on: + + `python joinmarketd.py 12345` + +###add-utxo.py + +This works exactly as in normal Joinmarket, with the exception of the location +of the `commitments.json` file, explained above. + +###sendtomany.py + +As above. + +More details above, and probably more scripts, will be added later. \ No newline at end of file diff --git a/scripts/add-utxo.py b/scripts/add-utxo.py new file mode 100644 index 0000000..71e69b3 --- /dev/null +++ b/scripts/add-utxo.py @@ -0,0 +1,233 @@ +#! /usr/bin/env python +from __future__ import absolute_import +"""A very simple command line tool to import utxos to be used +as commitments into joinmarket's commitments.json file, allowing +users to retry transactions more often without getting banned by +the anti-snooping feature employed by makers. +""" + +import binascii +import sys +import os +import json +from pprint import pformat + +from optparse import OptionParser +import jmclient.btc as btc +from jmclient import (load_program_config, jm_single, get_p2pk_vbyte, + Wallet, sync_wallet, add_external_commitments, + generate_podle, update_commitments, PoDLE, + set_commitment_file, get_podle_commitments, + get_utxo_info, validate_utxo_data, quit) + +def add_ext_commitments(utxo_datas): + """Persist the PoDLE commitments for this utxo + to the commitments.json file. The number of separate + entries is dependent on the taker_utxo_retries entry, by + default 3. + """ + def generate_single_podle_sig(u, priv, i): + """Make a podle entry for key priv at index i, using a dummy utxo value. + This calls the underlying 'raw' code based on the class PoDLE, not the + library 'generate_podle' which intelligently searches and updates commitments. + """ + #Convert priv to hex + hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte()) + podle = PoDLE(u, hexpriv) + r = podle.generate_podle(i) + return (r['P'], r['P2'], r['sig'], + r['e'], r['commit']) + ecs = {} + for u, priv in utxo_datas: + ecs[u] = {} + ecs[u]['reveal']={} + for j in range(jm_single().config.getint("POLICY", "taker_utxo_retries")): + P, P2, s, e, commit = generate_single_podle_sig(u, priv, j) + if 'P' not in ecs[u]: + ecs[u]['P']=P + ecs[u]['reveal'][j] = {'P2':P2, 's':s, 'e':e} + add_external_commitments(ecs) + +def main(): + parser = OptionParser( + usage= + 'usage: %prog [options] [txid:n]', + description="Adds one or more utxos to the list that can be used to make " + "commitments for anti-snooping. Note that this utxo, and its " + "PUBkey, will be revealed to makers, so consider the privacy " + "implication. " + + "It may be useful to those who are having trouble making " + "coinjoins due to several unsuccessful attempts (especially " + "if your joinmarket wallet is new). " + + "'Utxo' means unspent transaction output, it must not " + "already be spent. " + "The options -w, -r and -R offer ways to load these utxos " + "from a file or wallet. " + "If you enter a single utxo without these options, you will be " + "prompted to enter the private key here - it must be in " + "WIF compressed format. " + + "BE CAREFUL about handling private keys! " + "Don't do this in insecure environments. " + + "Also note this ONLY works for standard (p2pkh) utxos." + ) + parser.add_option( + '-r', + '--read-from-file', + action='store', + type='str', + dest='in_file', + help='name of plain text csv file containing utxos, one per line, format: ' + 'txid:N, WIF-compressed-privkey' + ) + parser.add_option( + '-R', + '--read-from-json', + action='store', + type='str', + dest='in_json', + help='name of json formatted file containing utxos with private keys, as ' + 'output from "python wallet-tool.py -u -p walletname showutxos"' + ) + parser.add_option( + '-w', + '--load-wallet', + action='store', + type='str', + dest='loadwallet', + help='name of wallet from which to load utxos and use as commitments.' + ) + parser.add_option( + '-g', + '--gap-limit', + action='store', + type='int', + dest='gaplimit', + default = 6, + help='Only to be used with -w; gap limit for Joinmarket wallet, default 6.' + ) + parser.add_option( + '-M', + '--max-mixdepth', + action='store', + type='int', + dest='maxmixdepth', + default=5, + help='Only to be used with -w; number of mixdepths for wallet, default 5.' + ) + parser.add_option( + '-d', + '--delete-external', + action='store_true', + dest='delete_ext', + help='deletes the current list of external commitment utxos', + default=False + ) + parser.add_option( + '-v', + '--validate-utxos', + action='store_true', + dest='validate', + help='validate the utxos and pubkeys provided against the blockchain', + default=False + ) + parser.add_option( + '-o', + '--validate-only', + action='store_true', + dest='vonly', + help='only validate the provided utxos (file or command line), not add', + default=False + ) + parser.add_option('--fast', + action='store_true', + dest='fastsync', + default=False, + help=('choose to do fast wallet sync, only for Core and ' + 'only for previously synced wallet')) + (options, args) = parser.parse_args() + load_program_config() + #TODO; sort out "commit file location" global so this script can + #run without this hardcoding: + utxo_data = [] + if options.delete_ext: + other = options.in_file or options.in_json or options.loadwallet + if len(args) > 0 or other: + if raw_input("You have chosen to delete commitments, other arguments " + "will be ignored; continue? (y/n)") != 'y': + print "Quitting" + sys.exit(0) + c, e = get_podle_commitments() + print pformat(e) + if raw_input( + "You will remove the above commitments; are you sure? (y/n): ") != 'y': + print "Quitting" + sys.exit(0) + update_commitments(external_to_remove=e) + print "Commitments deleted." + sys.exit(0) + + #Three options (-w, -r, -R) for loading utxo and privkey pairs from a wallet, + #csv file or json file. + if options.loadwallet: + wallet = Wallet(options.loadwallet, + options.maxmixdepth, + options.gaplimit) + sync_wallet(wallet, fast=options.fastsync) + unsp = {} + for u, av in wallet.unspent.iteritems(): + addr = av['address'] + key = wallet.get_key_from_addr(addr) + wifkey = btc.wif_compressed_privkey(key, vbyte=get_p2pk_vbyte()) + unsp[u] = {'address': av['address'], + 'value': av['value'], 'privkey': wifkey} + for u, pva in unsp.iteritems(): + utxo_data.append((u, pva['privkey'])) + elif options.in_file: + with open(options.in_file, "rb") as f: + utxo_info = f.readlines() + for ul in utxo_info: + ul = ul.rstrip() + if ul: + u, priv = get_utxo_info(ul) + if not u: + quit(parser, "Failed to parse utxo info: " + str(ul)) + utxo_data.append((u, priv)) + elif options.in_json: + if not os.path.isfile(options.in_json): + print "File: " + options.in_json + " not found." + sys.exit(0) + with open(options.in_json, "rb") as f: + try: + utxo_json = json.loads(f.read()) + except: + print "Failed to read json from " + options.in_json + sys.exit(0) + for u, pva in utxo_json.iteritems(): + utxo_data.append((u, pva['privkey'])) + elif len(args) == 1: + u = args[0] + priv = raw_input( + 'input private key for ' + u + ', in WIF compressed format : ') + u, priv = get_utxo_info(','.join([u, priv])) + if not u: + quit(parser, "Failed to parse utxo info: " + u) + utxo_data.append((u, priv)) + else: + quit(parser, 'Invalid syntax') + if options.validate or options.vonly: + if not validate_utxo_data(utxo_data): + quit(parser, "Utxos did not validate, quitting") + if options.vonly: + sys.exit(0) + + #We are adding utxos to the external list + assert len(utxo_data) + add_ext_commitments(utxo_data) + +if __name__ == "__main__": + main() + print('done') diff --git a/scripts/cmtdata/.gitignore b/scripts/cmtdata/.gitignore new file mode 100644 index 0000000..c012572 --- /dev/null +++ b/scripts/cmtdata/.gitignore @@ -0,0 +1,4 @@ +# Ignore all +* +# Except this file +!.gitignore diff --git a/scripts/sendtomany.py b/scripts/sendtomany.py new file mode 100644 index 0000000..252afc7 --- /dev/null +++ b/scripts/sendtomany.py @@ -0,0 +1,105 @@ +#! /usr/bin/env python +from __future__ import absolute_import, print_function +"""A simple command line tool to create a bunch +of utxos from one (thus giving more potential commitments +for a Joinmarket user, although of course it may be useful +for other reasons). +""" + +import binascii +import sys, os +from pprint import pformat +from optparse import OptionParser +import jmclient.btc as btc +from jmclient import (load_program_config, estimate_tx_fee, jm_single, + get_p2pk_vbyte, validate_address, get_log, + get_utxo_info, validate_utxo_data, quit) +log = get_log() + +def sign(utxo, priv, destaddrs): + """Sign a tx sending the amount amt, from utxo utxo, + equally to each of addresses in list destaddrs, + after fees; the purpose is to create a large + number of utxos. + """ + results = validate_utxo_data([(utxo, priv)], retrieve=True) + if not results: + return False + assert results[0][0] == utxo + amt = results[0][1] + ins = [utxo] + estfee = estimate_tx_fee(1, len(destaddrs)) + outs = [] + share = int((amt - estfee) / len(destaddrs)) + fee = amt - share*len(destaddrs) + assert fee >= estfee + log.info("Using fee: " + str(fee)) + for i, addr in enumerate(destaddrs): + outs.append({'address': addr, 'value': share}) + unsigned_tx = btc.mktx(ins, outs) + return btc.sign(unsigned_tx, 0, btc.from_wif_privkey( + priv, vbyte=get_p2pk_vbyte())) + +def main(): + parser = OptionParser( + usage= + 'usage: %prog [options] utxo destaddr1 destaddr2 ..', + description="For creating multiple utxos from one (for commitments in JM)." + "Provide a utxo in form txid:N that has some unspent coins;" + "Specify a list of destination addresses and the coins will" + "be split equally between them (after bitcoin fees)." + + "You'll be prompted to enter the private key for the utxo" + "during the run; it must be in WIF compressed format." + "After the transaction is completed, the utxo strings for" + + "the new outputs will be shown." + "Note that these utxos will not be ready for use as external" + + "commitments in Joinmarket until 5 confirmations have passed." + " BE CAREFUL about handling private keys!" + " Don't do this in insecure environments." + " Also note this ONLY works for standard (p2pkh) utxos." + ) + parser.add_option( + '-v', + '--validate-utxos', + action='store_true', + dest='validate', + help='validate the utxos and pubkeys provided against the blockchain', + default=False + ) + parser.add_option( + '-o', + '--validate-only', + action='store_true', + dest='vonly', + help='only validate the provided utxos (file or command line), not add', + default=False + ) + (options, args) = parser.parse_args() + load_program_config() + if len(args) < 2: + quit(parser, 'Invalid syntax') + u = args[0] + priv = raw_input( + 'input private key for ' + u + ', in WIF compressed format : ') + u, priv = get_utxo_info(','.join([u, priv])) + if not u: + quit(parser, "Failed to parse utxo info: " + u) + destaddrs = args[1:] + for d in destaddrs: + if not validate_address(d): + quit(parser, "Address was not valid; wrong network?: " + d) + txsigned = sign(u, priv, destaddrs) + log.debug("Got signed transaction:\n" + txsigned) + log.debug("Deserialized:") + log.debug(pformat(btc.deserialize(txsigned))) + if raw_input('Would you like to push to the network? (y/n):')[0] != 'y': + log.info("You chose not to broadcast the transaction, quitting.") + return + jm_single().bc_interface.pushtx(txsigned) + +if __name__ == "__main__": + main() + print('done')