From 074afc3106a9e3bb014fa1a1f31a83f95a53a9d7 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Fri, 10 Feb 2017 14:42:45 +0200 Subject: [PATCH] Permit restart of tumbler with flag, persist state of tumbler in schedule file. Also fixes minor bug in blockr data read. Modifies schedule syntax to include complete/incomplete flag, so if restart is chosen then the schedule is continued from the first incomplete transaction in the sequence. --- jmclient/jmclient/__init__.py | 3 +- jmclient/jmclient/blockchaininterface.py | 5 +- jmclient/jmclient/schedule.py | 16 +++++-- scripts/cli_options.py | 18 +++++-- scripts/sample-schedule-for-testnet | 7 +-- scripts/sendpayment.py | 3 +- scripts/tumbler.py | 61 +++++++++++++++++++----- 7 files changed, 88 insertions(+), 25 deletions(-) diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index fe98f71..26dd57b 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -31,7 +31,8 @@ from .podle import (set_commitment_file, get_commitment_file, PoDLE, generate_podle, get_podle_commitments, update_commitments) from .schedule import (get_schedule, get_tumble_schedule, schedule_to_text, - tweak_tumble_schedule, human_readable_schedule_entry) + tweak_tumble_schedule, human_readable_schedule_entry, + schedule_to_text) from .commitment_utils import get_utxo_info, validate_utxo_data, quit # Set default logging handler to avoid "No handler found" warnings. diff --git a/jmclient/jmclient/blockchaininterface.py b/jmclient/jmclient/blockchaininterface.py index e180ab6..7a84760 100644 --- a/jmclient/jmclient/blockchaininterface.py +++ b/jmclient/jmclient/blockchaininterface.py @@ -772,7 +772,10 @@ class BlockrInterface(BlockchainInterface): #pragma: no cover data += blockr_data result = [] for txo in txout: - txdata = [d for d in data if d['tx'] == txo[:64]][0] + txdata_candidate = [d for d in data if d['tx'] == txo[:64]] + if len(txdata_candidate) == 0: + continue + txdata = txdata_candidate[0] vout = [v for v in txdata['vouts'] if v['n'] == int(txo[65:])][0] if "is_spent" in vout and vout['is_spent'] == 1: result.append(None) diff --git a/jmclient/jmclient/schedule.py b/jmclient/jmclient/schedule.py index 51c8554..c83acfe 100644 --- a/jmclient/jmclient/schedule.py +++ b/jmclient/jmclient/schedule.py @@ -11,6 +11,9 @@ from jmclient import (validate_address, rand_exp_array, - get_tumble_schedule(options, destaddrs): generate a schedule for tumbling from a given wallet, using options dict and specified destinations +- tweak_tumble_schedule(options, schedule, last_completed): + make alterations to the remaining entries in a mixdepth to maximize + the chance of success on re-trying """ def get_schedule(filename): @@ -21,7 +24,8 @@ def get_schedule(filename): if sl.startswith("#"): continue try: - mixdepth, amount, makercount, destaddr, waittime = sl.split(',') + mixdepth, amount, makercount, destaddr, waittime, completed = \ + sl.split(',') except ValueError as e: return (False, "Failed to parse schedule line: " + sl) try: @@ -35,13 +39,15 @@ def get_schedule(filename): makercount = int(makercount) destaddr = destaddr.strip() waittime = float(waittime) + completed = int(completed) except ValueError as e: return (False, "Failed to parse schedule line: " + sl) if destaddr != "INTERNAL": success, errmsg = validate_address(destaddr) if not success: return (False, "Invalid address: " + destaddr + "," + errmsg) - schedule.append([mixdepth, amount, makercount, destaddr, waittime]) + schedule.append([mixdepth, amount, makercount, destaddr, + waittime, completed]) return (True, schedule) def get_amount_fractions(power, count): @@ -63,6 +69,10 @@ def get_tumble_schedule(options, destaddrs): and zero as sweep (as before). This is a modified version of tumbler.py/generate_tumbler_tx() """ + if options['mixdepthsrc'] != 0: + raise NotImplementedError("Non-zero mixdepth source not supported; " + "restart the tumbler with --restart instead") + def lower_bounded_int(thelist, lowerbound): return [int(l) if int(l) >= lowerbound else lowerbound for l in thelist] @@ -119,7 +129,7 @@ def get_tumble_schedule(options, destaddrs): schedule = [] for t in tx_list: schedule.append([t['srcmixdepth'], t['amount_fraction'], - t['makercount'], t['destination'], t['wait']]) + t['makercount'], t['destination'], t['wait'], 0]) return schedule def tweak_tumble_schedule(options, schedule, last_completed): diff --git a/scripts/cli_options.py b/scripts/cli_options.py index 2f4e3e7..a4354b9 100644 --- a/scripts/cli_options.py +++ b/scripts/cli_options.py @@ -25,10 +25,7 @@ def get_tumbler_parser(): type='int', dest='mixdepthsrc', help= - 'Mixing depth to spend from. Useful if a previous tumbler run prematurely ended with ' - + - 'coins being left in higher mixing levels, this option can be used to resume without needing' - + ' to send to another address. default=0', + 'Mixing depth to spend from. DEPRECATED, do not use.', default=0) parser.add_option( '-f', @@ -41,6 +38,19 @@ def get_tumbler_parser(): 'for the total transaction fee, default=dynamically estimated, note that this is adjusted '+ 'based on the estimated fee calculated after tx construction, based on '+ 'policy set in joinmarket.cfg.') + parser.add_option('--restart', + action='store_true', + dest='restart', + default=False, + help=('Restarts the schedule currently found in the schedule file in the ' + 'logs directory, with name TUMBLE.schedule or what is set in the ' + 'schedulefile option.')) + parser.add_option('--schedulefile', + type='str', + dest='schedulefile', + default='TUMBLE.schedule', + help=('Name of schedule file for tumbler, useful for restart, default ' + 'TUMBLE.schedule')) parser.add_option( '-a', '--addrcount', diff --git a/scripts/sample-schedule-for-testnet b/scripts/sample-schedule-for-testnet index 3d4f812..c7d5f18 100644 --- a/scripts/sample-schedule-for-testnet +++ b/scripts/sample-schedule-for-testnet @@ -1,5 +1,6 @@ #sample for testing #fields: source mixdepth, amount (satoshis or fraction if non-int), -# makercount, destination, waittime (ignored if 0) -1, 110000000, 3, INTERNAL, 0 -0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw, 0 \ No newline at end of file +# makercount, destination, waittime (ignored if 0), completed flag: +# 0 = not yet completed, otherwise completed +1, 110000000, 3, INTERNAL, 0, 0 +0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw, 0, 0 \ No newline at end of file diff --git a/scripts/sendpayment.py b/scripts/sendpayment.py index 6f02d75..01e92cc 100644 --- a/scripts/sendpayment.py +++ b/scripts/sendpayment.py @@ -109,7 +109,8 @@ def main(): if not addr_valid: print('ERROR: Address invalid. ' + errormsg) return - schedule = [[options.mixdepth, amount, options.makercount, destaddr]] + schedule = [[options.mixdepth, amount, options.makercount, + destaddr, 0.0, 0]] else: result, schedule = get_schedule(options.schedule) if not result: diff --git a/scripts/tumbler.py b/scripts/tumbler.py index 2d3e81b..61a9c0e 100644 --- a/scripts/tumbler.py +++ b/scripts/tumbler.py @@ -11,12 +11,13 @@ import pprint import copy import logging -from jmclient import (Taker, load_program_config, get_schedule, weighted_order_choose, - JMTakerClientProtocolFactory, start_reactor, - validate_address, jm_single, WalletError, - Wallet, sync_wallet, get_tumble_schedule, - RegtestBitcoinCoreInterface, estimate_tx_fee, - tweak_tumble_schedule, human_readable_schedule_entry) +from jmclient import (Taker, load_program_config, get_schedule, + weighted_order_choose, JMTakerClientProtocolFactory, + start_reactor, validate_address, jm_single, WalletError, + Wallet, sync_wallet, get_tumble_schedule, + RegtestBitcoinCoreInterface, estimate_tx_fee, + tweak_tumble_schedule, human_readable_schedule_entry, + schedule_to_text) from jmbase.support import get_log, debug_dump_object, get_password from cli_options import get_tumbler_parser @@ -31,8 +32,7 @@ def main(): ('%(asctime)s %(message)s')) logsdir = os.path.join(os.path.dirname( jm_single().config_location), "logs") - fileHandler = logging.FileHandler( - logsdir + '/TUMBLE.log') + fileHandler = logging.FileHandler(os.path.join(logsdir, 'TUMBLE.log')) fileHandler.setFormatter(logFormatter) tumble_log.addHandler(fileHandler) @@ -65,15 +65,37 @@ def main(): sync_wallet(wallet, fast=options['fastsync']) #Parse options and generate schedule + #Output information to log files jm_single().mincjamount = options['mincjamount'] destaddrs = args[1:] print(destaddrs) - schedule = get_tumble_schedule(options, destaddrs) - print("got schedule:") - print(pprint.pformat(schedule)) - tumble_log.info("TUMBLE STARTING") + #If the --restart flag is set we read the schedule + #from the file, and filter out entries that are + #already complete + if options['restart']: + res, schedule = get_schedule(os.path.join(logsdir, + options['schedulefile'])) + if not res: + print("Failed to load schedule, name: " + str( + options['schedulefile'])) + sys.exit(0) + #This removes all entries that are marked as done; + #assumes user has not edited by hand, and edits have only happened + #on tx completion. + schedule = [s for s in schedule if s[5] == 0] + tumble_log.info("TUMBLE RESTARTING") + else: + #Create a new schedule from scratch + schedule = get_tumble_schedule(options, destaddrs) + tumble_log.info("TUMBLE STARTING") + with open(os.path.join(logsdir, options['schedulefile']), "wb") as f: + f.write(schedule_to_text(schedule)) + print("Schedule written to logs/" + options['schedulefile']) tumble_log.info("With this schedule: ") tumble_log.info(pprint.pformat(schedule)) + + print("Progress logging to logs/TUMBLE.log") + #callback for order checking; dummy/passthrough def filter_orders_callback(orders_fees, cjamount): return True @@ -87,6 +109,16 @@ def main(): #is sufficient taker.wallet.update_cache_index() if res: + #We persist the fact that the transaction is complete to the + #schedule file. Note that if a tweak to the schedule occurred, + #it only affects future (non-complete) transactions, so the final + #full record should always be accurate; but TUMBLE.log should be + #used for checking what actually happened. + taker.schedule[taker.schedule_index][5] = 1 + with open(os.path.join(logsdir, options['schedulefile']), + "wb") as f: + f.write(schedule_to_text(taker.schedule)) + tumble_log.info("Completed successfully this entry:") #the log output depends on if it's a sweep, and if it's to INTERNAL hrdestn = None @@ -131,6 +163,11 @@ def main(): hramt = taker.cjamount tumble_log.info(human_readable_schedule_entry( taker.schedule[taker.schedule_index], hramt)) + #copy of above, TODO refactor out + taker.schedule[taker.schedule_index][5] = 1 + with open(os.path.join(logsdir, options['schedulefile']), + "wb") as f: + f.write(schedule_to_text(taker.schedule)) reactor.stop() #to allow testing of confirm/unconfirm callback for multiple txs