Permit restart of tumbler with flag, persist state of tumbler in schedule file.

Also fixes minor bug in blockr data read.
Modifies schedule syntax to include complete/incomplete flag, so if
restart is chosen then the schedule is continued from the first
incomplete transaction in the sequence.

jmclient/jmclient/__init__.py

@@ -31,7 +31,8 @@ from .podle import (set_commitment_file, get_commitment_file,
                     PoDLE, generate_podle, get_podle_commitments,
                     update_commitments)
 from .schedule import (get_schedule, get_tumble_schedule, schedule_to_text,
-                       tweak_tumble_schedule, human_readable_schedule_entry)
+                       tweak_tumble_schedule, human_readable_schedule_entry,
+                       schedule_to_text)
 from .commitment_utils import get_utxo_info, validate_utxo_data, quit
 # Set default logging handler to avoid "No handler found" warnings.
 

jmclient/jmclient/blockchaininterface.py

@@ -772,7 +772,10 @@ class BlockrInterface(BlockchainInterface): #pragma: no cover
             data += blockr_data
         result = []
         for txo in txout:
-            txdata = [d for d in data if d['tx'] == txo[:64]][0]
+            txdata_candidate = [d for d in data if d['tx'] == txo[:64]]
+            if len(txdata_candidate) == 0:
+                continue
+            txdata = txdata_candidate[0]
             vout = [v for v in txdata['vouts'] if v['n'] == int(txo[65:])][0]
             if "is_spent" in vout and vout['is_spent'] == 1:
                 result.append(None)

jmclient/jmclient/schedule.py

@@ -11,6 +11,9 @@ from jmclient import (validate_address, rand_exp_array,
 - get_tumble_schedule(options, destaddrs):
     generate a schedule for tumbling from a given wallet, using options dict
     and specified destinations
+- tweak_tumble_schedule(options, schedule, last_completed):
+    make alterations to the remaining entries in a mixdepth to maximize
+    the chance of success on re-trying
 """
 
 def get_schedule(filename):
@@ -21,7 +24,8 @@ def get_schedule(filename):
             if sl.startswith("#"):
                 continue
             try:
-                mixdepth, amount, makercount, destaddr, waittime = sl.split(',')
+                mixdepth, amount, makercount, destaddr, waittime, completed = \
+                    sl.split(',')
             except ValueError as e:
                 return (False, "Failed to parse schedule line: " + sl)
             try:
@@ -35,13 +39,15 @@ def get_schedule(filename):
                 makercount = int(makercount)
                 destaddr = destaddr.strip()
                 waittime = float(waittime)
+                completed = int(completed)
             except ValueError as e:
                 return (False, "Failed to parse schedule line: " + sl)
             if destaddr != "INTERNAL":
                 success, errmsg = validate_address(destaddr)
                 if not success:
                     return (False, "Invalid address: " + destaddr + "," + errmsg)
-            schedule.append([mixdepth, amount, makercount, destaddr, waittime])
+            schedule.append([mixdepth, amount, makercount, destaddr,
+                             waittime, completed])
     return (True, schedule)
 
 def get_amount_fractions(power, count):
@@ -63,6 +69,10 @@ def get_tumble_schedule(options, destaddrs):
     and zero as sweep (as before).
     This is a modified version of tumbler.py/generate_tumbler_tx()
     """
+    if options['mixdepthsrc'] != 0:
+        raise NotImplementedError("Non-zero mixdepth source not supported; "
+                                  "restart the tumbler with --restart instead")
+
     def lower_bounded_int(thelist, lowerbound):
         return [int(l) if int(l) >= lowerbound else lowerbound for l in thelist]
 
@@ -119,7 +129,7 @@ def get_tumble_schedule(options, destaddrs):
     schedule = []
     for t in tx_list:
         schedule.append([t['srcmixdepth'], t['amount_fraction'],
-                  t['makercount'], t['destination'], t['wait']])
+                  t['makercount'], t['destination'], t['wait'], 0])
     return schedule
 
 def tweak_tumble_schedule(options, schedule, last_completed):

scripts/cli_options.py

@@ -25,10 +25,7 @@ def get_tumbler_parser():
             type='int',
             dest='mixdepthsrc',
             help=
-            'Mixing depth to spend from. Useful if a previous tumbler run prematurely ended with '
-            +
-            'coins being left in higher mixing levels, this option can be used to resume without needing'
-            + ' to send to another address. default=0',
+            'Mixing depth to spend from. DEPRECATED, do not use.',
             default=0)
     parser.add_option(
             '-f',
@@ -41,6 +38,19 @@ def get_tumbler_parser():
         'for the total transaction fee, default=dynamically estimated, note that this is adjusted '+
         'based on the estimated fee calculated after tx construction, based on '+
         'policy set in joinmarket.cfg.')
+    parser.add_option('--restart',
+        action='store_true',
+        dest='restart',
+        default=False,
+        help=('Restarts the schedule currently found in the schedule file in the '
+              'logs directory, with name TUMBLE.schedule or what is set in the '
+              'schedulefile option.'))
+    parser.add_option('--schedulefile',
+            type='str',
+            dest='schedulefile',
+            default='TUMBLE.schedule',
+            help=('Name of schedule file for tumbler, useful for restart, default '
+                  'TUMBLE.schedule'))
     parser.add_option(
             '-a',
             '--addrcount',

scripts/sample-schedule-for-testnet

@@ -1,5 +1,6 @@
 #sample for testing
 #fields: source mixdepth, amount (satoshis or fraction if non-int),
-# makercount, destination, waittime (ignored if 0)
-1, 110000000, 3, INTERNAL, 0
-0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw, 0
+# makercount, destination, waittime (ignored if 0), completed flag:
+# 0 = not yet completed, otherwise completed
+1, 110000000, 3, INTERNAL, 0, 0
+0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw, 0, 0

scripts/sendpayment.py

@@ -109,7 +109,8 @@ def main():
         if not addr_valid:
             print('ERROR: Address invalid. ' + errormsg)
             return
-        schedule = [[options.mixdepth, amount, options.makercount, destaddr]]
+        schedule = [[options.mixdepth, amount, options.makercount,
+                     destaddr, 0.0, 0]]
     else:
         result, schedule = get_schedule(options.schedule)
         if not result:

scripts/tumbler.py

@@ -11,12 +11,13 @@ import pprint
 import copy
 import logging
 
-from jmclient import (Taker, load_program_config, get_schedule, weighted_order_choose,
-                              JMTakerClientProtocolFactory, start_reactor,
-                              validate_address, jm_single, WalletError,
-                              Wallet, sync_wallet, get_tumble_schedule,
-                              RegtestBitcoinCoreInterface, estimate_tx_fee,
-                              tweak_tumble_schedule, human_readable_schedule_entry)
+from jmclient import (Taker, load_program_config, get_schedule,
+                      weighted_order_choose, JMTakerClientProtocolFactory,
+                      start_reactor, validate_address, jm_single, WalletError,
+                      Wallet, sync_wallet, get_tumble_schedule,
+                      RegtestBitcoinCoreInterface, estimate_tx_fee,
+                      tweak_tumble_schedule, human_readable_schedule_entry,
+                      schedule_to_text)
 
 from jmbase.support import get_log, debug_dump_object, get_password
 from cli_options import get_tumbler_parser
@@ -31,8 +32,7 @@ def main():
         ('%(asctime)s %(message)s'))
     logsdir = os.path.join(os.path.dirname(
     jm_single().config_location), "logs")
-    fileHandler = logging.FileHandler(
-        logsdir + '/TUMBLE.log')
+    fileHandler = logging.FileHandler(os.path.join(logsdir, 'TUMBLE.log'))
     fileHandler.setFormatter(logFormatter)
     tumble_log.addHandler(fileHandler)
 
@@ -65,15 +65,37 @@ def main():
     sync_wallet(wallet, fast=options['fastsync'])
 
     #Parse options and generate schedule
+    #Output information to log files
     jm_single().mincjamount = options['mincjamount']
     destaddrs = args[1:]
     print(destaddrs)
-    schedule = get_tumble_schedule(options, destaddrs)
-    print("got schedule:")
-    print(pprint.pformat(schedule))
-    tumble_log.info("TUMBLE STARTING")
+    #If the --restart flag is set we read the schedule
+    #from the file, and filter out entries that are
+    #already complete
+    if options['restart']:
+        res, schedule = get_schedule(os.path.join(logsdir,
+                                                  options['schedulefile']))
+        if not res:
+            print("Failed to load schedule, name: " + str(
+                options['schedulefile']))
+            sys.exit(0)
+        #This removes all entries that are marked as done;
+        #assumes user has not edited by hand, and edits have only happened
+        #on tx completion.
+        schedule = [s for s in schedule if s[5] == 0]
+        tumble_log.info("TUMBLE RESTARTING")
+    else:
+        #Create a new schedule from scratch
+        schedule = get_tumble_schedule(options, destaddrs)
+        tumble_log.info("TUMBLE STARTING")
+        with open(os.path.join(logsdir, options['schedulefile']), "wb") as f:
+            f.write(schedule_to_text(schedule))
+        print("Schedule written to logs/" + options['schedulefile'])
     tumble_log.info("With this schedule: ")
     tumble_log.info(pprint.pformat(schedule))
+
+    print("Progress logging to logs/TUMBLE.log")
+
     #callback for order checking; dummy/passthrough
     def filter_orders_callback(orders_fees, cjamount):
         return True
@@ -87,6 +109,16 @@ def main():
             #is sufficient
             taker.wallet.update_cache_index()
             if res:
+                #We persist the fact that the transaction is complete to the
+                #schedule file. Note that if a tweak to the schedule occurred,
+                #it only affects future (non-complete) transactions, so the final
+                #full record should always be accurate; but TUMBLE.log should be
+                #used for checking what actually happened.
+                taker.schedule[taker.schedule_index][5] = 1
+                with open(os.path.join(logsdir, options['schedulefile']),
+                          "wb") as f:
+                    f.write(schedule_to_text(taker.schedule))
+
                 tumble_log.info("Completed successfully this entry:")
                 #the log output depends on if it's a sweep, and if it's to INTERNAL
                 hrdestn = None
@@ -131,6 +163,11 @@ def main():
                 hramt = taker.cjamount
                 tumble_log.info(human_readable_schedule_entry(
                     taker.schedule[taker.schedule_index], hramt))
+                #copy of above, TODO refactor out
+                taker.schedule[taker.schedule_index][5] = 1
+                with open(os.path.join(logsdir, options['schedulefile']),
+                          "wb") as f:
+                    f.write(schedule_to_text(taker.schedule))
             reactor.stop()
 
     #to allow testing of confirm/unconfirm callback for multiple txs