Add sweep coinjoins to start of tumbler schedule

The tumbler schedule is split into two stages. Stage 2 is the same
as before while stage 1 attempts to fully spend each mixdepth in a
sweep coinjoin with no change address.

The wait time between these stage 1 coinjoins is longer than for
stage 2 coinjoins, the increase is determined by a new parameter
called `stage1_timelambda_increase`.

This is part of the 2/2019 Plan to improve the privacy of JoinMarket's
tumbler script:
https://gist.github.com/chris-belcher/7e92810f07328fdfdef2ce444aad0968

jmclient/jmclient/schedule.py

@@ -3,8 +3,11 @@ from __future__ import (absolute_import, division,
                         print_function, unicode_literals)
 from builtins import * # noqa: F401
 import copy
-from jmclient import (validate_address, rand_exp_array,
-                      rand_norm_array, rand_pow_array, jm_single)
+import random
+import sys
+
+from .configure import validate_address, jm_single
+from .support import rand_exp_array, rand_norm_array, rand_weighted_choice
 """Utility functions for dealing with Taker schedules.
 
 - get_schedule(filename):
@@ -75,7 +78,7 @@ def get_amount_fractions(power, count):
             break
     return y
 
-def get_tumble_schedule(options, destaddrs):
+def get_tumble_schedule(options, destaddrs, mixdepth_balance_dict):
     """for the general intent and design of the tumbler algo, see the docs in
     joinmarket-org/joinmarket.
     Alterations:
@@ -97,6 +100,29 @@ def get_tumble_schedule(options, destaddrs):
                                options['txcountparams'][1], options['mixdepthcount'])
     txcounts = lower_bounded_int(txcounts, options['mintxcount'])
     tx_list = []
+    ### stage 1 coinjoins, which sweep the entire mixdepth without creating change
+    lowest_initial_filled_mixdepth = sys.maxsize
+    sweep_mixdepths = []
+    for mixdepth, balance in mixdepth_balance_dict.items():
+        if balance > 0:
+            lowest_initial_filled_mixdepth = min(mixdepth,
+                lowest_initial_filled_mixdepth)
+            sweep_mixdepths.append(mixdepth)
+    waits = rand_exp_array(options['timelambda']*options[
+        'stage1_timelambda_increase'], len(sweep_mixdepths))
+    makercounts = rand_norm_array(options['makercountrange'][0],
+        options['makercountrange'][1], len(sweep_mixdepths))
+    makercounts = lower_bounded_int(makercounts, options['minmakercount'])
+    sweep_mixdepths = sorted(sweep_mixdepths)[::-1]
+    for mixdepth, wait, makercount in zip(sweep_mixdepths, waits, makercounts):
+        tx = {'amount_fraction': 0,
+              'wait': round(wait, 2),
+              'srcmixdepth': mixdepth,
+              'makercount': makercount,
+              'destination': 'INTERNAL'
+        }
+        tx_list.append(tx)
+    ### stage 2 coinjoins, which create a number of random-amount coinjoins from each mixdepth
     for m, txcount in enumerate(txcounts):
         if options['mixdepthcount'] - options['addrcount'] <= m and m < \
                 options['mixdepthcount'] - 1:
@@ -117,7 +143,7 @@ def get_tumble_schedule(options, destaddrs):
                                                      makercounts):
             tx = {'amount_fraction': amount_fraction,
                   'wait': round(wait, 2),
-                  'srcmixdepth': m + options['mixdepthsrc'],
+                  'srcmixdepth': lowest_initial_filled_mixdepth + m + options['mixdepthsrc'] + 1,
                   'makercount': makercount,
                   'destination': 'INTERNAL'}
             tx_list.append(tx)
@@ -125,10 +151,10 @@ def get_tumble_schedule(options, destaddrs):
         tx_list[-1]['amount_fraction'] = 0
 
     addrask = options['addrcount'] - len(destaddrs)
-    external_dest_addrs = ['addrask'] * addrask + destaddrs
+    external_dest_addrs = ['addrask'] * addrask + destaddrs[::-1]
     for mix_offset in range(options['addrcount']):
-        srcmix = (options['mixdepthsrc'] + options['mixdepthcount'] -
-            mix_offset - 1)
+        srcmix = (lowest_initial_filled_mixdepth + options['mixdepthsrc']
+            + options['mixdepthcount'] - mix_offset)
         for tx in reversed(tx_list):
             if tx['srcmixdepth'] == srcmix:
                 tx['destination'] = external_dest_addrs[mix_offset]

jmclient/jmclient/taker.py

@@ -188,7 +188,7 @@ class Taker(object):
             # for sweeps to external addresses we need an in-wallet import
             # for the transaction monitor (this will be a no-op for txs to
             # in-wallet addresses).
-            if self.cjamount == 0:
+            if self.cjamount == 0 and self.my_cj_addr != "INTERNAL":
                 self.wallet_service.import_non_wallet_address(self.my_cj_addr)
 
             #if destination is flagged "INTERNAL", choose a destination

jmclient/test/test_schedule.py

@@ -68,6 +68,7 @@ def get_options():
     options.amountpower = 100
     options.timelambda = 0.2
     options.waittime = 10
+    options.stage1_timelambda_increase = 3
     options.mincjamount = 1000000
     options.liquiditywait = 5
     options = vars(options)
@@ -88,7 +89,7 @@ def test_tumble_schedule(destaddrs, txcparams, mixdepthcount):
     options = get_options()
     options['mixdepthcount'] = mixdepthcount
     options['txcountparams'] = txcparams
-    schedule = get_tumble_schedule(options, destaddrs)
+    schedule = get_tumble_schedule(options, destaddrs, {0:1})
     dests = [x[3] for x in schedule]
     assert set(destaddrs).issubset(set(dests))
 
@@ -126,7 +127,7 @@ def test_tumble_tweak(destaddrs, txcparams, mixdepthcount, lastcompleted,
     options['mixdepthcount'] = mixdepthcount
     options['txcountparams'] = txcparams
     options['makercountrange'] = makercountrange
-    schedule = get_tumble_schedule(options, destaddrs)
+    schedule = get_tumble_schedule(options, destaddrs, {0:1})
     dests = [x[3] for x in schedule]
     assert set(destaddrs).issubset(set(dests))
     new_schedule = tweak_tumble_schedule(options, schedule, lastcompleted)

scripts/cli_options.py

@@ -327,6 +327,15 @@ def get_tumbler_parser():
             'Average the number of minutes to wait between transactions. Randomly chosen '
             ' following an exponential distribution, which describes the time between uncorrelated'
             ' events. default=30')
+    parser.add_option(
+            '--stage1-timelambda-increase',
+            type='float',
+            dest='stage1_timelambda_increase',
+            default=3,
+            help=
+            'Stage 1 sweep coinjoins have a longer wait time. This parameter'
+            ' controls by what factor longer is this average wait time compared to stage2 coinjoins'
+            ' which are controlled by `--timelambda`, default=3')
     parser.add_option(
             '-w',
             '--wait-time',

scripts/joinmarket-qt.py

@@ -324,20 +324,24 @@ class SpendTab(QWidget):
         self.spendstate.reset() #trigger callback to 'ready' state
 
     def generateTumbleSchedule(self):
+        if not mainWindow.wallet_service:
+            JMQtMessageBox(self, "Cannot start without a loaded wallet.",
+                           mbtype="crit", title="Error")
+            return
         #needs a set of tumbler options and destination addresses, so needs
         #a wizard
         wizard = ScheduleWizard()
         wizard_return = wizard.exec_()
         if wizard_return == QDialog.Rejected:
             return
-        self.spendstate.loaded_schedule = wizard.get_schedule()
+        self.spendstate.loaded_schedule = wizard.get_schedule(
+            mainWindow.wallet_service.get_balance_by_mixdepth())
         self.spendstate.schedule_name = wizard.get_name()
         self.updateSchedView()
         self.tumbler_options = wizard.opts
         self.tumbler_destaddrs = wizard.get_destaddrs()
         #tumbler may require more mixdepths; update the wallet
-        required_mixdepths = self.tumbler_options['mixdepthsrc'] + \
-            self.tumbler_options['mixdepthcount']
+        required_mixdepths = max([tx[0] for tx in self.spendstate.loaded_schedule])
         if required_mixdepths > jm_single().config.getint("GUI", "max_mix_depth"):
             jm_single().config.set("GUI", "max_mix_depth", str(required_mixdepths))
             #recreate wallet and sync again; needed due to cache.

scripts/qtsupport.py

@@ -624,12 +624,14 @@ class SchFinishPage(QWizardPage):
         layout.setSpacing(4)
 
         results = []
-        sN = ['Makercount sdev', 'Tx count sdev',
+        sN = ['Makercount sdev',
+              'Tx count sdev',
               'Amount power',
               'Minimum maker count',
               'Minimum transaction count',
               'Min coinjoin amount',
-              'wait time']
+              'Response wait time',
+              'Stage 1 transaction wait time increase']
         #Tooltips
         sH = ["Standard deviation of the number of makers to use in each "
               "transaction.",
@@ -639,13 +641,14 @@ class SchFinishPage(QWizardPage):
         "The lowest allowed number of maker counterparties.",
         "The lowest allowed number of transactions in one mixdepth.",
         "The lowest allowed size of any coinjoin, in satoshis.",
-        "The time in seconds to wait for response from counterparties."]
+        "The time in seconds to wait for response from counterparties.",
+        "The factor increase in wait time for stage 1 sweep coinjoins"]
         #types
-        sT = [float, float, float, int, int, int, float]
+        sT = [float, float, float, int, int, int, float, float]
         #constraints
         sMM = [(0.0, 10.0, 2), (0.0, 10.0, 2), (1.0, 10000.0, 1), (2,20),
-               (1, 10), (100000, 100000000), (10.0, 500.0, 2)]
-        sD = ['1.0', '1.0', '100.0', '2', '1', '1000000', '20']
+               (1, 10), (100000, 100000000), (10.0, 500.0, 2), (0, 100, 1)]
+        sD = ['1.0', '1.0', '100.0', '2', '1', '1000000', '20', '3']
         for x in zip(sN, sH, sT, sD, sMM):
             ql = QLabel(x[0])
             ql.setToolTip(x[1])
@@ -669,6 +672,7 @@ class SchFinishPage(QWizardPage):
         self.registerField("mintxcount", results[4][1])
         self.registerField("mincjamount", results[5][1])
         self.registerField("waittime", results[6][1])
+        self.registerField("stage1_timelambda_increase", results[7][1])
 
 class SchIntroPage(QWizardPage):
     def __init__(self, parent):
@@ -715,7 +719,7 @@ class ScheduleWizard(QWizard):
     def get_destaddrs(self):
         return self.destaddrs
 
-    def get_schedule(self):
+    def get_schedule(self, wallet_balance_by_mixdepth):
         self.destaddrs = []
         for i in range(self.page(2).required_addresses):
             daddrstring = str(self.field("destaddr"+str(i)))
@@ -739,13 +743,15 @@ class ScheduleWizard(QWizard):
         self.opts['amountpower'] = float(self.field("amountpower"))
         self.opts['timelambda'] = float(self.field("timelambda"))
         self.opts['waittime'] = float(self.field("waittime"))
+        self.opts["stage1_timelambda_increase"] = float(self.field("stage1_timelambda_increase"))
         self.opts['mincjamount'] = int(self.field("mincjamount"))
         relfeeval = float(self.field("maxrelfee"))
         absfeeval = int(self.field("maxabsfee"))
         self.opts['maxcjfee'] = (relfeeval, absfeeval)
         #needed for Taker to check:
         jm_single().mincjamount = self.opts['mincjamount']
-        return get_tumble_schedule(self.opts, self.destaddrs)
+        return get_tumble_schedule(self.opts, self.destaddrs,
+            wallet_balance_by_mixdepth)
 
 class TumbleRestartWizard(QWizard):
     def __init__(self):

scripts/tumbler.py

@@ -106,7 +106,8 @@ def main():
         tumble_log.info("TUMBLE RESTARTING")
     else:
         #Create a new schedule from scratch
-        schedule = get_tumble_schedule(options, destaddrs)
+        schedule = get_tumble_schedule(options, destaddrs,
+            wallet.get_balance_by_mixdepth())
         tumble_log.info("TUMBLE STARTING")
         with open(os.path.join(logsdir, options['schedulefile']), "wb") as f:
             f.write(schedule_to_text(schedule))