From d3dc9d7bbd8f0464f86ff086f86e8d52e56424ac Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Mon, 25 Jul 2022 22:02:55 +0100 Subject: [PATCH 1/8] Change tumbler algo to cycle and add tests Prior to this commit, the tumbler algorithm assumed that destination mixdepths of INTERNAL transactions were incremented by 1, but the underlying taker code uses (mod maxmixdepth) logic always. This commit takes the decision to make the usage of the wallet "purely" cyclic, that is, not only the Taker object but also the tumbler algorithm now always treat the wallet as a cycle. This is not problematic in a tumbler algorith (or any other schedule generation algorithm), as long as we use the strict rule of "always exit each mixdepth with a sweep", which the tumbler always did and this commit does not change. Also, and importantly, several much more detailed tests of the tumbler schedule generation have been added. --- jmclient/jmclient/schedule.py | 32 ++++++++++++------- jmclient/test/test_schedule.py | 57 +++++++++++++++++++++++++++++++--- scripts/tumbler.py | 2 +- 3 files changed, 73 insertions(+), 18 deletions(-) diff --git a/jmclient/jmclient/schedule.py b/jmclient/jmclient/schedule.py index ba843c7..52ce491 100644 --- a/jmclient/jmclient/schedule.py +++ b/jmclient/jmclient/schedule.py @@ -83,7 +83,8 @@ def get_amount_fractions(count): break return y -def get_tumble_schedule(options, destaddrs, mixdepth_balance_dict): +def get_tumble_schedule(options, destaddrs, mixdepth_balance_dict, + max_mixdepth_in_wallet=4): """for the general intent and design of the tumbler algo, see the docs in joinmarket-org/joinmarket. Alterations: @@ -93,10 +94,14 @@ def get_tumble_schedule(options, destaddrs, mixdepth_balance_dict): any other taker algo; it interprets floats as fractions and integers as satoshis, and zero as sweep (as before). This is a modified version of tumbler.py/generate_tumbler_tx() + Args: + * options - as specified in scripts/tumbler.py and taken from cli_options.py + * destaddrs - a list of valid address strings for the destination of funds + * mixdepth_balance_dict - a dict mapping mixdepths to balances, note zero balances are not included. + * max_mixdepth_in_wallet - this is actually the highest mixdepth in this JM wallet ( + so 4 actually means 5 mixdepths in wallet; we do not consider the possibility of a wallet not + having a 0 mixdepth). The default value 4 is almost always used. """ - #if options['mixdepthsrc'] != 0: - # raise NotImplementedError("Non-zero mixdepth source not supported; " - # "restart the tumbler with --restart instead") def lower_bounded_int(thelist, lowerbound): return [int(l) if int(l) >= lowerbound else lowerbound for l in thelist] @@ -154,7 +159,8 @@ def get_tumble_schedule(options, destaddrs, mixdepth_balance_dict): rounding = rand_weighted_choice(len(weight_prob), weight_prob) + 1 tx = {'amount_fraction': amount_fraction, 'wait': round(wait, 2), - 'srcmixdepth': lowest_initial_filled_mixdepth + m + options['mixdepthsrc'] + 1, + 'srcmixdepth': (lowest_initial_filled_mixdepth + m + options[ + 'mixdepthsrc'] + 1) % (max_mixdepth_in_wallet + 1), 'makercount': makercount, 'destination': 'INTERNAL', 'rounding': rounding @@ -168,7 +174,7 @@ def get_tumble_schedule(options, destaddrs, mixdepth_balance_dict): external_dest_addrs = ['addrask'] * addrask + destaddrs[::-1] for mix_offset in range(options['addrcount']): srcmix = (lowest_initial_filled_mixdepth + options['mixdepthsrc'] - + options['mixdepthcount'] - mix_offset) + + options['mixdepthcount'] - mix_offset) % (max_mixdepth_in_wallet + 1) for tx in reversed(tx_list): if tx['srcmixdepth'] == srcmix: tx['destination'] = external_dest_addrs[mix_offset] @@ -176,12 +182,14 @@ def get_tumble_schedule(options, destaddrs, mixdepth_balance_dict): if mix_offset == 0: # setting last mixdepth to send all to dest tx_list_remove = [] - for tx in tx_list: - if tx['srcmixdepth'] == srcmix: - if tx['destination'] == 'INTERNAL': - tx_list_remove.append(tx) - else: - tx['amount_fraction'] = 0 + ending_mixdepth = tx_list[-1]['srcmixdepth'] + for tx in tx_list[::-1]: + if tx['srcmixdepth'] != ending_mixdepth: + break + if tx['destination'] == 'INTERNAL': + tx_list_remove.append(tx) + else: + tx['amount_fraction'] = 0 [tx_list.remove(t) for t in tx_list_remove] schedule = [] for t in tx_list: diff --git a/jmclient/test/test_schedule.py b/jmclient/test/test_schedule.py index fbd8dfa..3d01cae 100644 --- a/jmclient/test/test_schedule.py +++ b/jmclient/test/test_schedule.py @@ -73,23 +73,70 @@ def get_options(): return options @pytest.mark.parametrize( - "destaddrs, txcparams, mixdepthcount", + "destaddrs, txcparams, mixdepthcount, mixdepthbal", [ + # very simple case (["mzzAYbtPpANxpNVGCVBAhZYzrxyZtoix7i", "mifCWfmygxKhsP3qM3HZi3ZjBEJu7m39h8", - "mnTn9KVQQT9zy9R4E2ZGzWPK4EfcEcV9Y5"], (18,3), 4), + "mnTn9KVQQT9zy9R4E2ZGzWPK4EfcEcV9Y5"], (3,0), 3, + {0:1}), + # with 2 non-empty mixdepths + (["mzzAYbtPpANxpNVGCVBAhZYzrxyZtoix7i", + "mifCWfmygxKhsP3qM3HZi3ZjBEJu7m39h8", + "mnTn9KVQQT9zy9R4E2ZGzWPK4EfcEcV9Y5"], (7,0), 3, + {2:1, 3: 1}), #intended to trigger txcount=1 bump to 2 (["mzzAYbtPpANxpNVGCVBAhZYzrxyZtoix7i", "mifCWfmygxKhsP3qM3HZi3ZjBEJu7m39h8", - "mnTn9KVQQT9zy9R4E2ZGzWPK4EfcEcV9Y5"], (3,2), 80), + "mnTn9KVQQT9zy9R4E2ZGzWPK4EfcEcV9Y5"], (3,2), 8, + {2:1, 3: 1}), ]) -def test_tumble_schedule(destaddrs, txcparams, mixdepthcount): +def test_tumble_schedule(destaddrs, txcparams, mixdepthcount, mixdepthbal): + # note that these tests are currently only leaving the default + # value for the final argument to get_tumble_schedule, i.e. 4, + # and will fail if this is changed: + wallet_total_mixdepths = 5 options = get_options() options['mixdepthcount'] = mixdepthcount options['txcountparams'] = txcparams - schedule = get_tumble_schedule(options, destaddrs, {0:1}) + schedule = get_tumble_schedule(options, destaddrs, mixdepthbal) + # first, examine the destination addresses; all the requested + # ones should be in the list, and all the others should be one + # of the two standard 'code' alternatives. dests = [x[3] for x in schedule] assert set(destaddrs).issubset(set(dests)) + nondestaddrs = [x[3] for x in schedule if x[3] not in destaddrs] + assert all([x in ["INTERNAL", "addrask"] for x in nondestaddrs]) + # second: check that the total number of transactions is larger + # than the minimum it could be and smaller than the max; + # the last term accounts for the Phase 1 sweeps + assert len(schedule) >= (mixdepthcount - 1) * ( + txcparams[0] - txcparams[1]) + 1 + len(mixdepthbal.items()) + assert len(schedule) <= (mixdepthcount - 1) * ( + txcparams[0] + txcparams[1]) + 1 + len(mixdepthbal.items()) + # check that the source mixdepths for the phase 1 transactions are the + # expected, and that they are all sweeps: + for i, s in enumerate(schedule[:len(mixdepthbal)]): + assert s[1] == 0 + assert s[0] in mixdepthbal.keys() + # check that the list of created transactions in Phase 2 only + # progresses forward, one mixdepth at a time: + for first, second in zip(schedule[len(mixdepthbal):-1], + schedule[len(mixdepthbal) + 1:]): + assert (second[0] - first[0]) % wallet_total_mixdepths in [1, 0] + # check that the amount fractions are always total < 1 + last_s = [] + for s in schedule: + if last_s == []: + last_s = s + total_amt = 0 + continue + if s[0] == last_s[0]: + total_amt += s[1] + else: + assert total_amt < 1 + total_amt = 0 + last_s = s @pytest.mark.parametrize( "destaddrs, txcparams, mixdepthcount, lastcompleted, makercountrange", diff --git a/scripts/tumbler.py b/scripts/tumbler.py index 0cf2d71..825c44d 100755 --- a/scripts/tumbler.py +++ b/scripts/tumbler.py @@ -110,7 +110,7 @@ def main(): else: #Create a new schedule from scratch schedule = get_tumble_schedule(options, destaddrs, - wallet.get_balance_by_mixdepth()) + wallet.get_balance_by_mixdepth(), wallet_service.mixdepth) tumble_log.info("TUMBLE STARTING") with open(os.path.join(logsdir, options['schedulefile']), "wb") as f: f.write(schedule_to_text(schedule)) From c0df86819b4d28cc686b2d58aea942b99c7f3cd5 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Tue, 26 Jul 2022 12:54:01 +0100 Subject: [PATCH 2/8] fix tumbler tweak test to account for mixdepth cycling --- jmclient/test/test_schedule.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/jmclient/test/test_schedule.py b/jmclient/test/test_schedule.py index 3d01cae..9f54eb5 100644 --- a/jmclient/test/test_schedule.py +++ b/jmclient/test/test_schedule.py @@ -178,14 +178,12 @@ def test_tumble_tweak(destaddrs, txcparams, mixdepthcount, lastcompleted, new_schedule = tweak_tumble_schedule(options, schedule, lastcompleted) #sanity check: each amount fraction list should add up to near 1.0, #so some is left over for sweep + tally = 0 + current_mixdepth = new_schedule[0][0] for i in range(mixdepthcount): - entries = [x for x in new_schedule if x[0] == i] - total_frac_for_mixdepth = sum([x[1] for x in entries]) - #TODO spurious failure is possible here, not an ideal check - print('got total frac for mixdepth: ', str(total_frac_for_mixdepth)) - assert total_frac_for_mixdepth < 0.999 - from pprint import pformat - print("here is the new schedule: ") - print(pformat(new_schedule)) - print("and old:") - print(pformat(schedule)) + if new_schedule[i][0] != current_mixdepth: + print('got total frac for mixdepth: ', tally) + #TODO spurious failure is possible here, not an ideal check + assert tally < 0.999 + tally = 0 + tally += new_schedule[i][1] From e5ed7f2dcf875a0efad2f092ab3435bd46effb5e Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Tue, 26 Jul 2022 15:54:57 +0100 Subject: [PATCH 3/8] remove erroneous txcount test and check destaddrs better --- jmclient/test/test_schedule.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/jmclient/test/test_schedule.py b/jmclient/test/test_schedule.py index 9f54eb5..7cb7c2f 100644 --- a/jmclient/test/test_schedule.py +++ b/jmclient/test/test_schedule.py @@ -104,23 +104,20 @@ def test_tumble_schedule(destaddrs, txcparams, mixdepthcount, mixdepthbal): # ones should be in the list, and all the others should be one # of the two standard 'code' alternatives. dests = [x[3] for x in schedule] - assert set(destaddrs).issubset(set(dests)) + dests = [x for x in dests if x not in ["INTERNAL", "addrask"]] + assert len(dests) == len(destaddrs) + assert set(destaddrs) == set(dests) nondestaddrs = [x[3] for x in schedule if x[3] not in destaddrs] assert all([x in ["INTERNAL", "addrask"] for x in nondestaddrs]) - # second: check that the total number of transactions is larger - # than the minimum it could be and smaller than the max; - # the last term accounts for the Phase 1 sweeps - assert len(schedule) >= (mixdepthcount - 1) * ( - txcparams[0] - txcparams[1]) + 1 + len(mixdepthbal.items()) - assert len(schedule) <= (mixdepthcount - 1) * ( - txcparams[0] + txcparams[1]) + 1 + len(mixdepthbal.items()) # check that the source mixdepths for the phase 1 transactions are the # expected, and that they are all sweeps: for i, s in enumerate(schedule[:len(mixdepthbal)]): assert s[1] == 0 assert s[0] in mixdepthbal.keys() # check that the list of created transactions in Phase 2 only - # progresses forward, one mixdepth at a time: + # progresses forward, one mixdepth at a time. + # Note that due to the use of sdev calculation, we cannot check that + # the number of transactions per mixdepth is anything in particular. for first, second in zip(schedule[len(mixdepthbal):-1], schedule[len(mixdepthbal) + 1:]): assert (second[0] - first[0]) % wallet_total_mixdepths in [1, 0] From 7ffc74762ea56368f7b797c680c639f4b04ac6d7 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Tue, 26 Jul 2022 23:09:55 +0100 Subject: [PATCH 4/8] fix lowest filled mixdepth algo --- jmclient/jmclient/schedule.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/jmclient/jmclient/schedule.py b/jmclient/jmclient/schedule.py index 52ce491..5e033d9 100644 --- a/jmclient/jmclient/schedule.py +++ b/jmclient/jmclient/schedule.py @@ -1,6 +1,5 @@ import copy import random -import sys from .configure import validate_address, jm_single from .support import rand_exp_array, rand_norm_array, rand_weighted_choice @@ -111,12 +110,9 @@ def get_tumble_schedule(options, destaddrs, mixdepth_balance_dict, txcounts = lower_bounded_int(txcounts, options['mintxcount']) tx_list = [] ### stage 1 coinjoins, which sweep the entire mixdepth without creating change - lowest_initial_filled_mixdepth = sys.maxsize sweep_mixdepths = [] for mixdepth, balance in mixdepth_balance_dict.items(): if balance > 0: - lowest_initial_filled_mixdepth = min(mixdepth, - lowest_initial_filled_mixdepth) sweep_mixdepths.append(mixdepth) waits = rand_exp_array(options['timelambda']*options[ 'stage1_timelambda_increase'], len(sweep_mixdepths)) @@ -124,7 +120,14 @@ def get_tumble_schedule(options, destaddrs, mixdepth_balance_dict, options['makercountrange'][1], len(sweep_mixdepths)) makercounts = lower_bounded_int(makercounts, options['minmakercount']) sweep_mixdepths = sorted(sweep_mixdepths)[::-1] + nonempty_mixdepths = {} for mixdepth, wait, makercount in zip(sweep_mixdepths, waits, makercounts): + # to know which mixdepths contain coins at the end of phase 1, we + # can't simply check each filled mixdepth and add 1, because the sequencing + # of these joins may sweep up coins that have already been swept. + # so we keep track with a binary flag for every mixdepth during this sequence. + nonempty_mixdepths[mixdepth] = 0 + nonempty_mixdepths[(mixdepth + 1) % (max_mixdepth_in_wallet + 1)] = 1 tx = {'amount_fraction': 0, 'wait': round(wait, 2), 'srcmixdepth': mixdepth, @@ -133,6 +136,7 @@ def get_tumble_schedule(options, destaddrs, mixdepth_balance_dict, 'rounding': NO_ROUNDING } tx_list.append(tx) + lowest_nonempty_mixdepth = min([x for x, y in nonempty_mixdepths.items() if y == 1]) ### stage 2 coinjoins, which create a number of random-amount coinjoins from each mixdepth for m, txcount in enumerate(txcounts): if options['mixdepthcount'] - options['addrcount'] <= m and m < \ @@ -159,8 +163,8 @@ def get_tumble_schedule(options, destaddrs, mixdepth_balance_dict, rounding = rand_weighted_choice(len(weight_prob), weight_prob) + 1 tx = {'amount_fraction': amount_fraction, 'wait': round(wait, 2), - 'srcmixdepth': (lowest_initial_filled_mixdepth + m + options[ - 'mixdepthsrc'] + 1) % (max_mixdepth_in_wallet + 1), + 'srcmixdepth': (lowest_nonempty_mixdepth + m + options[ + 'mixdepthsrc']) % (max_mixdepth_in_wallet + 1), 'makercount': makercount, 'destination': 'INTERNAL', 'rounding': rounding @@ -169,12 +173,11 @@ def get_tumble_schedule(options, destaddrs, mixdepth_balance_dict, #reset the final amt_frac to zero, as it's the last one for this mixdepth: tx_list[-1]['amount_fraction'] = 0 tx_list[-1]['rounding'] = NO_ROUNDING - addrask = options['addrcount'] - len(destaddrs) external_dest_addrs = ['addrask'] * addrask + destaddrs[::-1] for mix_offset in range(options['addrcount']): - srcmix = (lowest_initial_filled_mixdepth + options['mixdepthsrc'] - + options['mixdepthcount'] - mix_offset) % (max_mixdepth_in_wallet + 1) + srcmix = (lowest_nonempty_mixdepth + options['mixdepthsrc'] + + options['mixdepthcount'] - mix_offset - 1) % (max_mixdepth_in_wallet + 1) for tx in reversed(tx_list): if tx['srcmixdepth'] == srcmix: tx['destination'] = external_dest_addrs[mix_offset] From d0bf888971aa7423e5a2dee3a4c90f035e68b694 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Fri, 29 Jul 2022 13:01:59 +0100 Subject: [PATCH 5/8] Remove mixdepthsrc from options, more tests --- jmclient/jmclient/cli_options.py | 12 +----------- jmclient/jmclient/schedule.py | 25 ++++++++++++++---------- jmclient/test/test_schedule.py | 9 ++++++++- scripts/tumbler.py | 33 ++++++++++++++++++++++++-------- 4 files changed, 49 insertions(+), 30 deletions(-) diff --git a/jmclient/jmclient/cli_options.py b/jmclient/jmclient/cli_options.py index 13a6cc6..8583d5f 100644 --- a/jmclient/jmclient/cli_options.py +++ b/jmclient/jmclient/cli_options.py @@ -275,14 +275,6 @@ def get_tumbler_parser(): ' be configured to ask for more address mid-run, giving the user' ' a chance to click `Generate New Deposit Address` on whatever service' ' they are using.') - parser.add_option( - '-m', - '--mixdepthsource', - type='int', - dest='mixdepthsrc', - help= - 'Mixing depth to start tumble process from. default=0.', - default=0) parser.add_option('--restart', action='store_true', dest='restart', @@ -398,9 +390,7 @@ def get_tumbler_parser(): type='int', dest='amtmixdepths', help='number of mixdepths ever used in wallet, ' - 'only to be used if mixdepths higher than ' - 'mixdepthsrc + number of mixdepths to tumble ' - 'have been used.', + 'will soon be deprecated.', default=-1) parser.add_option( '--rounding-chance', diff --git a/jmclient/jmclient/schedule.py b/jmclient/jmclient/schedule.py index 5e033d9..0503d50 100644 --- a/jmclient/jmclient/schedule.py +++ b/jmclient/jmclient/schedule.py @@ -17,6 +17,12 @@ from .support import rand_exp_array, rand_norm_array, rand_weighted_choice NO_ROUNDING = 16 #max btc significant figures not including LN +class ScheduleGenerationError(Exception): + pass + +class ScheduleGenerationErrorNoFunds(ScheduleGenerationError): + pass + def get_schedule(filename): with open(filename, "rb") as f: schedule = [] @@ -84,15 +90,11 @@ def get_amount_fractions(count): def get_tumble_schedule(options, destaddrs, mixdepth_balance_dict, max_mixdepth_in_wallet=4): - """for the general intent and design of the tumbler algo, see the docs in - joinmarket-org/joinmarket. - Alterations: - Donation removed for now. + """ Default final setting for "amount_fraction" is zero, for each mixdepth. This is because we now use a general "schedule" syntax for both tumbler and any other taker algo; it interprets floats as fractions and integers as satoshis, and zero as sweep (as before). - This is a modified version of tumbler.py/generate_tumbler_tx() Args: * options - as specified in scripts/tumbler.py and taken from cli_options.py * destaddrs - a list of valid address strings for the destination of funds @@ -136,7 +138,10 @@ def get_tumble_schedule(options, destaddrs, mixdepth_balance_dict, 'rounding': NO_ROUNDING } tx_list.append(tx) - lowest_nonempty_mixdepth = min([x for x, y in nonempty_mixdepths.items() if y == 1]) + try: + lowest_nonempty_mixdepth = min([x for x, y in nonempty_mixdepths.items() if y == 1]) + except ValueError: + raise ScheduleGenerationErrorNoFunds ### stage 2 coinjoins, which create a number of random-amount coinjoins from each mixdepth for m, txcount in enumerate(txcounts): if options['mixdepthcount'] - options['addrcount'] <= m and m < \ @@ -163,8 +168,8 @@ def get_tumble_schedule(options, destaddrs, mixdepth_balance_dict, rounding = rand_weighted_choice(len(weight_prob), weight_prob) + 1 tx = {'amount_fraction': amount_fraction, 'wait': round(wait, 2), - 'srcmixdepth': (lowest_nonempty_mixdepth + m + options[ - 'mixdepthsrc']) % (max_mixdepth_in_wallet + 1), + 'srcmixdepth': (lowest_nonempty_mixdepth + m) % ( + max_mixdepth_in_wallet + 1), 'makercount': makercount, 'destination': 'INTERNAL', 'rounding': rounding @@ -176,8 +181,8 @@ def get_tumble_schedule(options, destaddrs, mixdepth_balance_dict, addrask = options['addrcount'] - len(destaddrs) external_dest_addrs = ['addrask'] * addrask + destaddrs[::-1] for mix_offset in range(options['addrcount']): - srcmix = (lowest_nonempty_mixdepth + options['mixdepthsrc'] - + options['mixdepthcount'] - mix_offset - 1) % (max_mixdepth_in_wallet + 1) + srcmix = (lowest_nonempty_mixdepth + options['mixdepthcount'] + - mix_offset - 1) % (max_mixdepth_in_wallet + 1) for tx in reversed(tx_list): if tx['srcmixdepth'] == srcmix: tx['destination'] = external_dest_addrs[mix_offset] diff --git a/jmclient/test/test_schedule.py b/jmclient/test/test_schedule.py index 7cb7c2f..8b873eb 100644 --- a/jmclient/test/test_schedule.py +++ b/jmclient/test/test_schedule.py @@ -54,7 +54,6 @@ class Options(object): def get_options(): options = Options() - options.mixdepthsrc = 0 options.mixdepthcount = 4 options.txcountparams = (18, 3) options.minmakercount = 2 @@ -90,6 +89,13 @@ def get_options(): "mifCWfmygxKhsP3qM3HZi3ZjBEJu7m39h8", "mnTn9KVQQT9zy9R4E2ZGzWPK4EfcEcV9Y5"], (3,2), 8, {2:1, 3: 1}), + #slightly larger version + (["mzzAYbtPpANxpNVGCVBAhZYzrxyZtoix7i", + "mifCWfmygxKhsP3qM3HZi3ZjBEJu7m39h8", + "mnTn9KVQQT9zy9R4E2ZGzWPK4EfcEcV9Y5", + "bcrt1qcnv26w889eum5sekz5h8we45rxnr4sj5k08phv", + "bcrt1qgs0t239gj2kqgnsrvetvsv2qdva8y3j74cta4d"], (4,3), 8, + {0:2, 1: 1, 3: 1, 4: 1}), ]) def test_tumble_schedule(destaddrs, txcparams, mixdepthcount, mixdepthbal): # note that these tests are currently only leaving the default @@ -97,6 +103,7 @@ def test_tumble_schedule(destaddrs, txcparams, mixdepthcount, mixdepthbal): # and will fail if this is changed: wallet_total_mixdepths = 5 options = get_options() + options['addrcount'] = len(destaddrs) options['mixdepthcount'] = mixdepthcount options['txcountparams'] = txcparams schedule = get_tumble_schedule(options, destaddrs, mixdepthbal) diff --git a/scripts/tumbler.py b/scripts/tumbler.py index 825c44d..ef79a05 100755 --- a/scripts/tumbler.py +++ b/scripts/tumbler.py @@ -12,6 +12,8 @@ from jmclient import Taker, load_program_config, get_schedule,\ get_tumble_log, tumbler_taker_finished_update, check_regtest, \ tumbler_filter_orders_callback, validate_address, get_tumbler_parser, \ get_max_cj_fee_values +from jmclient.wallet_utils import DEFAULT_MIXDEPTH +from jmclient.schedule import ScheduleGenerationErrorNoFunds from jmbase.support import get_log, jmprint, EXIT_SUCCESS, \ EXIT_FAILURE, EXIT_ARGERROR @@ -38,11 +40,17 @@ def main(): #Load the wallet wallet_name = args[0] - max_mix_depth = options['mixdepthsrc'] + options['mixdepthcount'] - if options['amtmixdepths'] > max_mix_depth: + # as of #1324 the concept of a max_mix_depth distinct from + # the normal wallet value (4) no longer applies, since the + # tumbler cycles; but we keep the `amtmixdepths` option for now, + # deprecating it later. + if options['amtmixdepths'] > DEFAULT_MIXDEPTH: max_mix_depth = options['amtmixdepths'] + else: + max_mix_depth = DEFAULT_MIXDEPTH wallet_path = get_wallet_path(wallet_name, None) - wallet = open_test_wallet_maybe(wallet_path, wallet_name, max_mix_depth, wallet_password_stdin=options_org.wallet_password_stdin) + wallet = open_test_wallet_maybe(wallet_path, wallet_name, max_mix_depth, + wallet_password_stdin=options_org.wallet_password_stdin) wallet_service = WalletService(wallet) if wallet_service.rpc_error: sys.exit(EXIT_FAILURE) @@ -109,8 +117,12 @@ def main(): tumble_log.info("TUMBLE RESTARTING") else: #Create a new schedule from scratch - schedule = get_tumble_schedule(options, destaddrs, - wallet.get_balance_by_mixdepth(), wallet_service.mixdepth) + try: + schedule = get_tumble_schedule(options, destaddrs, + wallet.get_balance_by_mixdepth(), wallet_service.mixdepth) + except ScheduleGenerationErrorNoFunds: + jmprint("No funds in wallet to tumble.", "error") + sys.exit(EXIT_FAILURE) tumble_log.info("TUMBLE STARTING") with open(os.path.join(logsdir, options['schedulefile']), "wb") as f: f.write(schedule_to_text(schedule)) @@ -133,10 +145,15 @@ def main(): involved_parties = len(schedule) # own participation in each CJ for item in schedule: involved_parties += item[2] # number of total tumble counterparties + # calculating total coins that will be included in the tumble; + # in almost all cases all coins (unfrozen) in wallet will be tumbled, + # though it's technically possible with a very small mixdepthcount, to start + # at say m0, and only go through to 2 or 3, such that coins in 4 are untouched + # in phase 2 (after having been swept in phase 1). + used_mixdepths = set() + [used_mixdepths.add(x[0]) for x in schedule] total_tumble_amount = int(0) - max_mix_to_tumble = min(options['mixdepthsrc']+options['mixdepthcount'], \ - max_mix_depth) - for i in range(options['mixdepthsrc'], max_mix_to_tumble): + for i in used_mixdepths: total_tumble_amount += wallet_service.get_balance_by_mixdepth()[i] if total_tumble_amount == 0: raise ValueError("No confirmed coins in the selected mixdepth(s). Quitting") From 75c444e9b24adc1a5f1dcf277d1d855457ec4fbf Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Fri, 29 Jul 2022 17:53:43 +0100 Subject: [PATCH 6/8] Update wallet_rpc for new tumbler code --- jmclient/jmclient/__init__.py | 5 +++-- jmclient/jmclient/taker_utils.py | 16 ++++++++++++++++ jmclient/jmclient/wallet_rpc.py | 24 ++++++------------------ scripts/tumbler.py | 20 ++++++-------------- 4 files changed, 31 insertions(+), 34 deletions(-) diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index a739633..567be22 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -45,11 +45,12 @@ from .output import generate_podle_error_string, fmt_utxos, fmt_utxo,\ sweep_custom_change_warning from .schedule import (get_schedule, get_tumble_schedule, schedule_to_text, tweak_tumble_schedule, human_readable_schedule_entry, - schedule_to_text, NO_ROUNDING) + schedule_to_text, NO_ROUNDING, ScheduleGenerationErrorNoFunds) from .commitment_utils import get_utxo_info, validate_utxo_data, quit from .taker_utils import (tumbler_taker_finished_update, restart_waiter, restart_wait, get_tumble_log, direct_send, - tumbler_filter_orders_callback, direct_send) + tumbler_filter_orders_callback, direct_send, + get_total_tumble_amount) from .cli_options import (add_base_options, add_common_options, get_tumbler_parser, get_max_cj_fee_values, check_regtest, get_sendpayment_parser, diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index a8dfb72..a89fc60 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -229,6 +229,22 @@ def get_tumble_log(logsdir): tumble_log.addHandler(fileHandler) return tumble_log +def get_total_tumble_amount(mixdepth_balance_dict, schedule): + # calculating total coins that will be included in a tumble; + # in almost all cases all coins (unfrozen) in wallet will be tumbled, + # though it's technically possible with a very small mixdepthcount, to start + # at say m0, and only go through to 2 or 3, such that coins in 4 are untouched + # in phase 2 (after having been swept in phase 1). + used_mixdepths = set() + [used_mixdepths.add(x[0]) for x in schedule] + total_tumble_amount = int(0) + for i in used_mixdepths: + total_tumble_amount += mixdepth_balance_dict[i] + # Note; we assert since callers will have called `get_tumble_schedule`, + # which will already have thrown if no funds, so this would be a logic error. + assert total_tumble_amount > 0, "no coins to tumble." + return total_tumble_amount + def restart_wait(txid): """ Returns true only if the transaction txid is seen in the wallet, and confirmed (it must be an in-wallet transaction since it always diff --git a/jmclient/jmclient/wallet_rpc.py b/jmclient/jmclient/wallet_rpc.py index 4add2bd..8116cd6 100644 --- a/jmclient/jmclient/wallet_rpc.py +++ b/jmclient/jmclient/wallet_rpc.py @@ -25,7 +25,8 @@ from jmclient import Taker, jm_single, \ NotEnoughFundsException, get_tumble_log, get_tumble_schedule, \ get_schedule, get_tumbler_parser, schedule_to_text, \ tumbler_filter_orders_callback, tumbler_taker_finished_update, \ - validate_address, FidelityBondMixin + validate_address, FidelityBondMixin, \ + ScheduleGenerationErrorNoFunds from jmbase.support import get_log, utxostr_to_utxo jlog = get_log() @@ -1177,28 +1178,15 @@ class JMWalletDaemon(Service): jm_single().mincjamount = tumbler_options['mincjamount'] - # -- Check wallet balance ------------------------------------------ - - max_mix_depth = tumbler_options['mixdepthsrc'] + tumbler_options['mixdepthcount'] - - if tumbler_options['amtmixdepths'] > max_mix_depth: - max_mix_depth = tumbler_options['amtmixdepths'] - - max_mix_to_tumble = min(tumbler_options['mixdepthsrc'] + tumbler_options['mixdepthcount'], max_mix_depth) - - total_tumble_amount = int(0) - for i in range(tumbler_options['mixdepthsrc'], max_mix_to_tumble): - total_tumble_amount += self.services["wallet"].get_balance_by_mixdepth(verbose=False, minconfs=1)[i] - - if total_tumble_amount == 0: - raise NotEnoughCoinsForTumbler() - # -- Schedule generation ------------------------------------------- # Always generates a new schedule. No restart support for now. - schedule = get_tumble_schedule(tumbler_options, + try: + schedule = get_tumble_schedule(tumbler_options, destaddrs, self.services["wallet"].get_balance_by_mixdepth()) + except ScheduleGenerationErrorNoFunds: + raise NotEnoughCoinsForTumbler() logsdir = os.path.join(os.path.dirname(jm_single().config_location), "logs") diff --git a/scripts/tumbler.py b/scripts/tumbler.py index ef79a05..6340cb3 100755 --- a/scripts/tumbler.py +++ b/scripts/tumbler.py @@ -11,9 +11,9 @@ from jmclient import Taker, load_program_config, get_schedule,\ schedule_to_text, estimate_tx_fee, restart_waiter, WalletService,\ get_tumble_log, tumbler_taker_finished_update, check_regtest, \ tumbler_filter_orders_callback, validate_address, get_tumbler_parser, \ - get_max_cj_fee_values + get_max_cj_fee_values, get_total_tumble_amount, ScheduleGenerationErrorNoFunds from jmclient.wallet_utils import DEFAULT_MIXDEPTH -from jmclient.schedule import ScheduleGenerationErrorNoFunds + from jmbase.support import get_log, jmprint, EXIT_SUCCESS, \ EXIT_FAILURE, EXIT_ARGERROR @@ -145,18 +145,10 @@ def main(): involved_parties = len(schedule) # own participation in each CJ for item in schedule: involved_parties += item[2] # number of total tumble counterparties - # calculating total coins that will be included in the tumble; - # in almost all cases all coins (unfrozen) in wallet will be tumbled, - # though it's technically possible with a very small mixdepthcount, to start - # at say m0, and only go through to 2 or 3, such that coins in 4 are untouched - # in phase 2 (after having been swept in phase 1). - used_mixdepths = set() - [used_mixdepths.add(x[0]) for x in schedule] - total_tumble_amount = int(0) - for i in used_mixdepths: - total_tumble_amount += wallet_service.get_balance_by_mixdepth()[i] - if total_tumble_amount == 0: - raise ValueError("No confirmed coins in the selected mixdepth(s). Quitting") + + total_tumble_amount = get_total_tumble_amount( + wallet.get_balance_by_mixdepth(), schedule) + exp_tx_fees_ratio = (involved_parties * fee_per_cp_guess) \ / total_tumble_amount if exp_tx_fees_ratio > 0.05: From 524cbda8861dee883af7fa74ab834a4d06f3552a Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sun, 31 Jul 2022 12:29:01 +0100 Subject: [PATCH 7/8] update Qt for new tumbler algo --- scripts/joinmarket-qt.py | 14 +++++++++++--- scripts/qtsupport.py | 26 +++++++++++--------------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/scripts/joinmarket-qt.py b/scripts/joinmarket-qt.py index 416bce9..7553d55 100755 --- a/scripts/joinmarket-qt.py +++ b/scripts/joinmarket-qt.py @@ -73,7 +73,8 @@ from jmclient import load_program_config, get_network, update_persist_config,\ parse_payjoin_setup, send_payjoin, JMBIP78ReceiverManager, \ detect_script_type, general_custom_change_warning, \ nonwallet_custom_change_warning, sweep_custom_change_warning, EngineError,\ - TYPE_P2WPKH, check_and_start_tor, is_extended_public_key + TYPE_P2WPKH, check_and_start_tor, is_extended_public_key, \ + ScheduleGenerationErrorNoFunds from jmclient.wallet import BaseWallet from qtsupport import ScheduleWizard, TumbleRestartWizard, config_tips,\ @@ -413,8 +414,15 @@ class SpendTab(QWidget): wizard_return = wizard.exec_() if wizard_return == QDialog.Rejected: return - self.spendstate.loaded_schedule = wizard.get_schedule( - mainWindow.wallet_service.get_balance_by_mixdepth()) + try: + self.spendstate.loaded_schedule = wizard.get_schedule( + mainWindow.wallet_service.get_balance_by_mixdepth(), + mainWindow.wallet_service.mixdepth) + except ScheduleGenerationErrorNoFunds: + JMQtMessageBox(self, + "Failed to start tumbler; no funds available.", + title="Tumbler start failed.") + return self.spendstate.schedule_name = wizard.get_name() self.updateSchedView() self.tumbler_options = wizard.opts diff --git a/scripts/qtsupport.py b/scripts/qtsupport.py index debcc5f..c900178 100644 --- a/scripts/qtsupport.py +++ b/scripts/qtsupport.py @@ -641,13 +641,12 @@ class SchDynamicPage1(QWizardPage): self.setTitle("Tumble schedule generation") self.setSubTitle("Set parameters for the sequence of transactions in the tumble.") results = [] - sN = ['Starting mixdepth', 'Average number of counterparties', + sN = ['Average number of counterparties', 'How many mixdepths to tumble through', 'Average wait time between transactions, in minutes', 'Average number of transactions per mixdepth'] #Tooltips - sH = ["The starting mixdepth can be decided from the Wallet tab; it must\n" - "have coins in it, but it's OK if some coins are in other mixdepths.", + sH = [ "How many other participants are in each coinjoin, on average; but\n" "each individual coinjoin will have a number that's varied according to\n" "settings on the next page", @@ -657,11 +656,10 @@ class SchDynamicPage1(QWizardPage): "varied randomly.", "Will be varied randomly, see advanced settings next page"] #types - sT = [int, int, int, float, int] + sT = [int, int, float, int] #constraints - sMM = [(0, jm_single().config.getint("GUI", "max_mix_depth") - 1), (3, 20), - (2, 7), (0.00000001, 100.0, 8), (2, 10)] - sD = ['0', '9', '4', '60.0', '2'] + sMM = [(3, 20), (2, 7), (0.00000001, 100.0, 8), (2, 10)] + sD = ['9', '4', '60.0', '2'] for x in zip(sN, sH, sT, sD, sMM): ql = QLabel(x[0]) ql.setToolTip(x[1]) @@ -677,11 +675,10 @@ class SchDynamicPage1(QWizardPage): layout.addWidget(x[0], i + 1, 0) layout.addWidget(x[1], i + 1, 1, 1, 2) self.setLayout(layout) - self.registerField("mixdepthsrc", results[0][1]) - self.registerField("makercount", results[1][1]) - self.registerField("mixdepthcount", results[2][1]) - self.registerField("timelambda", results[3][1]) - self.registerField("txcountparams", results[4][1]) + self.registerField("makercount", results[0][1]) + self.registerField("mixdepthcount", results[1][1]) + self.registerField("timelambda", results[2][1]) + self.registerField("txcountparams", results[3][1]) class SchDynamicPage2(QWizardPage): @@ -834,7 +831,7 @@ class ScheduleWizard(QWizard): def get_destaddrs(self): return self.destaddrs - def get_schedule(self, wallet_balance_by_mixdepth): + def get_schedule(self, wallet_balance_by_mixdepth, max_mixdepth_in_wallet): self.destaddrs = [] for i in range(self.page(2).required_addresses): daddrstring = str(self.field("destaddr"+str(i))) @@ -845,7 +842,6 @@ class ScheduleWizard(QWizard): title='Error') return None self.opts = {} - self.opts['mixdepthsrc'] = int(self.field("mixdepthsrc")) self.opts['mixdepthcount'] = int(self.field("mixdepthcount")) self.opts['txfee'] = -1 self.opts['addrcount'] = len(self.destaddrs) @@ -864,7 +860,7 @@ class ScheduleWizard(QWizard): self.opts['rounding_sigfig_weights'] = tuple([int(self.field("rounding_sigfig_weight_" + str(i+1))) for i in range(5)]) jm_single().mincjamount = self.opts['mincjamount'] return get_tumble_schedule(self.opts, self.destaddrs, - wallet_balance_by_mixdepth) + wallet_balance_by_mixdepth, max_mixdepth_in_wallet) class TumbleRestartWizard(QWizard): def __init__(self): From b4e4f2a37a71f0fb33d0a66ac06a5a4173ba1671 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sun, 31 Jul 2022 14:10:58 +0100 Subject: [PATCH 8/8] susbstantial rewrite of tumblerguide.md --- docs/tumblerguide.md | 182 +++++++++++++++++++++++++++---------------- 1 file changed, 116 insertions(+), 66 deletions(-) diff --git a/docs/tumblerguide.md b/docs/tumblerguide.md index 41dc483..adac0b6 100644 --- a/docs/tumblerguide.md +++ b/docs/tumblerguide.md @@ -1,12 +1,14 @@ ## Running the tumbler. -The information in this guide is supplemental to that in the usage guide. Note that the tumbler can be run as a script: +Note that the tumbler can be run as a script: ``` (jmvenv)a@~/joinmarket-clientserver/scripts$ python tumbler.py --help ``` -or from the JoinmarketQt app in the "Multiple Joins" tab (see [the guide](JOINMARKET-QT-GUIDE.md)). +or from the JoinmarketQt app in the "Multiple Joins" tab (see [the guide](JOINMARKET-QT-GUIDE.md)), + +or using the RPC-API via a webapp like [JAM](https://github.com/joinmarket-webui/joinmarket-webui). # Contents @@ -14,33 +16,30 @@ or from the JoinmarketQt app in the "Multiple Joins" tab (see [the guide](JOINMA a. [A note on fees](#a-note-on-fees) -2. [Basic examples](#basic) - - a. [Example: Tumbling into your wallet after buying from an exchange to improve privacy](#example-tumbling-into-your-wallet-after-buying-from-an-exchange-to-improve-privacy) +2. [How it works - the algorithm](#algo) - b. [Example: Tumbling from your wallet into an exchange](#example-tumbling-from-your-wallet-into-an-exchange) + a. [Example 1](#example1) -3. [Schedules (transaction lists)](#schedules) + b. [Example 2](#example2) -4. [Tumbler schedule and logging](#logging) +4. [Schedules (transaction lists)](#schedules) -5. [Interpreting key console output](#interpreting) +5. [Tumbler schedule and logging](#logging) -6. [Tweaking the schedule](#tweaking) +6. [Interpreting key console output](#interpreting) -7. [How often do retries occur?](#how-often-do-retries-occur) +7. [Tweaking the schedule](#tweaking) -8. [Restarts](#restarts) -9. [Possible failure vectors](#failure-vectors) +8. [Possible failure vectors](#failure-vectors) ## Introduction to the tumbler -Tumbler is a JoinMarket bot which attempts to completely break the link between addresses. It is used to restore privacy where it has been damaged. It creates many many coinjoins to bounce coins around in different amounts and times. +Tumbler is a JoinMarket bot which attempts to completely break the link between addresses. It is used to restore privacy where it has been damaged. It creates many many coinjoins to bounce coins around in different amounts and times. The purpose of this is to help human beings preserve their security and dignity; the reason for doing this is the same as the reason for not broadcasting live camera feeds of every room inside your house. -Examples of users might be people who bought bitcoins with a very privacy-invading method, such as buying from an exchange, and wish to have privacy in all their purchases again. Some bitcoin users also just need it as a simple medium of exchange, buying bitcoins with traceable fiat and immediately spending them on goods and services. Example would be an anonymous buyer of a domain name, VPS hosting, email or VPN provisions. Users also might be those who engage in capital flight or want to store bitcoins without anyone knowing, tumbling them into cold storage. If bitcoin fungibility is ever attacked, leading to users being faced with messages like "Your coins are rejected because they were used for illegal or immoral activity X transactions ago" then the tumbler app can probably solve the problem. +Having said that, there is a lot of subtlety around *how* to use tools like this to upgrade privacy - neither Joinmarket generally, nor the "tumbler algorithm" presented here are a panacea. Think about how you're using it. @@ -52,67 +51,128 @@ That $50 total mining fee (example figure) is independent of the amount of bitco For much larger amounts (~$1000+ in value) on the other hand, the balance shifts significantly and the coinjoin fees will tend to become more important. However, there are enough bots with fixed-size fees and ultra-low percentage fees and you will probably find that the fee won't be an issue. Paying less than 1% fee for the whole run would be normal in this case. +Of course, the above doesn't really apply in very low fee regimes, but the principle that "tumbler represents of the order of 100 one-in-one-out transactions", still applies. + **TLDR: Pay attention to fees, especially if fees on the network are high, it may be worth avoiding using the tumbler in this case.** - + -## Basic Examples +## How it works - the algorithm -These simple examples focus on using the command line. You can do basically the same with JoinmarketQt. +The basic concepts: -Here's the simplest reasonable workflow: +First, we need multiple destination addresses; the default is 3, but you can go higher. Why? Because there's no point trying to "mix" 3.754 BTC through a bunch of transactions, if the entire amount ends up in the same place (minus fees; these are usually nontrivial but that's not good enough to break the link). -Follow the [usage guide](USAGE.md) on how to fund your wallet. Don't neglect to read [this](https://github.com/JoinMarket-Org/joinmarket/wiki/Sourcing-commitments-for-joins) page, otherwise you could encounter problems. +Second, we need to add randomized delays to spread these transactions out over a *significant* time. This is not really optional; even if Joinmarket had 10 times the volume it has today, it'd still be the case that if you do all these transactions over a period of 5-10 blocks, it would be very obvious and stand out like a sore thumb. You want these transactions to be *some* of the Joinmarket transactions over the period, not all of them! (If Joinmarket had 1000x the volume per unit time, then perhaps you could do this quickly, but Bitcoin cannot even support that!). -You will need three or more addresses of your destination. If you use just one address, the spy could see X amount of bitcoins going in and then just search for an output of similar size to X. Using three or more addresses means you can split up payments into different sizes which together add up to X. Just make sure you don't then recombine them into one transaction of size X. +So now, in outline, how the tumbler works. The basic idea is to move the coins from one mixdepth to another, in sequence, but (usually) in multiple transactions, and always emptying fully each mixdepth, at the end (i.e. using "sweeps", coinjoins with no leftover change outputs). The best way to understand the process, is by example. -The `tumbler.py` script can be made to ask you for a new address just before it needs to send (this is for now a command line only feature), giving you the chance to click Generate New Address on whatever service you're using and copypaste it in. (Beware: Some services like Bitstamp only allow one new address every 24 hours). If you're depositing to a normal bitcoin wallet (for example Electrum) then you can just obtain many addresses and tumbler won't need to ask you for more. By default, tumbler asks for addresses until it has 3 or more. + -**Warning: The above step is very important. You CANNOT use just a single address and expect good privacy.** +### Example 1. Three utxos in mixdepth 0 (each 1BTC), 4 mixdepths, 9 counterparties, 3 transactions per mixdepth, 3 destination addresses A, B, C. -Run `tumbler.py` with your wallet file and at least one OUTPUT address. Ex: +This is the simplest setup, pretty much default according to recommendations. Notice here we specifically mean that mixdepths 1,2,3,4 all start out empty. -``` -(jmvenv)a@~/joinmarket-clientserver/scripts$ python tumbler.py wallet.jmdat addr1 addr2 addr3 -``` +Phase 1: The coins in mixdepth 0 will be moved in a sweep, to mixdepth 1. No change is behind; the full amount *after* fees (let's say, 2.99 BTC) will arrive in an "internal" address in mixdepth 1. Of course the point is that there will be 9 exactly identical 2.99 BTC utxos in the transaction output; only one is yours. -It will print out an estimate of the time taken, +Phase 2: +* There will now be 3 transactions from mixdepth 1 to mixdepth 2. The last will be a sweep (also to mixdepth 2). +* Then, 3 transactions from mixdepth 2. The first two will go to mixdepth 3, the last (a sweep) will be to destination address A +* Then, 3 transactions from mixdepth 3. The first two will go to mixdepth 4, the last (a sweep) will be to destination address B. +* Then, *one* transaction from mixdepth 4, to the final destination address C. + +Here is what an example of that looks like, as a schedule, generated by Joinmarket's code (the addresses are testnet): ``` -waits in total for 19 blocks and 35.96 minutes -estimated time taken 225.96 minutes or 3.77 hours -tumble with these tx? (y/n): +0,0,9,INTERNAL,0.02,16,0 +1,0.07158886670804065,9,INTERNAL,0.15,16,0 +1,0.3104360747679161,9,INTERNAL,0.25,16,0 +1,0,9,INTERNAL,0.41,16,0 +2,0.28860335923421476,9,INTERNAL,0.31,16,0 +2,0.17728531788154556,9,INTERNAL,0.04,16,0 +2,0,9,mzzAYbtPpANxpNVGCVBAhZYzrxyZtoix7i,0.05,16,0 +3,0.1593149311659825,9,INTERNAL,0.05,16,0 +3,0.5469121480293317,9,INTERNAL,0.08,16,0 +3,0,9,mifCWfmygxKhsP3qM3HZi3ZjBEJu7m39h8,0.04,16,0 +4,0,9,mnTn9KVQQT9zy9R4E2ZGzWPK4EfcEcV9Y5,0.07,16,0 ``` -Type 'y' if you're happy to tumble. Bot will then connect to the JoinMarket pit and start doing transactions. +To understand the term 'schedule' here and the meaning of the above list, see [Schedules](#schedules) below. -When `tumbler.py` needs another destination address, it will ask for a new address. +Note a point of confusion re: counting "number of mixdepths 4" here means 4 mixdepths are used in Phase 2, so we start from 1 and end in 4. Also notice, the last mixdepth is always different in that there is only one sweep to one of the final destination addresses. -``` -insert new address: 3Axxxxxxxxxxxxxxxxxxxxxxxxxxxxxxfr -``` +Number of counterparties: controlled with `-N` on the command line, this defaults to `9 1` which means 9 with a standard deviation of 1 (so usually 8-10); this is probably best left at defaults, though you can go a little lower, or experiment with significantly higher (especially with the new message channels as of 2022), if the fees as discussed above don't cause a problem. -Come back later when the bot has finished. +Amounts and times: these are both randomized. You can't control how much goes to which of the different destination addresses, using the algorithm, and also the time delays between each transaction are randomized. See the options on CLI or the tumbler 'wizard' in the Qt app for how to control the *average* time. Note that for privacy, longer time waits are almost always better. Expect this process to take up hours or days, that is how it is intended to be used. - + -### Example: Tumbling into your wallet after buying from an exchange to improve privacy +### Example 2. Two utxos in mixdepth 2 (each 1BTC), one utxo in mixdepth 4, 8 mixdepths, 9 counterparties, 4 transactions per mixdepth, 4 destination addresses A, B, C, D. + -``` -(jmvenv)a@~/joinmarket-clientserver/scripts$ python tumbler.py wallet.jmdat addr1 addr2 addr3 addr4 addr5 -``` +First, note you can use more than 3 destination addresses (and it's good to do so), if you're mixing through more than the default 4 mixdepths. +Second, note that the tumbler algorithm as of [this commit](ADD_LINK_HERE), now **cycles** through the default 5 mixdepths, instead of creating extra ones. This means that the mixdepth path goes as follows: + +Phase 1: +* Sweep from mixdepth 2 to mixdepth 3 +* Sweep from mixdepth 4 to mixdepth 0 -The addresses are from the Addresses tab in Electrum. After tumbling is done you can spend bitcoins on normal things probably without the exchange collecting data on your purchases. All other parameters are left as default values. +Phase 2: +Starting mixdepth is 0 because that is the lowest non-empty *after* Phase 1. Then, the sequence is (0->1, 1->2, 2->3, 3->4, 4->0, 0->1, 1->2). The final transaction will sweep from mixdepth 2 to the final destination address. - +The multiple uses of the same mixdepth do not "step on each other's toes", for two reasons: one, Joinmarket never reuses an address, and two, we always sweep (and therefore entirely clean out) each mixdepth as we go through it. -### Example: Tumbling from your wallet into an exchange +Doing things this way is cleaner: we keep to a fixed number of mixdepths/accounts in the wallet, even if we want to do a very large run of the tumbler algo. + +Here is a test example schedule with those parameters: ``` -(jmvenv)a@~/joinmarket-clientserver/scripts$ python tumbler.py wallet.jmdat 1LspBoDEcFPUtdybkarJCu893EJMC4rsXc +4,0,9,INTERNAL,0.22,16,0 +2,0,9,INTERNAL,1.25,16,0 +0,0.1287547602736554,9,INTERNAL,0.34,16,0 +0,0.33777065308789445,9,INTERNAL,0.12,16,0 +0,0.2416658618765749,9,INTERNAL,0.05,16,0 +0,0,9,INTERNAL,0.23,16,0 +1,0.4248409290648639,9,INTERNAL,0.01,16,0 +1,0.33866158339454555,9,INTERNAL,0.02,4,0 +1,0.010807366510609207,9,INTERNAL,0.13,16,0 +1,0,9,INTERNAL,0.11,16,0 +2,0.04086022411519208,9,INTERNAL,0.82,16,0 +2,0.20924362829352816,9,INTERNAL,0.03,16,0 +2,0.03518603894933314,9,INTERNAL,0.05,16,0 +2,0,9,INTERNAL,0.16,16,0 +3,0.13973910506875786,9,INTERNAL,0.38,4,0 +3,0.21418596171826687,9,INTERNAL,0.24,16,0 +3,0.3792667736100306,9,INTERNAL,0.08,16,0 +3,0,9,INTERNAL,0.07,16,0 +4,0.23084503924196553,9,INTERNAL,0.02,16,0 +4,0.3566850751084202,9,INTERNAL,0.07,16,0 +4,0.06412832650536227,9,INTERNAL,0.09,16,0 +4,0,9,mzzAYbtPpANxpNVGCVBAhZYzrxyZtoix7i,0.04,16,0 +0,0.3794032390530363,9,INTERNAL,0.02,16,0 +0,0.10756327418131051,9,INTERNAL,0.93,16,0 +0,0.40107055434802497,9,INTERNAL,0.07,16,0 +0,0,9,mifCWfmygxKhsP3qM3HZi3ZjBEJu7m39h8,0.11,16,0 +1,0.05776628660005234,9,INTERNAL,0.93,16,0 +1,0.1936955942281181,9,INTERNAL,0.66,16,0 +1,0.13956928336353558,9,INTERNAL,0.14,16,0 +1,0,9,bcrt1qcnv26w889eum5sekz5h8we45rxnr4sj5k08phv,0.58,16,0 +2,0,9,mnTn9KVQQT9zy9R4E2ZGzWPK4EfcEcV9Y5,0.52,16,0 ``` -The first address is from the exchange (are '1' addresses as destinations OK for Joinmarket? Yes and no; the tumbler will still have a strong effect, but it's less ideal). Under default configuration, the bot will ask for two more addresses near the end of the tumble, allowing the user to click Generate New Deposit Address and copypaste it in (if the exchange supports that, which is sadly rarer nowadays). +### Restarting + +Even before discussing practical code-level actions, we can see: this approach allows us to have coins in *any* mixdepth when we start; so we no longer have the concept of "restarting" if you manually ended the run halfway, or if a transaction repeatedly failed and you had to give up. You can judge for yourself; if you started a tumbler run of 8 mixdepths and it stopped after 3, you can do another run with 5 mixdepths later, if you like. W.r.t the destination addresses, you were never able to control the ratio that arrives at different destinations anyway (it's technically possible but not recommended, you'd need to create schedules manually and think carefully about it), so this really doesn't change that aspect. + +Delaying the whole process by stopping and restarting it is quite sensible anyway; as explained above, we *want* this process to be slow, not fast. + +### Reminder about commitments. + +Follow the [usage guide](USAGE.md) on how to fund your wallet. Don't neglect to read [this](https://github.com/JoinMarket-Org/joinmarket/wiki/Sourcing-commitments-for-joins) page, otherwise you could encounter problems. + +This is actually a really important area with the tumbler, because we use sweeps often. It's not really crucial to use 3 utxos to fund at the start, but try to fund with 2, anyway. And: + +It's **strongly** recommended to use counterparty counts (as discussed above; `-N` on the command line) of 8 or higher, **and** `--minmakercount` of 4 (the default) or 5, to give maximum possibility to achieve a successful join every time you make a request (if makers are flaky in the first phase of negotiation, you can still complete as long as up to `--minmakercount` respond correctly). @@ -124,13 +184,13 @@ In this implementation, each coinjoin has an associated "schedule" of format lik [mixdepth, amount-fraction, N-counterparties (requested), destination address, wait time in minutes, rounding, flag indicating incomplete/broadcast/completed (0/txid/1)] ``` -`[]` here represents a Python list. It's recorded in files in a csv format (because in some cases users may edit). See [this](https://github.com/AdamISZ/joinmarket-clientserver/blob/master/scripts/sample-schedule-for-testnet) testnet sample given in the repo. A couple of extra notes on the format: +`[]` here represents a Python list. It's recorded in files in a csv format (because in some cases users may edit). See [this](https://github.com/Joinmarket-Org/joinmarket-clientserver/blob/master/scripts/sample-schedule-for-testnet) testnet sample given in the repo. A couple of extra notes on the format: -* the 4th entry, the destination address, can have special values "INTERNAL" and "addrask"; the former indicates that the coins are to be sent to the "next" mixdepth, modulo the maximum mixdepth. The latter is the same function as in the original implementation, i.e. it takes a destination from those provided by the user, either in the initial command line or on a prompt during the run. +* the 4th entry, the destination address, can have special values "INTERNAL" and "addrask"; the former indicates that the coins are to be sent to the "next" mixdepth, modulo the maximum mixdepth. The latter takes a destination from those provided by the user, either in the initial command line or on a prompt during the run. -* the 2nd entry, amount fraction, is a decimal between 0 and 1; *this is specific to the tumbler*; for sendpayment, this amount is an integer in satoshis. +* the 2nd entry, amount fraction, is a decimal between 0 and 1; *this is specific to the tumbler*; if a schedule has a (nonzero) integer, that is used (in `sendpayment`) for non-tumbler coinjoin sends.. -* 0 amounts for the second entry indicate, as for command line flags, a sweep; decimals indicate mixdepth fractions (for tumbler) +* 0 amounts for the second entry indicate, as for command line flags, a sweep; decimals indicate mixdepth fractions (for tumbler), e.g. if your mixdepth's total balance is 10.0 BTC and this value is 0.22 then 2.2 BTC will be sent. * the 6th entry, `rounding`, is how many significant figures to round the coinjoin amount to. For example a rounding of `2` means that `0.12498733` will be rounded to `0.1200000`. A rounding value of `16` means no rounding. Sweep coinjoin amounts are never rounded. @@ -142,9 +202,9 @@ As you can imagine, the idea for the `tumbler.py` script, and the MultiJoin wiza ## Tumbler schedule and logging -There are two log files to help tracking the progress of the tumble, and to allow restarts. The first is by default `~/.joinmarket/logs/TUMBLE.schedule` but its name can be changed with the new `--schedulefile` option. In this, the schedule that is generated on startup, according to the user command line options (such as -N for counterparties, -M for mixdepths etc.) is recorded, and updated as each transaction is seen on the network - in particular what is updated is the above-mentioned 'completed' flag, as well as the destination addresses for the user destinations (replacing 'addrask'). So by opening it at any time you can see a condensed view of the current state (note in particular '1' or '0' for the final entry; '1' means the transaction is done). The *main* purpose of this file is to allow restarts, see the section on "Restarts" [below](#restarts). Thus, **don't edit or delete this file until the tumble run is fully completed**. +There are two log files to help tracking the progress of the tumble. The first is by default `/logs/TUMBLE.schedule` but its name can be changed with the new `--schedulefile` option. In this, the schedule that is generated on startup, according to the user command line options (such as -N for counterparties, -M for mixdepths etc.) is recorded, and updated as each transaction is seen on the network - in particular what is updated is the above-mentioned 'completed' flag, as well as the destination addresses for the user destinations (replacing 'addrask'). So by opening it at any time you can see a condensed view of the current state (note in particular '1' or '0' for the final entry; '1' means the transaction is done). -However, another file is more specifically intended to help tracking: currently hardcoded as `~/.joinmarket/logs/TUMBLE.log`, it will show: transaction times, txids, destination addresses, and also any instances of failures and re-attempts. This is not used for restarting, so can be deleted at any time (it's a standard log file and operates in append by default for multiple runs). +However, another file is more specifically intended to help tracking: currently hardcoded as `/logs/TUMBLE.log`, it will show: transaction times, txids, destination addresses, and also any instances of failures and re-attempts. It's a standard log file and operates in append by default for multiple runs). @@ -186,7 +246,7 @@ If the tumbler continues to run after this (re: if it doesn't, see the section o ``` Makers didn't respond ``` -This will happen when aberrant makers don't complete the protocol (or strictly, when not enough of them do). As above, simply wait for regenerate-after-tweak occurs. +This will happen when too many aberrant makers don't complete the protocol. As above, simply wait for regenerate-after-tweak occurs. @@ -206,23 +266,13 @@ This tweaking process is repeated as many times as necessary until the transacti This is hardcoded currently to `20 * maker_timeout_sec`, the figure 20 being hardcoded is due to me not wanting yet another config variable, although that could be done of course. This is the rate at which the stall monitor wakes up in the client protocol, the setting is in the code [here](https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/acc00fc6f5a1cd1f21052c5af06cd06e78c6edda/jmclient/jmclient/client_protocol.py#L359-L363). Note that by default this is fairly slow, 10 minutes. - - -## Restarts - -In case of shutdown by the user or crash, the `TUMBLE.schedule` file mentioned above will have an up-to-date record of which transactions in the schedule completed successfully; and you can find the txids, for convenience, in `TUMBLE.log` to sanity check (of course you may want to run `wallet-tool.py` also, which is fine). By restarting the tumbler script with the same parameters, but appending an additional parameter `--restart`, the script will continue the tumble from the first not-successfully-completed transaction and continue (it will wait for confirmations on the last transaction, if it's not yet in a block). If you used a custom name for `TUMBLE.schedule`, or renamed it afterwards, don't forget to also pass the parameter `--schedulefile` so it can be found; note that these files are always assumed to be in the `logs/` subdirectory of where you're running (so `scripts/logs` here). (A small technical note: on restart, the `TUMBLE.schedule` is truncated in that the txs that already completed will be removed, something that should probably change, but all the info is logged in `TUMBLE.log`, which you should use as your primary record of what happened and when). - -Minor note, you could conceivably edit `TUMBLE.schedule` before restarting, but this would have to be considered "advanced" usage! - -This can of course be implemented in, say, a shell script (just add --restart to all re-runs except the first), although I haven't tried that out. - ## Possible failure vectors - crash or shutdown * **Failure to source commitment** - if there is no *unused* PoDLE commitment available, the script terminates as even with tweaks this condition will not change. This *could* be changed to allow dynamic update of the `commitments.json` file (adding external utxos), but I didn't judge that to be the right choice for now. On the other hand, as was noted above, if the commitments are simply too young, the script will keep tweaking and retrying. I recommend using the `add-utxo.py` script to prepare external commitments in advance of the run for more robustness, although it shouldn't be necessary for success. -* **Network errors** - this should not cause a problem. Joinmarket handles network interruptions to its IRC communications quite robustly. -* **Insufficient liquidity**. This is a tricky one - particulary for sweeps, if the number of potential counterparties is low, and if some of them are deliberately non-responsive, you may run out of counterparties. Currently the script will simply keep retrying indefinitely. **Use a reasonably high -N value** - I think going much below 5 is starting to introduce risk, so values like `-N 6 1` should be OK, but `-N 3 1` is dubious. Force-quitting after a very long timeout is conceivable, but obviously a slightly tricky/impractical proposition. +* **Network errors** - this should not cause a problem. Joinmarket handles network interruptions to its onion services and/or IRC servers quite robustly. +* **Insufficient liquidity**. This is a tricky one - particulary for sweeps, if the number of potential counterparties is low, and if some of them are deliberately non-responsive, you may run out of counterparties. Currently the script will simply keep retrying indefinitely. Note that various other failure vectors will not actually cause a problem, such as the infamous "txn-mempool-conflict"; tweaking handles these cases.