diff --git a/docs/release-notes/release-notes-tumbler-improve.md b/docs/release-notes/release-notes-tumbler-improve.md new file mode 100644 index 0000000..135746b --- /dev/null +++ b/docs/release-notes/release-notes-tumbler-improve.md @@ -0,0 +1,20 @@ +copypaste this into "release-notes" when the time comes to make a new release, then delete this file + +Notable changes +=============== + +### Tumbler privacy improvements + +The tumbler algorithm has been improved with the aim to increase privacy. This affects the `tumbler.py` script and `joinmarket-qt.py` GUI. + +* At the start of the run, tumbler will now fully spend all mixdepths with coinjoin with no change address (also known as a sweep transaction) back to its own internal wallet. After these initial sweeps are done tumbler will continue with the already-existing algorithm of sending coinjoins with randomly-generated amounts. + +* Tumbler will now occasionally send a round number of bitcoins, for example `0.20000000` or `0.15000000` instead of `0.24159873`. The default probability of this happening is 25% per coinjoin. + +* The default wait time between coinjoins is increased from 30 minutes to 60 minutes. + +* The default number of coinjoin counterparties is increased from 6 to 9. + +* The default number of coinjoins per mixdepth is decreased from 4 to 2. + +For a full discription and reasoning behind the changes see: [Plan to improve the privacy of JoinMarket's tumbler script](https://gist.github.com/chris-belcher/7e92810f07328fdfdef2ce444aad0968) diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index 226fcc0..cec7152 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -6,7 +6,8 @@ import logging from .support import (calc_cj_fee, choose_sweep_orders, choose_orders, cheapest_order_choose, weighted_order_choose, - rand_norm_array, rand_pow_array, rand_exp_array, select, + rand_norm_array, rand_pow_array, rand_exp_array, + rand_weighted_choice, select, select_gradual, select_greedy, select_greediest, get_random_bytes, random_under_max_order_choose, select_one_utxo) @@ -37,7 +38,7 @@ from .output import generate_podle_error_string, fmt_utxos, fmt_utxo,\ fmt_tx_data from .schedule import (get_schedule, get_tumble_schedule, schedule_to_text, tweak_tumble_schedule, human_readable_schedule_entry, - schedule_to_text) + schedule_to_text, NO_ROUNDING) from .commitment_utils import get_utxo_info, validate_utxo_data, quit from .taker_utils import (tumbler_taker_finished_update, restart_waiter, restart_wait, get_tumble_log, direct_send, diff --git a/jmclient/jmclient/configure.py b/jmclient/jmclient/configure.py index a50c07c..391c486 100644 --- a/jmclient/jmclient/configure.py +++ b/jmclient/jmclient/configure.py @@ -251,7 +251,7 @@ tx_broadcast = self # amount of makers which we are content with for the coinjoin to # succceed. Less makers means that the whole process will restart # after a timeout. -minimum_makers = 2 +minimum_makers = 4 ############################## #THE FOLLOWING SETTINGS ARE REQUIRED TO DEFEND AGAINST SNOOPERS. diff --git a/jmclient/jmclient/schedule.py b/jmclient/jmclient/schedule.py index c89edc5..31b7543 100644 --- a/jmclient/jmclient/schedule.py +++ b/jmclient/jmclient/schedule.py @@ -3,8 +3,11 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) from builtins import * # noqa: F401 import copy -from jmclient import (validate_address, rand_exp_array, - rand_norm_array, rand_pow_array, jm_single) +import random +import sys + +from .configure import validate_address, jm_single +from .support import rand_exp_array, rand_norm_array, rand_weighted_choice """Utility functions for dealing with Taker schedules. - get_schedule(filename): @@ -17,6 +20,8 @@ from jmclient import (validate_address, rand_exp_array, the chance of success on re-trying """ +NO_ROUNDING = 16 #max btc significant figures not including LN + def get_schedule(filename): with open(filename, "rb") as f: schedule = [] @@ -26,8 +31,8 @@ def get_schedule(filename): if sl.startswith("#"): continue try: - mixdepth, amount, makercount, destaddr, waittime, completed = \ - sl.split(',') + (mixdepth, amount, makercount, destaddr, waittime, + rounding, completed) = sl.split(',') except ValueError as e: return (False, "Failed to parse schedule line: " + sl) try: @@ -41,6 +46,7 @@ def get_schedule(filename): makercount = int(makercount) destaddr = destaddr.strip() waittime = float(waittime) + rounding = int(rounding) completed = completed.strip() if not len(completed) == 64: completed = int(completed) @@ -51,12 +57,11 @@ def get_schedule(filename): if not success: return (False, "Invalid address: " + destaddr + "," + errmsg) schedule.append([mixdepth, amount, makercount, destaddr, - waittime, completed]) + waittime, rounding, completed]) return (True, schedule) -def get_amount_fractions(power, count): - """Get 'count' fractions following power law distn according to - parameter 'power' +def get_amount_fractions(count): + """Get 'count' fractions following uniform distn Note that this function is not entirely generic; it ensures that the final entry is larger than a certain fraction, for a reason specific to the way the tumbler algo works: the last entry @@ -65,9 +70,16 @@ def get_amount_fractions(power, count): setting, so we make sure it's appreciable to begin with. """ while True: - amount_fractions = rand_pow_array(power, count) - amount_fractions = [1.0 - x for x in amount_fractions] - y = [x / sum(amount_fractions) for x in amount_fractions] + knives = [random.random() for i in range(count-1)] + knives = sorted(knives)[::-1] + y = [] + l = 1 + k = 1 + for k in knives: + y.append( l - k ) + l = k + y.append(k) + #Here we insist that the last entry in the list is more #than 5% of the total, to account for tweaks upwards #on previous joins. @@ -75,7 +87,7 @@ def get_amount_fractions(power, count): break return y -def get_tumble_schedule(options, destaddrs): +def get_tumble_schedule(options, destaddrs, mixdepth_balance_dict): """for the general intent and design of the tumbler algo, see the docs in joinmarket-org/joinmarket. Alterations: @@ -97,6 +109,30 @@ def get_tumble_schedule(options, destaddrs): options['txcountparams'][1], options['mixdepthcount']) txcounts = lower_bounded_int(txcounts, options['mintxcount']) tx_list = [] + ### stage 1 coinjoins, which sweep the entire mixdepth without creating change + lowest_initial_filled_mixdepth = sys.maxsize + sweep_mixdepths = [] + for mixdepth, balance in mixdepth_balance_dict.items(): + if balance > 0: + lowest_initial_filled_mixdepth = min(mixdepth, + lowest_initial_filled_mixdepth) + sweep_mixdepths.append(mixdepth) + waits = rand_exp_array(options['timelambda']*options[ + 'stage1_timelambda_increase'], len(sweep_mixdepths)) + makercounts = rand_norm_array(options['makercountrange'][0], + options['makercountrange'][1], len(sweep_mixdepths)) + makercounts = lower_bounded_int(makercounts, options['minmakercount']) + sweep_mixdepths = sorted(sweep_mixdepths)[::-1] + for mixdepth, wait, makercount in zip(sweep_mixdepths, waits, makercounts): + tx = {'amount_fraction': 0, + 'wait': round(wait, 2), + 'srcmixdepth': mixdepth, + 'makercount': makercount, + 'destination': 'INTERNAL', + 'rounding': NO_ROUNDING + } + tx_list.append(tx) + ### stage 2 coinjoins, which create a number of random-amount coinjoins from each mixdepth for m, txcount in enumerate(txcounts): if options['mixdepthcount'] - options['addrcount'] <= m and m < \ options['mixdepthcount'] - 1: @@ -104,7 +140,7 @@ def get_tumble_schedule(options, destaddrs): # amount_fraction cant be 1.0, some coins must be left over if txcount == 1: txcount = 2 - amount_fractions = get_amount_fractions(options['amountpower'], txcount) + amount_fractions = get_amount_fractions(txcount) # transaction times are uncorrelated # time between events in a poisson process followed exp waits = rand_exp_array(options['timelambda'], txcount) @@ -112,23 +148,31 @@ def get_tumble_schedule(options, destaddrs): makercounts = rand_norm_array(options['makercountrange'][0], options['makercountrange'][1], txcount) makercounts = lower_bounded_int(makercounts, options['minmakercount']) - - for amount_fraction, wait, makercount in zip(amount_fractions, waits, - makercounts): + do_rounds = [random.random() < options['rounding_chance'] for _ in range(txcount)] + for amount_fraction, wait, makercount, do_round in zip(amount_fractions, waits, + makercounts, do_rounds): + rounding = NO_ROUNDING + if do_round: + weight_sum = 1.0*sum(options['rounding_sigfig_weights']) + weight_prob = [a/weight_sum for a in options['rounding_sigfig_weights']] + rounding = rand_weighted_choice(len(weight_prob), weight_prob) + 1 tx = {'amount_fraction': amount_fraction, 'wait': round(wait, 2), - 'srcmixdepth': m + options['mixdepthsrc'], + 'srcmixdepth': lowest_initial_filled_mixdepth + m + options['mixdepthsrc'] + 1, 'makercount': makercount, - 'destination': 'INTERNAL'} + 'destination': 'INTERNAL', + 'rounding': rounding + } tx_list.append(tx) #reset the final amt_frac to zero, as it's the last one for this mixdepth: tx_list[-1]['amount_fraction'] = 0 + tx_list[-1]['rounding'] = NO_ROUNDING addrask = options['addrcount'] - len(destaddrs) - external_dest_addrs = ['addrask'] * addrask + destaddrs + external_dest_addrs = ['addrask'] * addrask + destaddrs[::-1] for mix_offset in range(options['addrcount']): - srcmix = (options['mixdepthsrc'] + options['mixdepthcount'] - - mix_offset - 1) + srcmix = (lowest_initial_filled_mixdepth + options['mixdepthsrc'] + + options['mixdepthcount'] - mix_offset) for tx in reversed(tx_list): if tx['srcmixdepth'] == srcmix: tx['destination'] = external_dest_addrs[mix_offset] @@ -146,7 +190,8 @@ def get_tumble_schedule(options, destaddrs): schedule = [] for t in tx_list: schedule.append([t['srcmixdepth'], t['amount_fraction'], - t['makercount'], t['destination'], t['wait'], 0]) + t['makercount'], t['destination'], t['wait'], + t['rounding'], 0]) return schedule def tweak_tumble_schedule(options, schedule, last_completed, destaddrs=[]): @@ -193,7 +238,7 @@ def tweak_tumble_schedule(options, schedule, last_completed, destaddrs=[]): alreadyspent = sum([x[1] for x in already_done]) tobespent = 1.0 - alreadyspent #power law for what's left: - new_fracs = get_amount_fractions(options['amountpower'], len(tobedone)) + new_fracs = get_amount_fractions(len(tobedone)) #rescale; the sum must be 'tobespent': new_fracs = [x*tobespent for x in new_fracs] #starting from the known 'last_completed+1' index, apply these new @@ -211,6 +256,8 @@ def human_readable_schedule_entry(se, amt=None, destn=None): amt_info = str(amt) if amt else str(se[1]) hrs.append("sends amount: " + amt_info + " satoshis") dest_info = destn if destn else str(se[3]) + hrs.append(("rounded to " + str(se[5]) + " significant figures" + if se[5] != NO_ROUNDING else "without rounding")) hrs.append("to destination address: " + dest_info) hrs.append("after coinjoin with " + str(se[2]) + " counterparties.") return ", ".join(hrs) diff --git a/jmclient/jmclient/taker.py b/jmclient/jmclient/taker.py index f811672..89886ad 100644 --- a/jmclient/jmclient/taker.py +++ b/jmclient/jmclient/taker.py @@ -21,6 +21,7 @@ from jmclient.podle import generate_podle, get_podle_commitments, PoDLE from jmclient.wallet_service import WalletService from .output import generate_podle_error_string from .cryptoengine import EngineError +from .schedule import NO_ROUNDING jlog = get_log() @@ -168,6 +169,7 @@ class Taker(object): si = self.schedule[self.schedule_index] self.mixdepth = si[0] self.cjamount = si[1] + rounding = si[5] #non-integer coinjoin amounts are treated as fractions #this is currently used by the tumbler algo if isinstance(self.cjamount, float): @@ -179,6 +181,9 @@ class Taker(object): )[self.mixdepth] #reset to satoshis self.cjamount = int(self.cjamount * self.mixdepthbal) + if rounding != NO_ROUNDING: + self.cjamount = round_to_significant_figures(self.cjamount, + rounding) if self.cjamount < jm_single().mincjamount: jlog.info("Coinjoin amount too low, bringing up to: " + str( jm_single().mincjamount)) @@ -188,7 +193,7 @@ class Taker(object): # for sweeps to external addresses we need an in-wallet import # for the transaction monitor (this will be a no-op for txs to # in-wallet addresses). - if self.cjamount == 0: + if self.cjamount == 0 and self.my_cj_addr != "INTERNAL": self.wallet_service.import_non_wallet_address(self.my_cj_addr) #if destination is flagged "INTERNAL", choose a destination @@ -1222,3 +1227,13 @@ class P2EPTaker(Taker): self.self_sign_and_push() # returning False here is not an error condition, only stops processing. return (False, "OK") + +def round_to_significant_figures(d, sf): + '''Rounding number d to sf significant figures in base 10''' + for p in range(-10, 15): + power10 = 10**p + if power10 > d: + sf_power10 = 10**sf + sigfiged = int(round(d/power10*sf_power10)*power10/sf_power10) + return sigfiged + raise RuntimeError() diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index 4ed6850..e8dd3a9 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -196,7 +196,7 @@ def unconf_update(taker, schedulefile, tumble_log, addtolog=False): #full record should always be accurate; but TUMBLE.log should be #used for checking what actually happened. completion_flag = 1 if not addtolog else taker.txid - taker.schedule[taker.schedule_index][5] = completion_flag + taker.schedule[taker.schedule_index][-1] = completion_flag with open(schedulefile, "wb") as f: f.write(schedule_to_text(taker.schedule)) diff --git a/jmclient/test/test_coinjoin.py b/jmclient/test/test_coinjoin.py index 1031d35..ed8bd0e 100644 --- a/jmclient/test/test_coinjoin.py +++ b/jmclient/test/test_coinjoin.py @@ -13,7 +13,8 @@ from twisted.internet import reactor from jmbase import get_log from jmclient import load_program_config, jm_single,\ - YieldGeneratorBasic, Taker, LegacyWallet, SegwitLegacyWallet + YieldGeneratorBasic, Taker, LegacyWallet, SegwitLegacyWallet,\ + NO_ROUNDING from jmclient.podle import set_commitment_file from commontest import make_wallets, binarize_tx from test_taker import dummy_filter_orderbook @@ -139,8 +140,8 @@ def test_simple_coinjoin(monkeypatch, tmpdir, setup_cj, wallet_cls): assert len(orderbook) == MAKER_NUM cj_amount = int(1.1 * 10**8) - # mixdepth, amount, counterparties, dest_addr, waittime - schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0)] + # mixdepth, amount, counterparties, dest_addr, waittime, rounding + schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0, NO_ROUNDING)] taker = create_taker(wallet_services[-1], schedule, monkeypatch) active_orders, maker_data = init_coinjoin(taker, makers, @@ -183,8 +184,8 @@ def test_coinjoin_mixdepth_wrap_taker(monkeypatch, tmpdir, setup_cj): assert len(orderbook) == MAKER_NUM cj_amount = int(1.1 * 10**8) - # mixdepth, amount, counterparties, dest_addr, waittime - schedule = [(4, cj_amount, MAKER_NUM, 'INTERNAL', 0)] + # mixdepth, amount, counterparties, dest_addr, waittime, rounding + schedule = [(4, cj_amount, MAKER_NUM, 'INTERNAL', 0, NO_ROUNDING)] taker = create_taker(wallet_services[-1], schedule, monkeypatch) active_orders, maker_data = init_coinjoin(taker, makers, @@ -239,8 +240,8 @@ def test_coinjoin_mixdepth_wrap_maker(monkeypatch, tmpdir, setup_cj): assert len(orderbook) == MAKER_NUM cj_amount = int(1.1 * 10**8) - # mixdepth, amount, counterparties, dest_addr, waittime - schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0)] + # mixdepth, amount, counterparties, dest_addr, waittime, rounding + schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0, NO_ROUNDING)] taker = create_taker(wallet_services[-1], schedule, monkeypatch) active_orders, maker_data = init_coinjoin(taker, makers, @@ -301,8 +302,8 @@ def test_coinjoin_mixed_maker_addresses(monkeypatch, tmpdir, setup_cj, orderbook = create_orderbook(makers) cj_amount = int(1.1 * 10**8) - # mixdepth, amount, counterparties, dest_addr, waittime - schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0)] + # mixdepth, amount, counterparties, dest_addr, waittime, rounding + schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0, NO_ROUNDING)] taker = create_taker(wallet_services[-1], schedule, monkeypatch) active_orders, maker_data = init_coinjoin(taker, makers, diff --git a/jmclient/test/test_schedule.py b/jmclient/test/test_schedule.py index 7ce7e0a..2b049f4 100644 --- a/jmclient/test/test_schedule.py +++ b/jmclient/test/test_schedule.py @@ -10,30 +10,30 @@ from jmclient import (get_schedule, get_tumble_schedule, import os valids = """#sample for testing -1, 110000000, 3, INTERNAL, 0, 1 -0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw, 9.88, 0 +1, 110000000, 3, INTERNAL, 0, 16, 1 +0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw, 9.88, 16, 0 """ invalids1 = """#sample for testing -1, 110000000, 3, 5, INTERNAL, 0 +1, 110000000, 3, 5, INTERNAL, 16, 0 #pointless comment here; following line has trailing spaces -0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw ,0, 0, +0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw ,0, 16, 0, """ invalids2 = """#sample for testing -1, 110000000, notinteger, INTERNAL, 0, 0 -0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw, 0, 0 +1, 110000000, notinteger, INTERNAL, 0, 16, 0 +0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw, 0, 16, 0 """ invalids3 = """#sample for testing -1, 110000000, 3, INTERNAL, 0, 0 -0, notinteger, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw, 0, 0 +1, 110000000, 3, INTERNAL, 0, 16, 0 +0, notinteger, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw, 0, 16, 0 """ #invalid address invalids4 = """#sample for testing -1, 110000000, 3, INTERNAL, 0, 0 -0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qq, 0, 0 +1, 110000000, 3, INTERNAL, 0, 16, 0 +0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qq, 0, 16, 0 """ @@ -65,11 +65,13 @@ def get_options(): options.txfee = 5000 options.addrcount = 3 options.mintxcount = 1 - options.amountpower = 100 options.timelambda = 0.2 options.waittime = 10 + options.stage1_timelambda_increase = 3 options.mincjamount = 1000000 options.liquiditywait = 5 + options.rounding_chance = 0.25 + options.rounding_sigfig_weights = (55, 15, 25, 65, 40) options = vars(options) return options @@ -88,7 +90,7 @@ def test_tumble_schedule(destaddrs, txcparams, mixdepthcount): options = get_options() options['mixdepthcount'] = mixdepthcount options['txcountparams'] = txcparams - schedule = get_tumble_schedule(options, destaddrs) + schedule = get_tumble_schedule(options, destaddrs, {0:1}) dests = [x[3] for x in schedule] assert set(destaddrs).issubset(set(dests)) @@ -126,7 +128,7 @@ def test_tumble_tweak(destaddrs, txcparams, mixdepthcount, lastcompleted, options['mixdepthcount'] = mixdepthcount options['txcountparams'] = txcparams options['makercountrange'] = makercountrange - schedule = get_tumble_schedule(options, destaddrs) + schedule = get_tumble_schedule(options, destaddrs, {0:1}) dests = [x[3] for x in schedule] assert set(destaddrs).issubset(set(dests)) new_schedule = tweak_tumble_schedule(options, schedule, lastcompleted) diff --git a/jmclient/test/test_taker.py b/jmclient/test/test_taker.py index a9707cb..1100678 100644 --- a/jmclient/test/test_taker.py +++ b/jmclient/test/test_taker.py @@ -15,7 +15,7 @@ import struct from base64 import b64encode from jmclient import load_program_config, jm_single, set_commitment_file,\ get_commitment_file, SegwitLegacyWallet, Taker, VolatileStorage,\ - get_network, WalletService + get_network, WalletService, NO_ROUNDING from taker_test_data import t_utxos_by_mixdepth, t_orderbook,\ t_maker_response, t_chosen_orders, t_dummy_ext @@ -124,7 +124,7 @@ def get_taker(schedule=None, schedule_len=0, on_finished=None, filter_orders=None): if not schedule: #note, for taker.initalize() this will result in junk - schedule = [['a', 'b', 'c', 'd', 'e']]*schedule_len + schedule = [['a', 'b', 'c', 'd', 'e', 'f']]*schedule_len print("Using schedule: " + str(schedule)) on_finished_callback = on_finished if on_finished else taker_finished filter_orders_callback = filter_orders if filter_orders else dummy_filter_orderbook @@ -138,11 +138,11 @@ def test_filter_rejection(setup_taker): print("calling filter orders rejection") return False taker = get_taker(filter_orders=filter_orders_reject) - taker.schedule = [[0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0]] + taker.schedule = [[0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING]] res = taker.initialize(t_orderbook) assert not res[0] taker = get_taker(filter_orders=filter_orders_reject) - taker.schedule = [[0, 0, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0]] + taker.schedule = [[0, 0, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING]] res = taker.initialize(t_orderbook) assert not res[0] @@ -171,7 +171,7 @@ def test_make_commitment(setup_taker, failquery, external): jm_single().config.set("POLICY", "taker_utxo_amtpercent", "20") mixdepth = 0 amount = 110000000 - taker = get_taker([(mixdepth, amount, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw")]) + taker = get_taker([(mixdepth, amount, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", NO_ROUNDING)]) taker.cjamount = amount taker.input_utxos = t_utxos_by_mixdepth[0] if failquery: @@ -180,7 +180,7 @@ def test_make_commitment(setup_taker, failquery, external): clean_up() def test_not_found_maker_utxos(setup_taker): - taker = get_taker([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)]) + taker = get_taker([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)]) orderbook = copy.deepcopy(t_orderbook) res = taker.initialize(orderbook) taker.orderbook = copy.deepcopy(t_chosen_orders) #total_cjfee unaffected, all same @@ -192,7 +192,7 @@ def test_not_found_maker_utxos(setup_taker): jm_single().bc_interface.setQUSFail(False) def test_auth_pub_not_found(setup_taker): - taker = get_taker([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)]) + taker = get_taker([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)]) orderbook = copy.deepcopy(t_orderbook) res = taker.initialize(orderbook) taker.orderbook = copy.deepcopy(t_chosen_orders) #total_cjfee unaffected, all same @@ -214,33 +214,33 @@ def test_auth_pub_not_found(setup_taker): @pytest.mark.parametrize( "schedule, highfee, toomuchcoins, minmakers, notauthed, ignored, nocommit", [ - ([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw")], False, False, + ([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False, 2, False, None, None), - ([(0, 0, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)], False, False, + ([(0, 0, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False, 2, False, None, None), #sweep - ([(0, 0.2, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)], False, False, + ([(0, 0.2, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False, 2, False, None, None), #tumble style non-int amounts #edge case triggers that don't fail - ([(0, 0, 4, "mxeLuX8PP7qLkcM8uarHmdZyvP1b5e1Ynf", 0)], False, False, + ([(0, 0, 4, "mxeLuX8PP7qLkcM8uarHmdZyvP1b5e1Ynf", 0, NO_ROUNDING)], False, False, 2, False, None, None), #sweep rounding error case - ([(0, 199850001, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)], False, False, + ([(0, 199850001, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False, 2, False, None, None), #trigger sub dust change for taker #edge case triggers that do fail - ([(0, 199850000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)], False, False, + ([(0, 199850000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False, 2, False, None, None), #trigger negative change - ([(0, 199599800, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)], False, False, + ([(0, 199599800, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False, 2, False, None, None), #trigger sub dust change for maker - ([(0, 20000000, 3, "INTERNAL", 0)], True, False, + ([(0, 20000000, 3, "INTERNAL", 0, NO_ROUNDING)], True, False, 2, False, None, None), #test high fee - ([(0, 20000000, 3, "INTERNAL", 0)], False, False, + ([(0, 20000000, 3, "INTERNAL", 0, NO_ROUNDING)], False, False, 7, False, None, None), #test not enough cp - ([(0, 80000000, 3, "INTERNAL", 0)], False, False, + ([(0, 80000000, 3, "INTERNAL", 0, NO_ROUNDING)], False, False, 2, False, None, "30000"), #test failed commit - ([(0, 20000000, 3, "INTERNAL", 0)], False, False, + ([(0, 20000000, 3, "INTERNAL", 0, NO_ROUNDING)], False, False, 2, True, None, None), #test unauthed response - ([(0, 5000000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)], False, True, + ([(0, 5000000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, True, 2, False, None, None), #test too much coins - ([(0, 0, 5, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)], False, False, + ([(0, 0, 5, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False, 2, False, ["J659UPUSLLjHJpaB", "J65z23xdjxJjC7er", 0], None), #test inadequate for sweep ]) def test_taker_init(setup_taker, schedule, highfee, toomuchcoins, minmakers, diff --git a/scripts/cli_options.py b/scripts/cli_options.py index 663a994..7b0d475 100644 --- a/scripts/cli_options.py +++ b/scripts/cli_options.py @@ -269,9 +269,9 @@ def get_tumbler_parser(): action='store', dest='makercountrange', help= - 'Input the mean and spread of number of makers to use. e.g. 6 1 will be a normal distribution ' - 'with mean 6 and standard deviation 1 inclusive, default=6 1 (floats are also OK)', - default=(6, 1)) + 'Input the mean and spread of number of makers to use. e.g. 9 1 will be a normal distribution ' + 'with mean 9 and standard deviation 1 inclusive, default=9 1 (floats are also OK)', + default=(9, 1)) parser.add_option( '--minmakercount', type='int', @@ -292,17 +292,17 @@ def get_tumbler_parser(): type='float', nargs=2, dest='txcountparams', - default=(4, 1), + default=(2, 1), help= 'The number of transactions to take coins from one mixing depth to the next, it is' ' randomly chosen following a normal distribution. Should be similar to --addrask. ' - 'This option controls the parameters of the normal distribution curve. (mean, standard deviation). default=4 1') + 'This option controls the parameters of the normal distribution curve. (mean, standard deviation). default=2 1') parser.add_option( '--mintxcount', type='int', dest='mintxcount', - default=1, - help='The minimum transaction count per mixing level, default=1') + default=2, + help='The minimum transaction count per mixing level, default=2') parser.add_option( '--donateamount', type='float', @@ -310,23 +310,25 @@ def get_tumbler_parser(): default=0, help= 'percent of funds to donate to joinmarket development, or zero to opt out (default=0%)') - parser.add_option( - '--amountpower', - type='float', - dest='amountpower', - default=100.0, - help= - 'The output amounts follow a power law distribution, this is the power, default=100.0') parser.add_option( '-l', '--timelambda', type='float', dest='timelambda', - default=30, + default=60, help= 'Average the number of minutes to wait between transactions. Randomly chosen ' ' following an exponential distribution, which describes the time between uncorrelated' - ' events. default=30') + ' events. default=60') + parser.add_option( + '--stage1-timelambda-increase', + type='float', + dest='stage1_timelambda_increase', + default=3, + help= + 'Stage 1 sweep coinjoins have a longer wait time. This parameter' + ' controls by what factor longer is this average wait time compared to stage2 coinjoins' + ' which are controlled by `--timelambda`, default=3') parser.add_option( '-w', '--wait-time', @@ -376,6 +378,24 @@ def get_tumbler_parser(): 'mixdepthsrc + number of mixdepths to tumble ' 'have been used.', default=-1) + parser.add_option( + '--rounding-chance', + action='store', + type='float', + dest='rounding_chance', + help='probability of non-sweep coinjoin amount being rounded, default=0.25 (25%)', + default=0.25) + parser.add_option( + '--rounding-sigfig-weights', + type='float', + nargs=5, + dest='rounding_sigfig_weights', + default=(55, 15, 25, 65, 40), + help= + "If rounding happens (determined by --rounding-chance) then the weights of how many" + " significant figures to round to. The five values refer to the probability of" + " rounding to one, two, three, four and five significant figures respectively." + " default=(55, 15, 25, 65, 40)") add_common_options(parser) return parser @@ -404,7 +424,7 @@ def get_sendpayment_parser(): type='int', dest='makercount', help='how many makers to coinjoin with, default random from 4 to 6', - default=random.randint(4, 6)) + default=random.randint(8, 10)) parser.add_option('-S', '--schedule-file', type='string', diff --git a/scripts/joinmarket-qt.py b/scripts/joinmarket-qt.py index d584d09..8229e16 100644 --- a/scripts/joinmarket-qt.py +++ b/scripts/joinmarket-qt.py @@ -74,7 +74,8 @@ from jmclient import load_program_config, get_network, update_persist_config,\ get_blockchain_interface_instance, direct_send, WalletService,\ RegtestBitcoinCoreInterface, tumbler_taker_finished_update,\ get_tumble_log, restart_wait, tumbler_filter_orders_callback,\ - wallet_generate_recover_bip39, wallet_display, get_utxos_enabled_disabled + wallet_generate_recover_bip39, wallet_display, get_utxos_enabled_disabled,\ + NO_ROUNDING from qtsupport import ScheduleWizard, TumbleRestartWizard, config_tips,\ config_types, QtHandler, XStream, Buttons, OkButton, CancelButton,\ PasswordDialog, MyTreeWidget, JMQtMessageBox, BLUE_FG,\ @@ -126,7 +127,7 @@ def getSettingsWidgets(): sMM = ['', (2, 20), (0, jm_single().config.getint("GUI", "max_mix_depth") - 1), (0.00000001, 100.0, 8)] - sD = ['', '3', '0', ''] + sD = ['', '9', '0', ''] for x in zip(sN, sH, sT, sD, sMM): ql = QLabel(x[0]) ql.setToolTip(x[1]) @@ -320,20 +321,24 @@ class SpendTab(QWidget): self.spendstate.reset() #trigger callback to 'ready' state def generateTumbleSchedule(self): + if not mainWindow.wallet_service: + JMQtMessageBox(self, "Cannot start without a loaded wallet.", + mbtype="crit", title="Error") + return #needs a set of tumbler options and destination addresses, so needs #a wizard wizard = ScheduleWizard() wizard_return = wizard.exec_() if wizard_return == QDialog.Rejected: return - self.spendstate.loaded_schedule = wizard.get_schedule() + self.spendstate.loaded_schedule = wizard.get_schedule( + mainWindow.wallet_service.get_balance_by_mixdepth()) self.spendstate.schedule_name = wizard.get_name() self.updateSchedView() self.tumbler_options = wizard.opts self.tumbler_destaddrs = wizard.get_destaddrs() #tumbler may require more mixdepths; update the wallet - required_mixdepths = self.tumbler_options['mixdepthsrc'] + \ - self.tumbler_options['mixdepthcount'] + required_mixdepths = max([tx[0] for tx in self.spendstate.loaded_schedule]) if required_mixdepths > jm_single().config.getint("GUI", "max_mix_depth"): jm_single().config.set("GUI", "max_mix_depth", str(required_mixdepths)) #recreate wallet and sync again; needed due to cache. @@ -553,17 +558,17 @@ class SpendTab(QWidget): #follow restart logic #1. filter out complete: self.spendstate.loaded_schedule = [ - s for s in self.spendstate.loaded_schedule if s[5] != 1] + s for s in self.spendstate.loaded_schedule if s[-1] != 1] #reload destination addresses self.tumbler_destaddrs = [x[3] for x in self.spendstate.loaded_schedule if x not in ["INTERNAL", "addrask"]] #2 Check for unconfirmed - if isinstance(self.spendstate.loaded_schedule[0][5], str) and len( - self.spendstate.loaded_schedule[0][5]) == 64: + if isinstance(self.spendstate.loaded_schedule[0][-1], str) and len( + self.spendstate.loaded_schedule[0][-1]) == 64: #ensure last transaction is confirmed before restart tumble_log.info("WAITING TO RESTART...") mainWindow.statusBar().showMessage("Waiting for confirmation to restart..") - txid = self.spendstate.loaded_schedule[0][5] + txid = self.spendstate.loaded_schedule[0][-1] #remove the already-done entry (this connects to the other TODO, #probably better *not* to truncate the done-already txs from file, #but simplest for now. @@ -639,7 +644,7 @@ class SpendTab(QWidget): #note 'amount' is integer, so not interpreted as fraction #see notes in sample testnet schedule for format self.spendstate.loaded_schedule = [[mixdepth, amount, makercount, - destaddr, 0, 0]] + destaddr, 0, NO_ROUNDING, 0]] self.spendstate.updateType('single') self.spendstate.updateRun('running') self.startJoin() diff --git a/scripts/qtsupport.py b/scripts/qtsupport.py index 289c603..fc1e9e1 100644 --- a/scripts/qtsupport.py +++ b/scripts/qtsupport.py @@ -553,7 +553,7 @@ class SchDynamicPage1(QWizardPage): sMM = [(0, jm_single().config.getint("GUI", "max_mix_depth") - 1), (3, 20), (2, 7), (0.00000001, 100.0, 8), (2, 10), (0.000001, 0.25, 6), (0, 10000000)] - sD = ['0', '6', '4', '30.0', '4', '0.005', '10000'] + sD = ['0', '9', '4', '60.0', '2', '0.005', '10000'] for x in zip(sN, sH, sT, sD, sMM): ql = QLabel(x[0]) ql.setToolTip(x[1]) @@ -623,28 +623,39 @@ class SchFinishPage(QWizardPage): layout.setSpacing(4) results = [] - sN = ['Makercount sdev', 'Tx count sdev', - 'Amount power', + sN = ['Makercount sdev', + 'Tx count sdev', 'Minimum maker count', 'Minimum transaction count', 'Min coinjoin amount', - 'wait time'] + 'Response wait time', + 'Stage 1 transaction wait time increase', + 'Rounding Chance'] + for w in ["One", "Two", "Three", "Four", "Five"]: + sN += [w + " significant figures rounding weight"] #Tooltips sH = ["Standard deviation of the number of makers to use in each " - "transaction.", - "Standard deviation of the number of transactions to use in each " - "mixdepth", - "A parameter to control the random coinjoin sizes.", - "The lowest allowed number of maker counterparties.", - "The lowest allowed number of transactions in one mixdepth.", - "The lowest allowed size of any coinjoin, in satoshis.", - "The time in seconds to wait for response from counterparties."] + "transaction.", + "Standard deviation of the number of transactions to use in each " + "mixdepth", + "The lowest allowed number of maker counterparties.", + "The lowest allowed number of transactions in one mixdepth.", + "The lowest allowed size of any coinjoin, in satoshis.", + "The time in seconds to wait for response from counterparties.", + "The factor increase in wait time for stage 1 sweep coinjoins", + "The probability of non-sweep coinjoin amounts being rounded"] + for w in ["one", "two", "three", "four", "five"]: + sH += ["If rounding happens (determined by Rounding Chance) then this " + "is the relative probability of rounding to " + w + + " significant figures"] #types - sT = [float, float, float, int, int, int, float] + sT = [float, float, int, int, int, float, float, float] + [int]*5 #constraints - sMM = [(0.0, 10.0, 2), (0.0, 10.0, 2), (1.0, 10000.0, 1), (2,20), - (1, 10), (100000, 100000000), (10.0, 500.0, 2)] - sD = ['1.0', '1.0', '100.0', '2', '1', '1000000', '20'] + sMM = [(0.0, 10.0, 2), (0.0, 10.0, 2), (2,20), + (1, 10), (100000, 100000000), (10.0, 500.0, 2), (0, 100, 1), + (0.0, 1.0, 3)] + [(0, 10000)]*5 + sD = ['1.0', '1.0', '2', '2', '1000000', '20', '3', '0.25'] +\ + ['55', '15', '25', '65', '40'] for x in zip(sN, sH, sT, sD, sMM): ql = QLabel(x[0]) ql.setToolTip(x[1]) @@ -663,11 +674,14 @@ class SchFinishPage(QWizardPage): #fields not considered 'mandatory' as defaults are accepted self.registerField("makercountsdev", results[0][1]) self.registerField("txcountsdev", results[1][1]) - self.registerField("amountpower", results[2][1]) - self.registerField("minmakercount", results[3][1]) - self.registerField("mintxcount", results[4][1]) - self.registerField("mincjamount", results[5][1]) - self.registerField("waittime", results[6][1]) + self.registerField("minmakercount", results[2][1]) + self.registerField("mintxcount", results[3][1]) + self.registerField("mincjamount", results[4][1]) + self.registerField("waittime", results[5][1]) + self.registerField("stage1_timelambda_increase", results[6][1]) + self.registerField("rounding_chance", results[7][1]) + for i in range(5): + self.registerField("rounding_sigfig_weight_" + str(i+1), results[8+i][1]) class SchIntroPage(QWizardPage): def __init__(self, parent): @@ -714,7 +728,7 @@ class ScheduleWizard(QWizard): def get_destaddrs(self): return self.destaddrs - def get_schedule(self): + def get_schedule(self, wallet_balance_by_mixdepth): self.destaddrs = [] for i in range(self.page(2).required_addresses): daddrstring = str(self.field("destaddr"+str(i))) @@ -735,16 +749,19 @@ class ScheduleWizard(QWizard): self.opts['txcountparams'] = (int(self.field("txcountparams")), float(self.field("txcountsdev"))) self.opts['mintxcount'] = int(self.field("mintxcount")) - self.opts['amountpower'] = float(self.field("amountpower")) self.opts['timelambda'] = float(self.field("timelambda")) self.opts['waittime'] = float(self.field("waittime")) + self.opts["stage1_timelambda_increase"] = float(self.field("stage1_timelambda_increase")) self.opts['mincjamount'] = int(self.field("mincjamount")) relfeeval = float(self.field("maxrelfee")) absfeeval = int(self.field("maxabsfee")) self.opts['maxcjfee'] = (relfeeval, absfeeval) #needed for Taker to check: + self.opts['rounding_chance'] = float(self.field("rounding_chance")) + self.opts['rounding_sigfig_weights'] = tuple([int(self.field("rounding_sigfig_weight_" + str(i+1))) for i in range(5)]) jm_single().mincjamount = self.opts['mincjamount'] - return get_tumble_schedule(self.opts, self.destaddrs) + return get_tumble_schedule(self.opts, self.destaddrs, + wallet_balance_by_mixdepth) class TumbleRestartWizard(QWizard): def __init__(self): @@ -754,7 +771,6 @@ class TumbleRestartWizard(QWizard): def getOptions(self): self.opts = {} - self.opts['amountpower'] = float(self.field("amountpower")) self.opts['mincjamount'] = int(self.field("mincjamount")) relfeeval = float(self.field("maxrelfee")) absfeeval = int(self.field("maxabsfee")) @@ -773,21 +789,19 @@ class RestartSettingsPage(QWizardPage): layout.setSpacing(4) results = [] - sN = ['Amount power', - 'Min coinjoin amount', + sN = ['Min coinjoin amount', 'Max relative fee per counterparty (e.g. 0.005)', 'Max fee per counterparty, satoshis (e.g. 10000)'] #Tooltips - sH = ["A parameter to control the random coinjoin sizes.", - "The lowest allowed size of any coinjoin, in satoshis.", + sH = ["The lowest allowed size of any coinjoin, in satoshis.", "A decimal fraction (e.g. 0.001 = 0.1%) (this AND next must be violated to reject", "Integer number of satoshis (this AND previous must be violated to reject)"] #types - sT = [float, int, float, int] + sT = [int, float, int] #constraints - sMM = [(1.0, 10000.0, 1), (100000, 100000000), (0.000001, 0.25, 6), + sMM = [(100000, 100000000), (0.000001, 0.25, 6), (0, 10000000)] - sD = ['100.0', '1000000', '0.0005', '10000'] + sD = ['1000000', '0.0005', '10000'] for x in zip(sN, sH, sT, sD, sMM): ql = QLabel(x[0]) ql.setToolTip(x[1]) @@ -804,7 +818,6 @@ class RestartSettingsPage(QWizardPage): layout.addWidget(x[1], i + 1, 1, 1, 2) self.setLayout(layout) #fields not considered 'mandatory' as defaults are accepted - self.registerField("amountpower", results[0][1]) - self.registerField("mincjamount", results[1][1]) - self.registerField("maxrelfee", results[2][1]) - self.registerField("maxabsfee", results[3][1]) + self.registerField("mincjamount", results[0][1]) + self.registerField("maxrelfee", results[1][1]) + self.registerField("maxabsfee", results[2][1]) diff --git a/scripts/sendpayment.py b/scripts/sendpayment.py index c0b632a..2610ef7 100644 --- a/scripts/sendpayment.py +++ b/scripts/sendpayment.py @@ -17,7 +17,7 @@ import pprint from jmclient import Taker, P2EPTaker, load_program_config, get_schedule,\ JMClientProtocolFactory, start_reactor, validate_address, jm_single,\ estimate_tx_fee, direct_send, WalletService,\ - open_test_wallet_maybe, get_wallet_path + open_test_wallet_maybe, get_wallet_path, NO_ROUNDING from twisted.python.log import startLogging from jmbase.support import get_log, set_logging_level, jmprint, \ EXIT_FAILURE, EXIT_ARGERROR @@ -77,7 +77,7 @@ def main(): jmprint('ERROR: Address invalid. ' + errormsg, "error") sys.exit(EXIT_ARGERROR) schedule = [[options.mixdepth, amount, options.makercount, - destaddr, 0.0, 0]] + destaddr, 0.0, NO_ROUNDING, 0]] else: if options.p2ep: parser.error("Schedule files are not compatible with PayJoin") diff --git a/scripts/tumbler.py b/scripts/tumbler.py index 7b303fa..ba326af 100644 --- a/scripts/tumbler.py +++ b/scripts/tumbler.py @@ -77,7 +77,7 @@ def main(): jmprint("Error was: " + str(schedule), "error") sys.exit(EXIT_FAILURE) #This removes all entries that are marked as done - schedule = [s for s in schedule if s[5] != 1] + schedule = [s for s in schedule if s[-1] != 1] # remaining destination addresses must be stored in Taker.tdestaddrs # in case of tweaks; note we can't change, so any passed on command # line must be ignored: @@ -90,16 +90,16 @@ def main(): destaddrs = [s[3] for s in schedule if s[3] not in ["INTERNAL", "addrask"]] jmprint("Remaining destination addresses in restart: " + ",".join(destaddrs), "important") - if isinstance(schedule[0][5], str) and len(schedule[0][5]) == 64: + if isinstance(schedule[0][-1], str) and len(schedule[0][-1]) == 64: #ensure last transaction is confirmed before restart tumble_log.info("WAITING TO RESTART...") - txid = schedule[0][5] + txid = schedule[0][-1] restart_waiter(txid) #remove the already-done entry (this connects to the other TODO, #probably better *not* to truncate the done-already txs from file, #but simplest for now. schedule = schedule[1:] - elif schedule[0][5] != 0: + elif schedule[0][-1] != 0: print("Error: first schedule entry is invalid.") sys.exit(EXIT_FAILURE) with open(os.path.join(logsdir, options['schedulefile']), "wb") as f: @@ -107,7 +107,8 @@ def main(): tumble_log.info("TUMBLE RESTARTING") else: #Create a new schedule from scratch - schedule = get_tumble_schedule(options, destaddrs) + schedule = get_tumble_schedule(options, destaddrs, + wallet.get_balance_by_mixdepth()) tumble_log.info("TUMBLE STARTING") with open(os.path.join(logsdir, options['schedulefile']), "wb") as f: f.write(schedule_to_text(schedule))