Browse Source

Merge #387: Implement 2/2019 plan to improve tumbler privacy

32479ae Modify tumbler defaults to improve privacy (chris-belcher)
b79d34a Remove amountpower and use uniform distn instead (chris-belcher)
f40ef2c Occasionally round amounts in tumbler schedule (chris-belcher)
35f23eb Add sweep coinjoins to start of tumbler schedule (chris-belcher)

Tree-SHA512: 8b1dc978b69deccc62cec43d22254a18f4860aae6fd87026e7b86d8f537a302ec5ed34f26efa04e1e29648cabe9dac57e0b452fa9fe6f74f6a22b0ce6b2be04a
master
chris-belcher 6 years ago
parent
commit
c52d1fb0d6
No known key found for this signature in database
GPG Key ID: EF734EA677F31129
  1. 20
      docs/release-notes/release-notes-tumbler-improve.md
  2. 5
      jmclient/jmclient/__init__.py
  3. 2
      jmclient/jmclient/configure.py
  4. 93
      jmclient/jmclient/schedule.py
  5. 17
      jmclient/jmclient/taker.py
  6. 2
      jmclient/jmclient/taker_utils.py
  7. 19
      jmclient/test/test_coinjoin.py
  8. 28
      jmclient/test/test_schedule.py
  9. 40
      jmclient/test/test_taker.py
  10. 54
      scripts/cli_options.py
  11. 25
      scripts/joinmarket-qt.py
  12. 85
      scripts/qtsupport.py
  13. 4
      scripts/sendpayment.py
  14. 11
      scripts/tumbler.py

20
docs/release-notes/release-notes-tumbler-improve.md

@ -0,0 +1,20 @@
copypaste this into "release-notes" when the time comes to make a new release, then delete this file
Notable changes
===============
### Tumbler privacy improvements
The tumbler algorithm has been improved with the aim to increase privacy. This affects the `tumbler.py` script and `joinmarket-qt.py` GUI.
* At the start of the run, tumbler will now fully spend all mixdepths with coinjoin with no change address (also known as a sweep transaction) back to its own internal wallet. After these initial sweeps are done tumbler will continue with the already-existing algorithm of sending coinjoins with randomly-generated amounts.
* Tumbler will now occasionally send a round number of bitcoins, for example `0.20000000` or `0.15000000` instead of `0.24159873`. The default probability of this happening is 25% per coinjoin.
* The default wait time between coinjoins is increased from 30 minutes to 60 minutes.
* The default number of coinjoin counterparties is increased from 6 to 9.
* The default number of coinjoins per mixdepth is decreased from 4 to 2.
For a full discription and reasoning behind the changes see: [Plan to improve the privacy of JoinMarket's tumbler script](https://gist.github.com/chris-belcher/7e92810f07328fdfdef2ce444aad0968)

5
jmclient/jmclient/__init__.py

@ -6,7 +6,8 @@ import logging
from .support import (calc_cj_fee, choose_sweep_orders, choose_orders,
cheapest_order_choose, weighted_order_choose,
rand_norm_array, rand_pow_array, rand_exp_array, select,
rand_norm_array, rand_pow_array, rand_exp_array,
rand_weighted_choice, select,
select_gradual, select_greedy, select_greediest,
get_random_bytes, random_under_max_order_choose,
select_one_utxo)
@ -37,7 +38,7 @@ from .output import generate_podle_error_string, fmt_utxos, fmt_utxo,\
fmt_tx_data
from .schedule import (get_schedule, get_tumble_schedule, schedule_to_text,
tweak_tumble_schedule, human_readable_schedule_entry,
schedule_to_text)
schedule_to_text, NO_ROUNDING)
from .commitment_utils import get_utxo_info, validate_utxo_data, quit
from .taker_utils import (tumbler_taker_finished_update, restart_waiter,
restart_wait, get_tumble_log, direct_send,

2
jmclient/jmclient/configure.py

@ -251,7 +251,7 @@ tx_broadcast = self
# amount of makers which we are content with for the coinjoin to
# succceed. Less makers means that the whole process will restart
# after a timeout.
minimum_makers = 2
minimum_makers = 4
##############################
#THE FOLLOWING SETTINGS ARE REQUIRED TO DEFEND AGAINST SNOOPERS.

93
jmclient/jmclient/schedule.py

@ -3,8 +3,11 @@ from __future__ import (absolute_import, division,
print_function, unicode_literals)
from builtins import * # noqa: F401
import copy
from jmclient import (validate_address, rand_exp_array,
rand_norm_array, rand_pow_array, jm_single)
import random
import sys
from .configure import validate_address, jm_single
from .support import rand_exp_array, rand_norm_array, rand_weighted_choice
"""Utility functions for dealing with Taker schedules.
- get_schedule(filename):
@ -17,6 +20,8 @@ from jmclient import (validate_address, rand_exp_array,
the chance of success on re-trying
"""
NO_ROUNDING = 16 #max btc significant figures not including LN
def get_schedule(filename):
with open(filename, "rb") as f:
schedule = []
@ -26,8 +31,8 @@ def get_schedule(filename):
if sl.startswith("#"):
continue
try:
mixdepth, amount, makercount, destaddr, waittime, completed = \
sl.split(',')
(mixdepth, amount, makercount, destaddr, waittime,
rounding, completed) = sl.split(',')
except ValueError as e:
return (False, "Failed to parse schedule line: " + sl)
try:
@ -41,6 +46,7 @@ def get_schedule(filename):
makercount = int(makercount)
destaddr = destaddr.strip()
waittime = float(waittime)
rounding = int(rounding)
completed = completed.strip()
if not len(completed) == 64:
completed = int(completed)
@ -51,12 +57,11 @@ def get_schedule(filename):
if not success:
return (False, "Invalid address: " + destaddr + "," + errmsg)
schedule.append([mixdepth, amount, makercount, destaddr,
waittime, completed])
waittime, rounding, completed])
return (True, schedule)
def get_amount_fractions(power, count):
"""Get 'count' fractions following power law distn according to
parameter 'power'
def get_amount_fractions(count):
"""Get 'count' fractions following uniform distn
Note that this function is not entirely generic; it ensures that
the final entry is larger than a certain fraction, for a reason
specific to the way the tumbler algo works: the last entry
@ -65,9 +70,16 @@ def get_amount_fractions(power, count):
setting, so we make sure it's appreciable to begin with.
"""
while True:
amount_fractions = rand_pow_array(power, count)
amount_fractions = [1.0 - x for x in amount_fractions]
y = [x / sum(amount_fractions) for x in amount_fractions]
knives = [random.random() for i in range(count-1)]
knives = sorted(knives)[::-1]
y = []
l = 1
k = 1
for k in knives:
y.append( l - k )
l = k
y.append(k)
#Here we insist that the last entry in the list is more
#than 5% of the total, to account for tweaks upwards
#on previous joins.
@ -75,7 +87,7 @@ def get_amount_fractions(power, count):
break
return y
def get_tumble_schedule(options, destaddrs):
def get_tumble_schedule(options, destaddrs, mixdepth_balance_dict):
"""for the general intent and design of the tumbler algo, see the docs in
joinmarket-org/joinmarket.
Alterations:
@ -97,6 +109,30 @@ def get_tumble_schedule(options, destaddrs):
options['txcountparams'][1], options['mixdepthcount'])
txcounts = lower_bounded_int(txcounts, options['mintxcount'])
tx_list = []
### stage 1 coinjoins, which sweep the entire mixdepth without creating change
lowest_initial_filled_mixdepth = sys.maxsize
sweep_mixdepths = []
for mixdepth, balance in mixdepth_balance_dict.items():
if balance > 0:
lowest_initial_filled_mixdepth = min(mixdepth,
lowest_initial_filled_mixdepth)
sweep_mixdepths.append(mixdepth)
waits = rand_exp_array(options['timelambda']*options[
'stage1_timelambda_increase'], len(sweep_mixdepths))
makercounts = rand_norm_array(options['makercountrange'][0],
options['makercountrange'][1], len(sweep_mixdepths))
makercounts = lower_bounded_int(makercounts, options['minmakercount'])
sweep_mixdepths = sorted(sweep_mixdepths)[::-1]
for mixdepth, wait, makercount in zip(sweep_mixdepths, waits, makercounts):
tx = {'amount_fraction': 0,
'wait': round(wait, 2),
'srcmixdepth': mixdepth,
'makercount': makercount,
'destination': 'INTERNAL',
'rounding': NO_ROUNDING
}
tx_list.append(tx)
### stage 2 coinjoins, which create a number of random-amount coinjoins from each mixdepth
for m, txcount in enumerate(txcounts):
if options['mixdepthcount'] - options['addrcount'] <= m and m < \
options['mixdepthcount'] - 1:
@ -104,7 +140,7 @@ def get_tumble_schedule(options, destaddrs):
# amount_fraction cant be 1.0, some coins must be left over
if txcount == 1:
txcount = 2
amount_fractions = get_amount_fractions(options['amountpower'], txcount)
amount_fractions = get_amount_fractions(txcount)
# transaction times are uncorrelated
# time between events in a poisson process followed exp
waits = rand_exp_array(options['timelambda'], txcount)
@ -112,23 +148,31 @@ def get_tumble_schedule(options, destaddrs):
makercounts = rand_norm_array(options['makercountrange'][0],
options['makercountrange'][1], txcount)
makercounts = lower_bounded_int(makercounts, options['minmakercount'])
for amount_fraction, wait, makercount in zip(amount_fractions, waits,
makercounts):
do_rounds = [random.random() < options['rounding_chance'] for _ in range(txcount)]
for amount_fraction, wait, makercount, do_round in zip(amount_fractions, waits,
makercounts, do_rounds):
rounding = NO_ROUNDING
if do_round:
weight_sum = 1.0*sum(options['rounding_sigfig_weights'])
weight_prob = [a/weight_sum for a in options['rounding_sigfig_weights']]
rounding = rand_weighted_choice(len(weight_prob), weight_prob) + 1
tx = {'amount_fraction': amount_fraction,
'wait': round(wait, 2),
'srcmixdepth': m + options['mixdepthsrc'],
'srcmixdepth': lowest_initial_filled_mixdepth + m + options['mixdepthsrc'] + 1,
'makercount': makercount,
'destination': 'INTERNAL'}
'destination': 'INTERNAL',
'rounding': rounding
}
tx_list.append(tx)
#reset the final amt_frac to zero, as it's the last one for this mixdepth:
tx_list[-1]['amount_fraction'] = 0
tx_list[-1]['rounding'] = NO_ROUNDING
addrask = options['addrcount'] - len(destaddrs)
external_dest_addrs = ['addrask'] * addrask + destaddrs
external_dest_addrs = ['addrask'] * addrask + destaddrs[::-1]
for mix_offset in range(options['addrcount']):
srcmix = (options['mixdepthsrc'] + options['mixdepthcount'] -
mix_offset - 1)
srcmix = (lowest_initial_filled_mixdepth + options['mixdepthsrc']
+ options['mixdepthcount'] - mix_offset)
for tx in reversed(tx_list):
if tx['srcmixdepth'] == srcmix:
tx['destination'] = external_dest_addrs[mix_offset]
@ -146,7 +190,8 @@ def get_tumble_schedule(options, destaddrs):
schedule = []
for t in tx_list:
schedule.append([t['srcmixdepth'], t['amount_fraction'],
t['makercount'], t['destination'], t['wait'], 0])
t['makercount'], t['destination'], t['wait'],
t['rounding'], 0])
return schedule
def tweak_tumble_schedule(options, schedule, last_completed, destaddrs=[]):
@ -193,7 +238,7 @@ def tweak_tumble_schedule(options, schedule, last_completed, destaddrs=[]):
alreadyspent = sum([x[1] for x in already_done])
tobespent = 1.0 - alreadyspent
#power law for what's left:
new_fracs = get_amount_fractions(options['amountpower'], len(tobedone))
new_fracs = get_amount_fractions(len(tobedone))
#rescale; the sum must be 'tobespent':
new_fracs = [x*tobespent for x in new_fracs]
#starting from the known 'last_completed+1' index, apply these new
@ -211,6 +256,8 @@ def human_readable_schedule_entry(se, amt=None, destn=None):
amt_info = str(amt) if amt else str(se[1])
hrs.append("sends amount: " + amt_info + " satoshis")
dest_info = destn if destn else str(se[3])
hrs.append(("rounded to " + str(se[5]) + " significant figures"
if se[5] != NO_ROUNDING else "without rounding"))
hrs.append("to destination address: " + dest_info)
hrs.append("after coinjoin with " + str(se[2]) + " counterparties.")
return ", ".join(hrs)

17
jmclient/jmclient/taker.py

@ -21,6 +21,7 @@ from jmclient.podle import generate_podle, get_podle_commitments, PoDLE
from jmclient.wallet_service import WalletService
from .output import generate_podle_error_string
from .cryptoengine import EngineError
from .schedule import NO_ROUNDING
jlog = get_log()
@ -168,6 +169,7 @@ class Taker(object):
si = self.schedule[self.schedule_index]
self.mixdepth = si[0]
self.cjamount = si[1]
rounding = si[5]
#non-integer coinjoin amounts are treated as fractions
#this is currently used by the tumbler algo
if isinstance(self.cjamount, float):
@ -179,6 +181,9 @@ class Taker(object):
)[self.mixdepth]
#reset to satoshis
self.cjamount = int(self.cjamount * self.mixdepthbal)
if rounding != NO_ROUNDING:
self.cjamount = round_to_significant_figures(self.cjamount,
rounding)
if self.cjamount < jm_single().mincjamount:
jlog.info("Coinjoin amount too low, bringing up to: " + str(
jm_single().mincjamount))
@ -188,7 +193,7 @@ class Taker(object):
# for sweeps to external addresses we need an in-wallet import
# for the transaction monitor (this will be a no-op for txs to
# in-wallet addresses).
if self.cjamount == 0:
if self.cjamount == 0 and self.my_cj_addr != "INTERNAL":
self.wallet_service.import_non_wallet_address(self.my_cj_addr)
#if destination is flagged "INTERNAL", choose a destination
@ -1222,3 +1227,13 @@ class P2EPTaker(Taker):
self.self_sign_and_push()
# returning False here is not an error condition, only stops processing.
return (False, "OK")
def round_to_significant_figures(d, sf):
'''Rounding number d to sf significant figures in base 10'''
for p in range(-10, 15):
power10 = 10**p
if power10 > d:
sf_power10 = 10**sf
sigfiged = int(round(d/power10*sf_power10)*power10/sf_power10)
return sigfiged
raise RuntimeError()

2
jmclient/jmclient/taker_utils.py

@ -196,7 +196,7 @@ def unconf_update(taker, schedulefile, tumble_log, addtolog=False):
#full record should always be accurate; but TUMBLE.log should be
#used for checking what actually happened.
completion_flag = 1 if not addtolog else taker.txid
taker.schedule[taker.schedule_index][5] = completion_flag
taker.schedule[taker.schedule_index][-1] = completion_flag
with open(schedulefile, "wb") as f:
f.write(schedule_to_text(taker.schedule))

19
jmclient/test/test_coinjoin.py

@ -13,7 +13,8 @@ from twisted.internet import reactor
from jmbase import get_log
from jmclient import load_program_config, jm_single,\
YieldGeneratorBasic, Taker, LegacyWallet, SegwitLegacyWallet
YieldGeneratorBasic, Taker, LegacyWallet, SegwitLegacyWallet,\
NO_ROUNDING
from jmclient.podle import set_commitment_file
from commontest import make_wallets, binarize_tx
from test_taker import dummy_filter_orderbook
@ -139,8 +140,8 @@ def test_simple_coinjoin(monkeypatch, tmpdir, setup_cj, wallet_cls):
assert len(orderbook) == MAKER_NUM
cj_amount = int(1.1 * 10**8)
# mixdepth, amount, counterparties, dest_addr, waittime
schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0)]
# mixdepth, amount, counterparties, dest_addr, waittime, rounding
schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0, NO_ROUNDING)]
taker = create_taker(wallet_services[-1], schedule, monkeypatch)
active_orders, maker_data = init_coinjoin(taker, makers,
@ -183,8 +184,8 @@ def test_coinjoin_mixdepth_wrap_taker(monkeypatch, tmpdir, setup_cj):
assert len(orderbook) == MAKER_NUM
cj_amount = int(1.1 * 10**8)
# mixdepth, amount, counterparties, dest_addr, waittime
schedule = [(4, cj_amount, MAKER_NUM, 'INTERNAL', 0)]
# mixdepth, amount, counterparties, dest_addr, waittime, rounding
schedule = [(4, cj_amount, MAKER_NUM, 'INTERNAL', 0, NO_ROUNDING)]
taker = create_taker(wallet_services[-1], schedule, monkeypatch)
active_orders, maker_data = init_coinjoin(taker, makers,
@ -239,8 +240,8 @@ def test_coinjoin_mixdepth_wrap_maker(monkeypatch, tmpdir, setup_cj):
assert len(orderbook) == MAKER_NUM
cj_amount = int(1.1 * 10**8)
# mixdepth, amount, counterparties, dest_addr, waittime
schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0)]
# mixdepth, amount, counterparties, dest_addr, waittime, rounding
schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0, NO_ROUNDING)]
taker = create_taker(wallet_services[-1], schedule, monkeypatch)
active_orders, maker_data = init_coinjoin(taker, makers,
@ -301,8 +302,8 @@ def test_coinjoin_mixed_maker_addresses(monkeypatch, tmpdir, setup_cj,
orderbook = create_orderbook(makers)
cj_amount = int(1.1 * 10**8)
# mixdepth, amount, counterparties, dest_addr, waittime
schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0)]
# mixdepth, amount, counterparties, dest_addr, waittime, rounding
schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0, NO_ROUNDING)]
taker = create_taker(wallet_services[-1], schedule, monkeypatch)
active_orders, maker_data = init_coinjoin(taker, makers,

28
jmclient/test/test_schedule.py

@ -10,30 +10,30 @@ from jmclient import (get_schedule, get_tumble_schedule,
import os
valids = """#sample for testing
1, 110000000, 3, INTERNAL, 0, 1
0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw, 9.88, 0
1, 110000000, 3, INTERNAL, 0, 16, 1
0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw, 9.88, 16, 0
"""
invalids1 = """#sample for testing
1, 110000000, 3, 5, INTERNAL, 0
1, 110000000, 3, 5, INTERNAL, 16, 0
#pointless comment here; following line has trailing spaces
0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw ,0, 0,
0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw ,0, 16, 0,
"""
invalids2 = """#sample for testing
1, 110000000, notinteger, INTERNAL, 0, 0
0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw, 0, 0
1, 110000000, notinteger, INTERNAL, 0, 16, 0
0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw, 0, 16, 0
"""
invalids3 = """#sample for testing
1, 110000000, 3, INTERNAL, 0, 0
0, notinteger, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw, 0, 0
1, 110000000, 3, INTERNAL, 0, 16, 0
0, notinteger, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw, 0, 16, 0
"""
#invalid address
invalids4 = """#sample for testing
1, 110000000, 3, INTERNAL, 0, 0
0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qq, 0, 0
1, 110000000, 3, INTERNAL, 0, 16, 0
0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qq, 0, 16, 0
"""
@ -65,11 +65,13 @@ def get_options():
options.txfee = 5000
options.addrcount = 3
options.mintxcount = 1
options.amountpower = 100
options.timelambda = 0.2
options.waittime = 10
options.stage1_timelambda_increase = 3
options.mincjamount = 1000000
options.liquiditywait = 5
options.rounding_chance = 0.25
options.rounding_sigfig_weights = (55, 15, 25, 65, 40)
options = vars(options)
return options
@ -88,7 +90,7 @@ def test_tumble_schedule(destaddrs, txcparams, mixdepthcount):
options = get_options()
options['mixdepthcount'] = mixdepthcount
options['txcountparams'] = txcparams
schedule = get_tumble_schedule(options, destaddrs)
schedule = get_tumble_schedule(options, destaddrs, {0:1})
dests = [x[3] for x in schedule]
assert set(destaddrs).issubset(set(dests))
@ -126,7 +128,7 @@ def test_tumble_tweak(destaddrs, txcparams, mixdepthcount, lastcompleted,
options['mixdepthcount'] = mixdepthcount
options['txcountparams'] = txcparams
options['makercountrange'] = makercountrange
schedule = get_tumble_schedule(options, destaddrs)
schedule = get_tumble_schedule(options, destaddrs, {0:1})
dests = [x[3] for x in schedule]
assert set(destaddrs).issubset(set(dests))
new_schedule = tweak_tumble_schedule(options, schedule, lastcompleted)

40
jmclient/test/test_taker.py

@ -15,7 +15,7 @@ import struct
from base64 import b64encode
from jmclient import load_program_config, jm_single, set_commitment_file,\
get_commitment_file, SegwitLegacyWallet, Taker, VolatileStorage,\
get_network, WalletService
get_network, WalletService, NO_ROUNDING
from taker_test_data import t_utxos_by_mixdepth, t_orderbook,\
t_maker_response, t_chosen_orders, t_dummy_ext
@ -124,7 +124,7 @@ def get_taker(schedule=None, schedule_len=0, on_finished=None,
filter_orders=None):
if not schedule:
#note, for taker.initalize() this will result in junk
schedule = [['a', 'b', 'c', 'd', 'e']]*schedule_len
schedule = [['a', 'b', 'c', 'd', 'e', 'f']]*schedule_len
print("Using schedule: " + str(schedule))
on_finished_callback = on_finished if on_finished else taker_finished
filter_orders_callback = filter_orders if filter_orders else dummy_filter_orderbook
@ -138,11 +138,11 @@ def test_filter_rejection(setup_taker):
print("calling filter orders rejection")
return False
taker = get_taker(filter_orders=filter_orders_reject)
taker.schedule = [[0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0]]
taker.schedule = [[0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING]]
res = taker.initialize(t_orderbook)
assert not res[0]
taker = get_taker(filter_orders=filter_orders_reject)
taker.schedule = [[0, 0, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0]]
taker.schedule = [[0, 0, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING]]
res = taker.initialize(t_orderbook)
assert not res[0]
@ -171,7 +171,7 @@ def test_make_commitment(setup_taker, failquery, external):
jm_single().config.set("POLICY", "taker_utxo_amtpercent", "20")
mixdepth = 0
amount = 110000000
taker = get_taker([(mixdepth, amount, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw")])
taker = get_taker([(mixdepth, amount, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", NO_ROUNDING)])
taker.cjamount = amount
taker.input_utxos = t_utxos_by_mixdepth[0]
if failquery:
@ -180,7 +180,7 @@ def test_make_commitment(setup_taker, failquery, external):
clean_up()
def test_not_found_maker_utxos(setup_taker):
taker = get_taker([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)])
taker = get_taker([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)])
orderbook = copy.deepcopy(t_orderbook)
res = taker.initialize(orderbook)
taker.orderbook = copy.deepcopy(t_chosen_orders) #total_cjfee unaffected, all same
@ -192,7 +192,7 @@ def test_not_found_maker_utxos(setup_taker):
jm_single().bc_interface.setQUSFail(False)
def test_auth_pub_not_found(setup_taker):
taker = get_taker([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)])
taker = get_taker([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)])
orderbook = copy.deepcopy(t_orderbook)
res = taker.initialize(orderbook)
taker.orderbook = copy.deepcopy(t_chosen_orders) #total_cjfee unaffected, all same
@ -214,33 +214,33 @@ def test_auth_pub_not_found(setup_taker):
@pytest.mark.parametrize(
"schedule, highfee, toomuchcoins, minmakers, notauthed, ignored, nocommit",
[
([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw")], False, False,
([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False,
2, False, None, None),
([(0, 0, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)], False, False,
([(0, 0, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False,
2, False, None, None), #sweep
([(0, 0.2, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)], False, False,
([(0, 0.2, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False,
2, False, None, None), #tumble style non-int amounts
#edge case triggers that don't fail
([(0, 0, 4, "mxeLuX8PP7qLkcM8uarHmdZyvP1b5e1Ynf", 0)], False, False,
([(0, 0, 4, "mxeLuX8PP7qLkcM8uarHmdZyvP1b5e1Ynf", 0, NO_ROUNDING)], False, False,
2, False, None, None), #sweep rounding error case
([(0, 199850001, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)], False, False,
([(0, 199850001, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False,
2, False, None, None), #trigger sub dust change for taker
#edge case triggers that do fail
([(0, 199850000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)], False, False,
([(0, 199850000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False,
2, False, None, None), #trigger negative change
([(0, 199599800, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)], False, False,
([(0, 199599800, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False,
2, False, None, None), #trigger sub dust change for maker
([(0, 20000000, 3, "INTERNAL", 0)], True, False,
([(0, 20000000, 3, "INTERNAL", 0, NO_ROUNDING)], True, False,
2, False, None, None), #test high fee
([(0, 20000000, 3, "INTERNAL", 0)], False, False,
([(0, 20000000, 3, "INTERNAL", 0, NO_ROUNDING)], False, False,
7, False, None, None), #test not enough cp
([(0, 80000000, 3, "INTERNAL", 0)], False, False,
([(0, 80000000, 3, "INTERNAL", 0, NO_ROUNDING)], False, False,
2, False, None, "30000"), #test failed commit
([(0, 20000000, 3, "INTERNAL", 0)], False, False,
([(0, 20000000, 3, "INTERNAL", 0, NO_ROUNDING)], False, False,
2, True, None, None), #test unauthed response
([(0, 5000000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)], False, True,
([(0, 5000000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, True,
2, False, None, None), #test too much coins
([(0, 0, 5, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0)], False, False,
([(0, 0, 5, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False,
2, False, ["J659UPUSLLjHJpaB", "J65z23xdjxJjC7er", 0], None), #test inadequate for sweep
])
def test_taker_init(setup_taker, schedule, highfee, toomuchcoins, minmakers,

54
scripts/cli_options.py

@ -269,9 +269,9 @@ def get_tumbler_parser():
action='store',
dest='makercountrange',
help=
'Input the mean and spread of number of makers to use. e.g. 6 1 will be a normal distribution '
'with mean 6 and standard deviation 1 inclusive, default=6 1 (floats are also OK)',
default=(6, 1))
'Input the mean and spread of number of makers to use. e.g. 9 1 will be a normal distribution '
'with mean 9 and standard deviation 1 inclusive, default=9 1 (floats are also OK)',
default=(9, 1))
parser.add_option(
'--minmakercount',
type='int',
@ -292,17 +292,17 @@ def get_tumbler_parser():
type='float',
nargs=2,
dest='txcountparams',
default=(4, 1),
default=(2, 1),
help=
'The number of transactions to take coins from one mixing depth to the next, it is'
' randomly chosen following a normal distribution. Should be similar to --addrask. '
'This option controls the parameters of the normal distribution curve. (mean, standard deviation). default=4 1')
'This option controls the parameters of the normal distribution curve. (mean, standard deviation). default=2 1')
parser.add_option(
'--mintxcount',
type='int',
dest='mintxcount',
default=1,
help='The minimum transaction count per mixing level, default=1')
default=2,
help='The minimum transaction count per mixing level, default=2')
parser.add_option(
'--donateamount',
type='float',
@ -310,23 +310,25 @@ def get_tumbler_parser():
default=0,
help=
'percent of funds to donate to joinmarket development, or zero to opt out (default=0%)')
parser.add_option(
'--amountpower',
type='float',
dest='amountpower',
default=100.0,
help=
'The output amounts follow a power law distribution, this is the power, default=100.0')
parser.add_option(
'-l',
'--timelambda',
type='float',
dest='timelambda',
default=30,
default=60,
help=
'Average the number of minutes to wait between transactions. Randomly chosen '
' following an exponential distribution, which describes the time between uncorrelated'
' events. default=30')
' events. default=60')
parser.add_option(
'--stage1-timelambda-increase',
type='float',
dest='stage1_timelambda_increase',
default=3,
help=
'Stage 1 sweep coinjoins have a longer wait time. This parameter'
' controls by what factor longer is this average wait time compared to stage2 coinjoins'
' which are controlled by `--timelambda`, default=3')
parser.add_option(
'-w',
'--wait-time',
@ -376,6 +378,24 @@ def get_tumbler_parser():
'mixdepthsrc + number of mixdepths to tumble '
'have been used.',
default=-1)
parser.add_option(
'--rounding-chance',
action='store',
type='float',
dest='rounding_chance',
help='probability of non-sweep coinjoin amount being rounded, default=0.25 (25%)',
default=0.25)
parser.add_option(
'--rounding-sigfig-weights',
type='float',
nargs=5,
dest='rounding_sigfig_weights',
default=(55, 15, 25, 65, 40),
help=
"If rounding happens (determined by --rounding-chance) then the weights of how many"
" significant figures to round to. The five values refer to the probability of"
" rounding to one, two, three, four and five significant figures respectively."
" default=(55, 15, 25, 65, 40)")
add_common_options(parser)
return parser
@ -404,7 +424,7 @@ def get_sendpayment_parser():
type='int',
dest='makercount',
help='how many makers to coinjoin with, default random from 4 to 6',
default=random.randint(4, 6))
default=random.randint(8, 10))
parser.add_option('-S',
'--schedule-file',
type='string',

25
scripts/joinmarket-qt.py

@ -74,7 +74,8 @@ from jmclient import load_program_config, get_network, update_persist_config,\
get_blockchain_interface_instance, direct_send, WalletService,\
RegtestBitcoinCoreInterface, tumbler_taker_finished_update,\
get_tumble_log, restart_wait, tumbler_filter_orders_callback,\
wallet_generate_recover_bip39, wallet_display, get_utxos_enabled_disabled
wallet_generate_recover_bip39, wallet_display, get_utxos_enabled_disabled,\
NO_ROUNDING
from qtsupport import ScheduleWizard, TumbleRestartWizard, config_tips,\
config_types, QtHandler, XStream, Buttons, OkButton, CancelButton,\
PasswordDialog, MyTreeWidget, JMQtMessageBox, BLUE_FG,\
@ -126,7 +127,7 @@ def getSettingsWidgets():
sMM = ['', (2, 20),
(0, jm_single().config.getint("GUI", "max_mix_depth") - 1),
(0.00000001, 100.0, 8)]
sD = ['', '3', '0', '']
sD = ['', '9', '0', '']
for x in zip(sN, sH, sT, sD, sMM):
ql = QLabel(x[0])
ql.setToolTip(x[1])
@ -320,20 +321,24 @@ class SpendTab(QWidget):
self.spendstate.reset() #trigger callback to 'ready' state
def generateTumbleSchedule(self):
if not mainWindow.wallet_service:
JMQtMessageBox(self, "Cannot start without a loaded wallet.",
mbtype="crit", title="Error")
return
#needs a set of tumbler options and destination addresses, so needs
#a wizard
wizard = ScheduleWizard()
wizard_return = wizard.exec_()
if wizard_return == QDialog.Rejected:
return
self.spendstate.loaded_schedule = wizard.get_schedule()
self.spendstate.loaded_schedule = wizard.get_schedule(
mainWindow.wallet_service.get_balance_by_mixdepth())
self.spendstate.schedule_name = wizard.get_name()
self.updateSchedView()
self.tumbler_options = wizard.opts
self.tumbler_destaddrs = wizard.get_destaddrs()
#tumbler may require more mixdepths; update the wallet
required_mixdepths = self.tumbler_options['mixdepthsrc'] + \
self.tumbler_options['mixdepthcount']
required_mixdepths = max([tx[0] for tx in self.spendstate.loaded_schedule])
if required_mixdepths > jm_single().config.getint("GUI", "max_mix_depth"):
jm_single().config.set("GUI", "max_mix_depth", str(required_mixdepths))
#recreate wallet and sync again; needed due to cache.
@ -553,17 +558,17 @@ class SpendTab(QWidget):
#follow restart logic
#1. filter out complete:
self.spendstate.loaded_schedule = [
s for s in self.spendstate.loaded_schedule if s[5] != 1]
s for s in self.spendstate.loaded_schedule if s[-1] != 1]
#reload destination addresses
self.tumbler_destaddrs = [x[3] for x in self.spendstate.loaded_schedule
if x not in ["INTERNAL", "addrask"]]
#2 Check for unconfirmed
if isinstance(self.spendstate.loaded_schedule[0][5], str) and len(
self.spendstate.loaded_schedule[0][5]) == 64:
if isinstance(self.spendstate.loaded_schedule[0][-1], str) and len(
self.spendstate.loaded_schedule[0][-1]) == 64:
#ensure last transaction is confirmed before restart
tumble_log.info("WAITING TO RESTART...")
mainWindow.statusBar().showMessage("Waiting for confirmation to restart..")
txid = self.spendstate.loaded_schedule[0][5]
txid = self.spendstate.loaded_schedule[0][-1]
#remove the already-done entry (this connects to the other TODO,
#probably better *not* to truncate the done-already txs from file,
#but simplest for now.
@ -639,7 +644,7 @@ class SpendTab(QWidget):
#note 'amount' is integer, so not interpreted as fraction
#see notes in sample testnet schedule for format
self.spendstate.loaded_schedule = [[mixdepth, amount, makercount,
destaddr, 0, 0]]
destaddr, 0, NO_ROUNDING, 0]]
self.spendstate.updateType('single')
self.spendstate.updateRun('running')
self.startJoin()

85
scripts/qtsupport.py

@ -553,7 +553,7 @@ class SchDynamicPage1(QWizardPage):
sMM = [(0, jm_single().config.getint("GUI", "max_mix_depth") - 1), (3, 20),
(2, 7), (0.00000001, 100.0, 8), (2, 10), (0.000001, 0.25, 6),
(0, 10000000)]
sD = ['0', '6', '4', '30.0', '4', '0.005', '10000']
sD = ['0', '9', '4', '60.0', '2', '0.005', '10000']
for x in zip(sN, sH, sT, sD, sMM):
ql = QLabel(x[0])
ql.setToolTip(x[1])
@ -623,28 +623,39 @@ class SchFinishPage(QWizardPage):
layout.setSpacing(4)
results = []
sN = ['Makercount sdev', 'Tx count sdev',
'Amount power',
sN = ['Makercount sdev',
'Tx count sdev',
'Minimum maker count',
'Minimum transaction count',
'Min coinjoin amount',
'wait time']
'Response wait time',
'Stage 1 transaction wait time increase',
'Rounding Chance']
for w in ["One", "Two", "Three", "Four", "Five"]:
sN += [w + " significant figures rounding weight"]
#Tooltips
sH = ["Standard deviation of the number of makers to use in each "
"transaction.",
"Standard deviation of the number of transactions to use in each "
"mixdepth",
"A parameter to control the random coinjoin sizes.",
"The lowest allowed number of maker counterparties.",
"The lowest allowed number of transactions in one mixdepth.",
"The lowest allowed size of any coinjoin, in satoshis.",
"The time in seconds to wait for response from counterparties."]
"transaction.",
"Standard deviation of the number of transactions to use in each "
"mixdepth",
"The lowest allowed number of maker counterparties.",
"The lowest allowed number of transactions in one mixdepth.",
"The lowest allowed size of any coinjoin, in satoshis.",
"The time in seconds to wait for response from counterparties.",
"The factor increase in wait time for stage 1 sweep coinjoins",
"The probability of non-sweep coinjoin amounts being rounded"]
for w in ["one", "two", "three", "four", "five"]:
sH += ["If rounding happens (determined by Rounding Chance) then this "
"is the relative probability of rounding to " + w +
" significant figures"]
#types
sT = [float, float, float, int, int, int, float]
sT = [float, float, int, int, int, float, float, float] + [int]*5
#constraints
sMM = [(0.0, 10.0, 2), (0.0, 10.0, 2), (1.0, 10000.0, 1), (2,20),
(1, 10), (100000, 100000000), (10.0, 500.0, 2)]
sD = ['1.0', '1.0', '100.0', '2', '1', '1000000', '20']
sMM = [(0.0, 10.0, 2), (0.0, 10.0, 2), (2,20),
(1, 10), (100000, 100000000), (10.0, 500.0, 2), (0, 100, 1),
(0.0, 1.0, 3)] + [(0, 10000)]*5
sD = ['1.0', '1.0', '2', '2', '1000000', '20', '3', '0.25'] +\
['55', '15', '25', '65', '40']
for x in zip(sN, sH, sT, sD, sMM):
ql = QLabel(x[0])
ql.setToolTip(x[1])
@ -663,11 +674,14 @@ class SchFinishPage(QWizardPage):
#fields not considered 'mandatory' as defaults are accepted
self.registerField("makercountsdev", results[0][1])
self.registerField("txcountsdev", results[1][1])
self.registerField("amountpower", results[2][1])
self.registerField("minmakercount", results[3][1])
self.registerField("mintxcount", results[4][1])
self.registerField("mincjamount", results[5][1])
self.registerField("waittime", results[6][1])
self.registerField("minmakercount", results[2][1])
self.registerField("mintxcount", results[3][1])
self.registerField("mincjamount", results[4][1])
self.registerField("waittime", results[5][1])
self.registerField("stage1_timelambda_increase", results[6][1])
self.registerField("rounding_chance", results[7][1])
for i in range(5):
self.registerField("rounding_sigfig_weight_" + str(i+1), results[8+i][1])
class SchIntroPage(QWizardPage):
def __init__(self, parent):
@ -714,7 +728,7 @@ class ScheduleWizard(QWizard):
def get_destaddrs(self):
return self.destaddrs
def get_schedule(self):
def get_schedule(self, wallet_balance_by_mixdepth):
self.destaddrs = []
for i in range(self.page(2).required_addresses):
daddrstring = str(self.field("destaddr"+str(i)))
@ -735,16 +749,19 @@ class ScheduleWizard(QWizard):
self.opts['txcountparams'] = (int(self.field("txcountparams")),
float(self.field("txcountsdev")))
self.opts['mintxcount'] = int(self.field("mintxcount"))
self.opts['amountpower'] = float(self.field("amountpower"))
self.opts['timelambda'] = float(self.field("timelambda"))
self.opts['waittime'] = float(self.field("waittime"))
self.opts["stage1_timelambda_increase"] = float(self.field("stage1_timelambda_increase"))
self.opts['mincjamount'] = int(self.field("mincjamount"))
relfeeval = float(self.field("maxrelfee"))
absfeeval = int(self.field("maxabsfee"))
self.opts['maxcjfee'] = (relfeeval, absfeeval)
#needed for Taker to check:
self.opts['rounding_chance'] = float(self.field("rounding_chance"))
self.opts['rounding_sigfig_weights'] = tuple([int(self.field("rounding_sigfig_weight_" + str(i+1))) for i in range(5)])
jm_single().mincjamount = self.opts['mincjamount']
return get_tumble_schedule(self.opts, self.destaddrs)
return get_tumble_schedule(self.opts, self.destaddrs,
wallet_balance_by_mixdepth)
class TumbleRestartWizard(QWizard):
def __init__(self):
@ -754,7 +771,6 @@ class TumbleRestartWizard(QWizard):
def getOptions(self):
self.opts = {}
self.opts['amountpower'] = float(self.field("amountpower"))
self.opts['mincjamount'] = int(self.field("mincjamount"))
relfeeval = float(self.field("maxrelfee"))
absfeeval = int(self.field("maxabsfee"))
@ -773,21 +789,19 @@ class RestartSettingsPage(QWizardPage):
layout.setSpacing(4)
results = []
sN = ['Amount power',
'Min coinjoin amount',
sN = ['Min coinjoin amount',
'Max relative fee per counterparty (e.g. 0.005)',
'Max fee per counterparty, satoshis (e.g. 10000)']
#Tooltips
sH = ["A parameter to control the random coinjoin sizes.",
"The lowest allowed size of any coinjoin, in satoshis.",
sH = ["The lowest allowed size of any coinjoin, in satoshis.",
"A decimal fraction (e.g. 0.001 = 0.1%) (this AND next must be violated to reject",
"Integer number of satoshis (this AND previous must be violated to reject)"]
#types
sT = [float, int, float, int]
sT = [int, float, int]
#constraints
sMM = [(1.0, 10000.0, 1), (100000, 100000000), (0.000001, 0.25, 6),
sMM = [(100000, 100000000), (0.000001, 0.25, 6),
(0, 10000000)]
sD = ['100.0', '1000000', '0.0005', '10000']
sD = ['1000000', '0.0005', '10000']
for x in zip(sN, sH, sT, sD, sMM):
ql = QLabel(x[0])
ql.setToolTip(x[1])
@ -804,7 +818,6 @@ class RestartSettingsPage(QWizardPage):
layout.addWidget(x[1], i + 1, 1, 1, 2)
self.setLayout(layout)
#fields not considered 'mandatory' as defaults are accepted
self.registerField("amountpower", results[0][1])
self.registerField("mincjamount", results[1][1])
self.registerField("maxrelfee", results[2][1])
self.registerField("maxabsfee", results[3][1])
self.registerField("mincjamount", results[0][1])
self.registerField("maxrelfee", results[1][1])
self.registerField("maxabsfee", results[2][1])

4
scripts/sendpayment.py

@ -17,7 +17,7 @@ import pprint
from jmclient import Taker, P2EPTaker, load_program_config, get_schedule,\
JMClientProtocolFactory, start_reactor, validate_address, jm_single,\
estimate_tx_fee, direct_send, WalletService,\
open_test_wallet_maybe, get_wallet_path
open_test_wallet_maybe, get_wallet_path, NO_ROUNDING
from twisted.python.log import startLogging
from jmbase.support import get_log, set_logging_level, jmprint, \
EXIT_FAILURE, EXIT_ARGERROR
@ -77,7 +77,7 @@ def main():
jmprint('ERROR: Address invalid. ' + errormsg, "error")
sys.exit(EXIT_ARGERROR)
schedule = [[options.mixdepth, amount, options.makercount,
destaddr, 0.0, 0]]
destaddr, 0.0, NO_ROUNDING, 0]]
else:
if options.p2ep:
parser.error("Schedule files are not compatible with PayJoin")

11
scripts/tumbler.py

@ -77,7 +77,7 @@ def main():
jmprint("Error was: " + str(schedule), "error")
sys.exit(EXIT_FAILURE)
#This removes all entries that are marked as done
schedule = [s for s in schedule if s[5] != 1]
schedule = [s for s in schedule if s[-1] != 1]
# remaining destination addresses must be stored in Taker.tdestaddrs
# in case of tweaks; note we can't change, so any passed on command
# line must be ignored:
@ -90,16 +90,16 @@ def main():
destaddrs = [s[3] for s in schedule if s[3] not in ["INTERNAL", "addrask"]]
jmprint("Remaining destination addresses in restart: " + ",".join(destaddrs),
"important")
if isinstance(schedule[0][5], str) and len(schedule[0][5]) == 64:
if isinstance(schedule[0][-1], str) and len(schedule[0][-1]) == 64:
#ensure last transaction is confirmed before restart
tumble_log.info("WAITING TO RESTART...")
txid = schedule[0][5]
txid = schedule[0][-1]
restart_waiter(txid)
#remove the already-done entry (this connects to the other TODO,
#probably better *not* to truncate the done-already txs from file,
#but simplest for now.
schedule = schedule[1:]
elif schedule[0][5] != 0:
elif schedule[0][-1] != 0:
print("Error: first schedule entry is invalid.")
sys.exit(EXIT_FAILURE)
with open(os.path.join(logsdir, options['schedulefile']), "wb") as f:
@ -107,7 +107,8 @@ def main():
tumble_log.info("TUMBLE RESTARTING")
else:
#Create a new schedule from scratch
schedule = get_tumble_schedule(options, destaddrs)
schedule = get_tumble_schedule(options, destaddrs,
wallet.get_balance_by_mixdepth())
tumble_log.info("TUMBLE STARTING")
with open(os.path.join(logsdir, options['schedulefile']), "wb") as f:
f.write(schedule_to_text(schedule))

Loading…
Cancel
Save