Browse Source

Merge #166: add random-under-max order chooser

a2c74ee add random-under-max order chooser (undeath)
master
AdamISZ 7 years ago
parent
commit
2d434ee3ed
No known key found for this signature in database
GPG Key ID: B3AE09F1E9A3197A
  1. 19
      jmclient/jmclient/configure.py
  2. 39
      jmclient/jmclient/support.py
  3. 11
      jmclient/jmclient/taker.py
  4. 7
      jmclient/jmclient/taker_utils.py
  5. 1
      jmclient/test/test_schedule.py
  6. 251
      scripts/cli_options.py
  7. 18
      scripts/sendpayment.py
  8. 19
      scripts/tumbler.py

19
jmclient/jmclient/configure.py

@ -163,11 +163,13 @@ confirm_timeout_hours = 6
[POLICY]
#Use segwit style wallets and transactions
segwit = true
# for dust sweeping, try merge_algorithm = gradual
# for more rapid dust sweeping, try merge_algorithm = greedy
# for most rapid dust sweeping, try merge_algorithm = greediest
# but don't forget to bump your miner fees!
merge_algorithm = default
# the fee estimate is based on a projection of how many satoshis
# per kB are needed to get in one of the next N blocks, N set here
# as the value of 'tx_fees'. This estimate is high if you set N=1,
@ -178,12 +180,26 @@ merge_algorithm = default
# example: N=30000 will use 30000 sat/kB as a fee, while N=5
# will use the estimate from your selected blockchain source
tx_fees = 3
# For users getting transaction fee estimates over an API,
# place a sanity check limit on the satoshis-per-kB to be paid.
# This limit is also applied to users using Core, even though
# Core has its own sanity check limit, which is currently
# 1,000,000 satoshis.
absurd_fee_per_kb = 350000
# Maximum absolute coinjoin fee in satoshi to pay to a single
# market maker for a transaction. Both the limits given in
# max_cj_fee_abs and max_cj_fee_rel must be exceeded in order
# to not consider a certain offer.
#max_cj_fee_abs = x
# Maximum relative coinjoin fee, in fractions of the coinjoin value
# e.g. if your coinjoin amount is 2 btc (200000000 satoshi) and
# max_cj_fee_rel = 0.001 (0.1%), the maximum fee allowed would
# be 0.002 btc (200000 satoshi)
#max_cj_fee_rel = x
# the range of confirmations passed to the `listunspent` bitcoind RPC call
# 1st value is the inclusive minimum, defaults to one confirmation
# 2nd value is the exclusive maximum, defaults to most-positive-bignum (Google Me!)
@ -206,8 +222,11 @@ absurd_fee_per_kb = 350000
# not-self = never broadcast with your own ip
tx_broadcast = self
minimum_makers = 2
##############################
#THE FOLLOWING SETTINGS ARE REQUIRED TO DEFEND AGAINST SNOOPERS.
#DON'T ALTER THEM UNLESS YOU UNDERSTAND THE IMPLICATIONS.
##############################
# number of retries allowed for a specific utxo, to prevent DOS/snooping.
# Lower settings make snooping more expensive, but also prevent honest users

39
jmclient/jmclient/support.py

@ -203,6 +203,11 @@ def weighted_order_choose(orders, n):
return orders[chosen_order_index]
def random_under_max_order_choose(orders, n):
# orders are already pre-filtered for max_cj_fee
return random.choice(orders)
def cheapest_order_choose(orders, n):
"""
Return the cheapest order from the orders.
@ -210,8 +215,18 @@ def cheapest_order_choose(orders, n):
return orders[0]
def _get_is_within_max_limits(max_fee_rel, max_fee_abs, cjvalue):
def check_max_fee(fee):
# only reject if fee is bigger than the relative and absolute limit
return not (fee > max_fee_abs and fee > cjvalue * max_fee_rel)
return check_max_fee
def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None,
pick=False, allowed_types=["swreloffer", "swabsoffer"]):
pick=False, allowed_types=["swreloffer", "swabsoffer"],
max_cj_fee=(1, float('inf'))):
is_within_max_limits = _get_is_within_max_limits(
max_cj_fee[0], max_cj_fee[1], cj_amount)
if ignored_makers is None:
ignored_makers = []
#Filter ignored makers and inappropriate amounts
@ -220,15 +235,18 @@ def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None,
orders = [o for o in orders if o['maxsize'] > cj_amount]
#Filter those not using wished-for offertypes
orders = [o for o in orders if o["ordertype"] in allowed_types]
orders_fees = [(
o, calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount) - o['txfee'])
for o in orders]
counterparties = set([o['counterparty'] for o in orders])
orders_fees = []
for o in orders:
fee = calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount) - o['txfee']
if is_within_max_limits(fee):
orders_fees.append((o, fee))
counterparties = set(o['counterparty'] for o, f in orders_fees)
if n > len(counterparties):
log.warn(('ERROR not enough liquidity in the orderbook n=%d '
'suitable-counterparties=%d amount=%d totalorders=%d') %
(n, len(counterparties), cj_amount, len(orders)))
(n, len(counterparties), cj_amount, len(orders_fees)))
# TODO handle not enough liquidity better, maybe an Exception
return None, 0
"""
@ -271,7 +289,8 @@ def choose_sweep_orders(offers,
n,
chooseOrdersBy,
ignored_makers=None,
allowed_types=['swreloffer', 'swabsoffer']):
allowed_types=['swreloffer', 'swabsoffer'],
max_cj_fee=(1, float('inf'))):
"""
choose an order given that we want to be left with no change
i.e. sweep an entire group of utxos
@ -282,6 +301,8 @@ def choose_sweep_orders(offers,
=> 0 = totalin - mytxfee - sum(absfee) - cjamount*(1 + sum(relfee))
=> cjamount = (totalin - mytxfee - sum(absfee)) / (1 + sum(relfee))
"""
is_within_max_limits = _get_is_within_max_limits(
max_cj_fee[0], max_cj_fee[1], total_input_value)
if ignored_makers is None:
ignored_makers = []
@ -326,7 +347,8 @@ def choose_sweep_orders(offers,
dict((v[0]['counterparty'], v)
for v in sorted(orders_fees,
key=feekey,
reverse=True)).values(),
reverse=True)
if is_within_max_limits(v[1])).values(),
key=feekey)
chosen_orders = []
while len(chosen_orders) < n:
@ -355,4 +377,3 @@ def choose_sweep_orders(offers,
result = dict([(o['counterparty'], o) for o in chosen_orders])
log.debug('cj amount = ' + str(cj_amount))
return result, cj_amount, total_fee

11
jmclient/jmclient/taker.py

@ -14,7 +14,6 @@ from jmclient.support import (calc_cj_fee, weighted_order_choose, choose_orders,
from jmclient.wallet import estimate_tx_fee
from jmclient.podle import generate_podle, get_podle_commitments, PoDLE
from .output import generate_podle_error_string
jlog = get_log()
@ -29,7 +28,8 @@ class Taker(object):
order_chooser=weighted_order_choose,
callbacks=None,
tdestaddrs=None,
ignored_makers=None):
ignored_makers=None,
max_cj_fee=(1, float('inf'))):
"""Schedule must be a list of tuples: (see sample_schedule_for_testnet
for explanation of syntax, also schedule.py module in this directory),
which will be a sequence of joins to do.
@ -75,6 +75,7 @@ class Taker(object):
self.wallet = wallet
self.schedule = schedule
self.order_chooser = order_chooser
self.max_cj_fee = max_cj_fee
#List (which persists between transactions) of makers
#who have not responded or behaved maliciously at any
@ -236,7 +237,8 @@ class Taker(object):
"POLICY", "segwit") == "false" else ["swreloffer", "swabsoffer"]
self.orderbook, self.total_cj_fee = choose_orders(
orderbook, self.cjamount, self.n_counterparties, self.order_chooser,
self.ignored_makers, allowed_types=allowed_types)
self.ignored_makers, allowed_types=allowed_types,
max_cj_fee=self.max_cj_fee)
if self.orderbook is None:
#Failure to get an orderbook means order selection failed
#for some reason; no action is taken, we let the stallMonitor
@ -310,7 +312,8 @@ class Taker(object):
self.orderbook, self.cjamount, self.total_cj_fee = choose_sweep_orders(
self.orderbook, total_value, self.total_txfee,
self.n_counterparties, self.order_chooser,
self.ignored_makers, allowed_types=allowed_types)
self.ignored_makers, allowed_types=allowed_types,
max_cj_fee=self.max_cj_fee)
if not self.orderbook:
self.taker_info_callback("ABORT",
"Could not find orders to complete transaction")

7
jmclient/jmclient/taker_utils.py

@ -327,6 +327,7 @@ def tumbler_taker_finished_update(taker, schedulefile, tumble_log, options,
with open(schedulefile, "wb") as f:
f.write(schedule_to_text(taker.schedule))
def tumbler_filter_orders_callback(orders_fees, cjamount, taker, options):
"""Since the tumbler does not use interactive fee checking,
we use the -x values from the command line instead.
@ -337,8 +338,8 @@ def tumbler_filter_orders_callback(orders_fees, cjamount, taker, options):
log.info('rel/abs average fee = ' + str(rel_cj_fee) + ' / ' + str(
abs_cj_fee))
if rel_cj_fee > options['maxcjfee'][
0] and abs_cj_fee > options['maxcjfee'][1]:
log.info("Rejected fees as too high according to options, will retry.")
if rel_cj_fee > taker.max_cj_fee[0] and abs_cj_fee > taker.max_cj_fee[1]:
log.info("Rejected fees as too high according to options, will "
"retry.")
return "retry"
return True

1
jmclient/test/test_schedule.py

@ -60,7 +60,6 @@ def get_options():
options.txcountparams = (18, 3)
options.minmakercount = 2
options.makercountrange = (6, 0)
options.maxcjfee = (0.01, 10000)
options.txfee = 5000
options.addrcount = 3
options.mintxcount = 1

251
scripts/cli_options.py

@ -1,13 +1,201 @@
#! /usr/bin/env python
from __future__ import absolute_import, print_function
import random
from optparse import OptionParser, OptionValueError
from ConfigParser import NoOptionError
import jmclient.support
"""This exists as a separate module for two reasons:
to reduce clutter in main scripts, and (TODO) refactor out
to reduce clutter in main scripts, and refactor out
options which are common to more than one script in a base class.
"""
from optparse import OptionParser
order_choose_algorithms = {
'random_under_max_order_choose': '-R',
'cheapest_order_choose': '-C',
'weighted_order_choose': '-W'
}
def add_common_options(parser):
parser.add_option(
'-f',
'--txfee',
action='store',
type='int',
dest='txfee',
default=-1,
help='number of satoshis per participant to use as the initial estimate '
'for the total transaction fee, default=dynamically estimated, note that this is adjusted '
'based on the estimated fee calculated after tx construction, based on '
'policy set in joinmarket.cfg.')
parser.add_option('--fast',
action='store_true',
dest='fastsync',
default=False,
help=('choose to do fast wallet sync, only for Core and '
'only for previously synced wallet'))
parser.add_option(
'-x',
'--max-cj-fee-abs',
type='int',
dest='max_cj_fee_abs',
help="Maximum absolute coinjoin fee in satoshi to pay to a single "
"market maker for a transaction. Both the limits given in "
"--max-cj-fee-abs and --max-cj-fee-rel must be exceeded in order "
"to not consider a certain offer.")
parser.add_option(
'-r',
'--max-cj-fee-rel',
type='float',
dest='max_cj_fee_rel',
help="Maximum relative coinjoin fee, in fractions of the coinjoin "
"value, to pay to a single market maker for a transaction. Both "
"the limits given in --max-cj-fee-abs and --max-cj-fee-rel must "
"be exceeded in order to not consider a certain offer.\n"
"Example: 0.001 for a maximum fee of 0.1% of the cj amount")
parser.add_option(
'--order-choose-algorithm',
action='callback',
type=str,
default=jmclient.support.random_under_max_order_choose,
callback=get_order_choose_algorithm,
help="Set the algorithm to use for selecting orders from the order book.\n"
"Default: {}\n"
"Available options: {}"
.format('random_under_max_order_choose',
', '.join(order_choose_algorithms.keys())),
dest='order_choose_fn')
add_order_choose_short_options(parser)
def add_order_choose_short_options(parser):
for name in sorted(order_choose_algorithms.keys()):
option = order_choose_algorithms[name]
parser.add_option(
option,
help="alias for --order-choose-algorithm={}".format(name),
nargs=0,
action='callback',
callback=get_order_choose_algorithm,
callback_kwargs={'value_kw': name},
dest='order_choose_fn')
def get_order_choose_algorithm(option, opt_str, value, parser, value_kw=None):
value = value_kw or value
if value not in order_choose_algorithms:
raise OptionValueError("{} must be one of {}".format(
opt_str, list(order_choose_algorithms.keys())))
fn = getattr(jmclient.support, value, None)
if not fn:
raise OptionValueError("internal error: '{}' order choose algorithm not"
" found".format(value))
setattr(parser.values, option.dest, fn)
def get_max_cj_fee_values(config, parser_options):
CONFIG_SECTION = 'POLICY'
CONFIG_OPTION = 'max_cj_fee_'
# rel, abs
fee_values = [None, None]
fee_types = [float, int]
for i, option in enumerate(('rel', 'abs')):
value = getattr(parser_options, CONFIG_OPTION + option, None)
if value is not None:
fee_values[i] = fee_types[i](value)
continue
try:
fee_values[i] = config.get(CONFIG_SECTION, CONFIG_OPTION + option)
except NoOptionError:
pass
if len(filter(lambda x: x is None, fee_values)):
fee_values = prompt_user_for_cj_fee(*fee_values)
return tuple(map(lambda j: fee_types[j](fee_values[j]),
range(len(fee_values))))
def prompt_user_for_cj_fee(rel_val, abs_val):
msg = """Joinmarket will choose market makers randomly as long as their
fees are below a certain maximum value, or fraction. The suggested maximums are:
X = {rel_val}
Y = {abs_val} satoshis
Those values were chosen randomly for you (unless you already set one of them
in your joinmarket.cfg or via a CLI option).
Since you are using N counterparties, if you agree to these values, your
**maximum** coinjoin fee will be either N*Y satoshis or N*X percent of your
coinjoin amount, depending on which is larger. The actual fee is likely to be
significantly less; perhaps half that amount, depending on which
counterparties are selected."""
def prompt_user_value(m, val, check):
while True:
data = raw_input(m)
if data == 'y':
return val
try:
val_user = float(data)
except ValueError:
print("Bad answer, try again.")
continue
if not check(val_user):
continue
return val_user
rel_prompt = False
if rel_val is None:
rel_prompt = True
rel_val = 0.001
abs_prompt = False
if abs_val is None:
abs_prompt = True
abs_val = random.randint(1000, 10000)
print(msg.format(rel_val=rel_val, abs_val=abs_val))
if rel_prompt:
msg = ("\nIf you want to keep this relative limit, enter 'y';"
"\notherwise choose your own fraction (between 1 and 0): ")
def rel_check(val):
if val >= 1:
print("Choose a number below 1! Else you will spend all your "
"bitcoins for fees!")
return False
return True
rel_val = prompt_user_value(msg, rel_val, rel_check)
print("Success! Using relative fee limit of {:%}".format(rel_val))
if abs_prompt:
msg = ("\nIf you want to keep this absolute limit, enter 'y';"
"\notherwise choose your own limit in satoshi: ")
def abs_check(val):
if val % 1 != 0:
print("You must choose a full number!")
return False
return True
abs_val = int(prompt_user_value(msg, abs_val, abs_check))
print("Success! Using absolute fee limit of {}".format(abs_val))
print("""\nIf you don't want to see this message again, make an entry like
this in the POLICY section of joinmarket.cfg:
max_cj_fee_abs = {abs_val}
max_cj_fee_rel = {rel_val}\n""".format(rel_val=rel_val, abs_val=abs_val))
return rel_val, abs_val
def get_tumbler_parser():
parser = OptionParser(
@ -27,17 +215,6 @@ def get_tumbler_parser():
help=
'Mixing depth to start tumble process from. default=0.',
default=0)
parser.add_option(
'-f',
'--txfee',
action='store',
type='int',
dest='txfee',
default=-1,
help='number of satoshis per participant to use as the initial estimate '+
'for the total transaction fee, default=dynamically estimated, note that this is adjusted '+
'based on the estimated fee calculated after tx construction, based on '+
'policy set in joinmarket.cfg.')
parser.add_option('--restart',
action='store_true',
dest='restart',
@ -61,16 +238,6 @@ def get_tumbler_parser():
'How many destination addresses in total should be used. If not enough are given'
' as command line arguments, the script will ask for more. This parameter is required'
' to stop amount correlation. default=3')
parser.add_option(
'-x',
'--maxcjfee',
type='float',
dest='maxcjfee',
nargs=2,
default=(0.01, 10000),
help='maximum coinjoin fee and bitcoin value the tumbler is '
'willing to pay to a single market maker. Both values need to be exceeded, so if '
'the fee is 30% but only 500satoshi is paid the tx will go ahead. default=0.01, 10000 (1%, 10000satoshi)')
parser.add_option(
'-N',
'--makercountrange',
@ -174,14 +341,10 @@ def get_tumbler_parser():
default=9,
help=
'maximum amount of times to re-create a transaction before giving up, default 9')
parser.add_option('--fast',
action='store_true',
dest='fastsync',
default=False,
help=('choose to do fast wallet sync, only for Core and '
'only for previously synced wallet'))
add_common_options(parser)
return parser
def get_sendpayment_parser():
parser = OptionParser(
usage=
@ -191,19 +354,6 @@ def get_sendpayment_parser():
'wallet to an given address using coinjoin and then switches off. Also sends from bitcoinqt. '
+
'Setting amount to zero will do a sweep, where the entire mix depth is emptied')
parser.add_option(
'-f',
'--txfee',
action='store',
type='int',
dest='txfee',
default=-1,
help=
'number of satoshis per participant to use as the initial estimate ' +
'for the total transaction fee, default=dynamically estimated, note that this is adjusted '
+
'based on the estimated fee calculated after tx construction, based on '
+ 'policy set in joinmarket.cfg.')
parser.add_option(
'-w',
'--wait-time',
@ -226,14 +376,6 @@ def get_sendpayment_parser():
dest='schedule',
help='schedule file name; see file "sample-schedule-for-testnet" for explanation and example',
default='')
parser.add_option(
'-C',
'--choose-cheapest',
action='store_true',
dest='choosecheapest',
default=False,
help=
'override weightened offers picking and choose cheapest. this might reduce anonymity.')
parser.add_option(
'-P',
'--pick-orders',
@ -268,12 +410,5 @@ def get_sendpayment_parser():
dest='answeryes',
default=False,
help='answer yes to everything')
parser.add_option('--fast',
action='store_true',
dest='fastsync',
default=False,
help=('choose to do fast wallet sync, only for Core and '
'only for previously synced wallet'))
add_common_options(parser)
return parser

18
scripts/sendpayment.py

@ -14,12 +14,11 @@ import pprint
from jmclient import Taker, load_program_config, get_schedule,\
JMClientProtocolFactory, start_reactor, validate_address, jm_single,\
cheapest_order_choose, weighted_order_choose, sync_wallet,\
RegtestBitcoinCoreInterface, estimate_tx_fee, direct_send,\
sync_wallet, RegtestBitcoinCoreInterface, estimate_tx_fee, direct_send,\
open_test_wallet_maybe, get_wallet_path
from twisted.python.log import startLogging
from jmbase.support import get_log
from cli_options import get_sendpayment_parser
from cli_options import get_sendpayment_parser, get_max_cj_fee_values
log = get_log()
@ -91,16 +90,13 @@ def main():
jm_single().bc_interface.simulating = True
jm_single().maker_timeout_sec = 15
chooseOrdersFunc = None
if options.pickorders:
chooseOrdersFunc = pick_order
if sweeping:
print('WARNING: You may have to pick offers multiple times')
print('WARNING: due to manual offer picking while sweeping')
elif options.choosecheapest:
chooseOrdersFunc = cheapest_order_choose
else: # choose randomly (weighted)
chooseOrdersFunc = weighted_order_choose
else:
chooseOrdersFunc = options.order_choose_fn
# Dynamically estimate a realistic fee if it currently is the default value.
# At this point we do not know even the number of our own inputs, so
@ -112,6 +108,11 @@ def main():
options.txfee))
assert (options.txfee >= 0)
if options.makercount != 0:
maxcjfee = get_max_cj_fee_values(jm_single().config, options)
log.info("Using maximum coinjoin fee limits per maker of {:.4%}, {} "
"sat".format(*maxcjfee))
log.debug('starting sendpayment')
max_mix_depth = max([mixdepth, options.amtmixdepths - 1])
@ -222,6 +223,7 @@ def main():
taker = Taker(wallet,
schedule,
order_chooser=chooseOrdersFunc,
max_cj_fee=maxcjfee,
callbacks=(filter_orders_callback, None, taker_finished))
clientfactory = JMClientProtocolFactory(taker)
nodaemon = jm_single().config.getint("DAEMON", "no_daemon")

19
scripts/tumbler.py

@ -7,13 +7,13 @@ import os
import pprint
from twisted.python.log import startLogging
from jmclient import Taker, load_program_config, get_schedule,\
weighted_order_choose, JMClientProtocolFactory, start_reactor, jm_single,\
get_wallet_path, open_test_wallet_maybe, sync_wallet,\
get_tumble_schedule, RegtestBitcoinCoreInterface, schedule_to_text,\
restart_waiter, get_tumble_log, tumbler_taker_finished_update,\
JMClientProtocolFactory, start_reactor, jm_single, get_wallet_path,\
open_test_wallet_maybe, sync_wallet, get_tumble_schedule,\
RegtestBitcoinCoreInterface, schedule_to_text, restart_waiter,\
get_tumble_log, tumbler_taker_finished_update,\
tumbler_filter_orders_callback
from jmbase.support import get_log
from cli_options import get_tumbler_parser
from cli_options import get_tumbler_parser, get_max_cj_fee_values
log = get_log()
logsdir = os.path.join(os.path.dirname(
jm_single().config_location), "logs")
@ -22,11 +22,13 @@ logsdir = os.path.join(os.path.dirname(
def main():
tumble_log = get_tumble_log(logsdir)
(options, args) = get_tumbler_parser().parse_args()
options_org = options
options = vars(options)
if len(args) < 1:
print('Error: Needs a wallet file')
sys.exit(0)
load_program_config()
#Load the wallet
wallet_name = args[0]
max_mix_depth = options['mixdepthsrc'] + options['mixdepthcount']
@ -38,6 +40,10 @@ def main():
while not jm_single().bc_interface.wallet_synced:
sync_wallet(wallet, fast=options['fastsync'])
maxcjfee = get_max_cj_fee_values(jm_single().config, options_org)
log.info("Using maximum coinjoin fee limits per maker of {:.4%}, {} sat"
.format(*maxcjfee))
#Parse options and generate schedule
#Output information to log files
jm_single().mincjamount = options['mincjamount']
@ -110,7 +116,8 @@ def main():
#instantiate Taker with given schedule and run
taker = Taker(wallet,
schedule,
order_chooser=weighted_order_choose,
order_chooser=options['order_choose_fn'],
max_cj_fee=maxcjfee,
callbacks=(filter_orders_callback, None, taker_finished),
tdestaddrs=destaddrs)
clientfactory = JMClientProtocolFactory(taker)

Loading…
Cancel
Save