diff --git a/jmclient/jmclient/configure.py b/jmclient/jmclient/configure.py index 13e42ec..ff0cb5c 100644 --- a/jmclient/jmclient/configure.py +++ b/jmclient/jmclient/configure.py @@ -163,11 +163,13 @@ confirm_timeout_hours = 6 [POLICY] #Use segwit style wallets and transactions segwit = true + # for dust sweeping, try merge_algorithm = gradual # for more rapid dust sweeping, try merge_algorithm = greedy # for most rapid dust sweeping, try merge_algorithm = greediest # but don't forget to bump your miner fees! merge_algorithm = default + # the fee estimate is based on a projection of how many satoshis # per kB are needed to get in one of the next N blocks, N set here # as the value of 'tx_fees'. This estimate is high if you set N=1, @@ -178,12 +180,26 @@ merge_algorithm = default # example: N=30000 will use 30000 sat/kB as a fee, while N=5 # will use the estimate from your selected blockchain source tx_fees = 3 + # For users getting transaction fee estimates over an API, # place a sanity check limit on the satoshis-per-kB to be paid. # This limit is also applied to users using Core, even though # Core has its own sanity check limit, which is currently # 1,000,000 satoshis. absurd_fee_per_kb = 350000 + +# Maximum absolute coinjoin fee in satoshi to pay to a single +# market maker for a transaction. Both the limits given in +# max_cj_fee_abs and max_cj_fee_rel must be exceeded in order +# to not consider a certain offer. +#max_cj_fee_abs = x + +# Maximum relative coinjoin fee, in fractions of the coinjoin value +# e.g. if your coinjoin amount is 2 btc (200000000 satoshi) and +# max_cj_fee_rel = 0.001 (0.1%), the maximum fee allowed would +# be 0.002 btc (200000 satoshi) +#max_cj_fee_rel = x + # the range of confirmations passed to the `listunspent` bitcoind RPC call # 1st value is the inclusive minimum, defaults to one confirmation # 2nd value is the exclusive maximum, defaults to most-positive-bignum (Google Me!) @@ -206,8 +222,11 @@ absurd_fee_per_kb = 350000 # not-self = never broadcast with your own ip tx_broadcast = self minimum_makers = 2 + +############################## #THE FOLLOWING SETTINGS ARE REQUIRED TO DEFEND AGAINST SNOOPERS. #DON'T ALTER THEM UNLESS YOU UNDERSTAND THE IMPLICATIONS. +############################## # number of retries allowed for a specific utxo, to prevent DOS/snooping. # Lower settings make snooping more expensive, but also prevent honest users diff --git a/jmclient/jmclient/support.py b/jmclient/jmclient/support.py index 4c9f649..34fe1cc 100644 --- a/jmclient/jmclient/support.py +++ b/jmclient/jmclient/support.py @@ -203,6 +203,11 @@ def weighted_order_choose(orders, n): return orders[chosen_order_index] +def random_under_max_order_choose(orders, n): + # orders are already pre-filtered for max_cj_fee + return random.choice(orders) + + def cheapest_order_choose(orders, n): """ Return the cheapest order from the orders. @@ -210,8 +215,18 @@ def cheapest_order_choose(orders, n): return orders[0] +def _get_is_within_max_limits(max_fee_rel, max_fee_abs, cjvalue): + def check_max_fee(fee): + # only reject if fee is bigger than the relative and absolute limit + return not (fee > max_fee_abs and fee > cjvalue * max_fee_rel) + return check_max_fee + + def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None, - pick=False, allowed_types=["swreloffer", "swabsoffer"]): + pick=False, allowed_types=["swreloffer", "swabsoffer"], + max_cj_fee=(1, float('inf'))): + is_within_max_limits = _get_is_within_max_limits( + max_cj_fee[0], max_cj_fee[1], cj_amount) if ignored_makers is None: ignored_makers = [] #Filter ignored makers and inappropriate amounts @@ -220,15 +235,18 @@ def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None, orders = [o for o in orders if o['maxsize'] > cj_amount] #Filter those not using wished-for offertypes orders = [o for o in orders if o["ordertype"] in allowed_types] - orders_fees = [( - o, calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount) - o['txfee']) - for o in orders] - counterparties = set([o['counterparty'] for o in orders]) + orders_fees = [] + for o in orders: + fee = calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount) - o['txfee'] + if is_within_max_limits(fee): + orders_fees.append((o, fee)) + + counterparties = set(o['counterparty'] for o, f in orders_fees) if n > len(counterparties): log.warn(('ERROR not enough liquidity in the orderbook n=%d ' 'suitable-counterparties=%d amount=%d totalorders=%d') % - (n, len(counterparties), cj_amount, len(orders))) + (n, len(counterparties), cj_amount, len(orders_fees))) # TODO handle not enough liquidity better, maybe an Exception return None, 0 """ @@ -271,7 +289,8 @@ def choose_sweep_orders(offers, n, chooseOrdersBy, ignored_makers=None, - allowed_types=['swreloffer', 'swabsoffer']): + allowed_types=['swreloffer', 'swabsoffer'], + max_cj_fee=(1, float('inf'))): """ choose an order given that we want to be left with no change i.e. sweep an entire group of utxos @@ -282,6 +301,8 @@ def choose_sweep_orders(offers, => 0 = totalin - mytxfee - sum(absfee) - cjamount*(1 + sum(relfee)) => cjamount = (totalin - mytxfee - sum(absfee)) / (1 + sum(relfee)) """ + is_within_max_limits = _get_is_within_max_limits( + max_cj_fee[0], max_cj_fee[1], total_input_value) if ignored_makers is None: ignored_makers = [] @@ -326,7 +347,8 @@ def choose_sweep_orders(offers, dict((v[0]['counterparty'], v) for v in sorted(orders_fees, key=feekey, - reverse=True)).values(), + reverse=True) + if is_within_max_limits(v[1])).values(), key=feekey) chosen_orders = [] while len(chosen_orders) < n: @@ -355,4 +377,3 @@ def choose_sweep_orders(offers, result = dict([(o['counterparty'], o) for o in chosen_orders]) log.debug('cj amount = ' + str(cj_amount)) return result, cj_amount, total_fee - diff --git a/jmclient/jmclient/taker.py b/jmclient/jmclient/taker.py index 42d526d..d05197c 100644 --- a/jmclient/jmclient/taker.py +++ b/jmclient/jmclient/taker.py @@ -14,7 +14,6 @@ from jmclient.support import (calc_cj_fee, weighted_order_choose, choose_orders, from jmclient.wallet import estimate_tx_fee from jmclient.podle import generate_podle, get_podle_commitments, PoDLE from .output import generate_podle_error_string - jlog = get_log() @@ -29,7 +28,8 @@ class Taker(object): order_chooser=weighted_order_choose, callbacks=None, tdestaddrs=None, - ignored_makers=None): + ignored_makers=None, + max_cj_fee=(1, float('inf'))): """Schedule must be a list of tuples: (see sample_schedule_for_testnet for explanation of syntax, also schedule.py module in this directory), which will be a sequence of joins to do. @@ -75,6 +75,7 @@ class Taker(object): self.wallet = wallet self.schedule = schedule self.order_chooser = order_chooser + self.max_cj_fee = max_cj_fee #List (which persists between transactions) of makers #who have not responded or behaved maliciously at any @@ -236,7 +237,8 @@ class Taker(object): "POLICY", "segwit") == "false" else ["swreloffer", "swabsoffer"] self.orderbook, self.total_cj_fee = choose_orders( orderbook, self.cjamount, self.n_counterparties, self.order_chooser, - self.ignored_makers, allowed_types=allowed_types) + self.ignored_makers, allowed_types=allowed_types, + max_cj_fee=self.max_cj_fee) if self.orderbook is None: #Failure to get an orderbook means order selection failed #for some reason; no action is taken, we let the stallMonitor @@ -310,7 +312,8 @@ class Taker(object): self.orderbook, self.cjamount, self.total_cj_fee = choose_sweep_orders( self.orderbook, total_value, self.total_txfee, self.n_counterparties, self.order_chooser, - self.ignored_makers, allowed_types=allowed_types) + self.ignored_makers, allowed_types=allowed_types, + max_cj_fee=self.max_cj_fee) if not self.orderbook: self.taker_info_callback("ABORT", "Could not find orders to complete transaction") diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index 4e255d9..b83300a 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -327,6 +327,7 @@ def tumbler_taker_finished_update(taker, schedulefile, tumble_log, options, with open(schedulefile, "wb") as f: f.write(schedule_to_text(taker.schedule)) + def tumbler_filter_orders_callback(orders_fees, cjamount, taker, options): """Since the tumbler does not use interactive fee checking, we use the -x values from the command line instead. @@ -337,8 +338,8 @@ def tumbler_filter_orders_callback(orders_fees, cjamount, taker, options): log.info('rel/abs average fee = ' + str(rel_cj_fee) + ' / ' + str( abs_cj_fee)) - if rel_cj_fee > options['maxcjfee'][ - 0] and abs_cj_fee > options['maxcjfee'][1]: - log.info("Rejected fees as too high according to options, will retry.") + if rel_cj_fee > taker.max_cj_fee[0] and abs_cj_fee > taker.max_cj_fee[1]: + log.info("Rejected fees as too high according to options, will " + "retry.") return "retry" return True diff --git a/jmclient/test/test_schedule.py b/jmclient/test/test_schedule.py index 9f14c6f..578491d 100644 --- a/jmclient/test/test_schedule.py +++ b/jmclient/test/test_schedule.py @@ -60,7 +60,6 @@ def get_options(): options.txcountparams = (18, 3) options.minmakercount = 2 options.makercountrange = (6, 0) - options.maxcjfee = (0.01, 10000) options.txfee = 5000 options.addrcount = 3 options.mintxcount = 1 diff --git a/scripts/cli_options.py b/scripts/cli_options.py index 91428ba..0ae91dd 100644 --- a/scripts/cli_options.py +++ b/scripts/cli_options.py @@ -1,13 +1,201 @@ #! /usr/bin/env python from __future__ import absolute_import, print_function import random +from optparse import OptionParser, OptionValueError +from ConfigParser import NoOptionError + +import jmclient.support """This exists as a separate module for two reasons: -to reduce clutter in main scripts, and (TODO) refactor out +to reduce clutter in main scripts, and refactor out options which are common to more than one script in a base class. """ -from optparse import OptionParser +order_choose_algorithms = { + 'random_under_max_order_choose': '-R', + 'cheapest_order_choose': '-C', + 'weighted_order_choose': '-W' +} + + +def add_common_options(parser): + parser.add_option( + '-f', + '--txfee', + action='store', + type='int', + dest='txfee', + default=-1, + help='number of satoshis per participant to use as the initial estimate ' + 'for the total transaction fee, default=dynamically estimated, note that this is adjusted ' + 'based on the estimated fee calculated after tx construction, based on ' + 'policy set in joinmarket.cfg.') + parser.add_option('--fast', + action='store_true', + dest='fastsync', + default=False, + help=('choose to do fast wallet sync, only for Core and ' + 'only for previously synced wallet')) + parser.add_option( + '-x', + '--max-cj-fee-abs', + type='int', + dest='max_cj_fee_abs', + help="Maximum absolute coinjoin fee in satoshi to pay to a single " + "market maker for a transaction. Both the limits given in " + "--max-cj-fee-abs and --max-cj-fee-rel must be exceeded in order " + "to not consider a certain offer.") + parser.add_option( + '-r', + '--max-cj-fee-rel', + type='float', + dest='max_cj_fee_rel', + help="Maximum relative coinjoin fee, in fractions of the coinjoin " + "value, to pay to a single market maker for a transaction. Both " + "the limits given in --max-cj-fee-abs and --max-cj-fee-rel must " + "be exceeded in order to not consider a certain offer.\n" + "Example: 0.001 for a maximum fee of 0.1% of the cj amount") + parser.add_option( + '--order-choose-algorithm', + action='callback', + type=str, + default=jmclient.support.random_under_max_order_choose, + callback=get_order_choose_algorithm, + help="Set the algorithm to use for selecting orders from the order book.\n" + "Default: {}\n" + "Available options: {}" + .format('random_under_max_order_choose', + ', '.join(order_choose_algorithms.keys())), + dest='order_choose_fn') + add_order_choose_short_options(parser) + + +def add_order_choose_short_options(parser): + for name in sorted(order_choose_algorithms.keys()): + option = order_choose_algorithms[name] + parser.add_option( + option, + help="alias for --order-choose-algorithm={}".format(name), + nargs=0, + action='callback', + callback=get_order_choose_algorithm, + callback_kwargs={'value_kw': name}, + dest='order_choose_fn') + + +def get_order_choose_algorithm(option, opt_str, value, parser, value_kw=None): + value = value_kw or value + if value not in order_choose_algorithms: + raise OptionValueError("{} must be one of {}".format( + opt_str, list(order_choose_algorithms.keys()))) + fn = getattr(jmclient.support, value, None) + if not fn: + raise OptionValueError("internal error: '{}' order choose algorithm not" + " found".format(value)) + setattr(parser.values, option.dest, fn) + + +def get_max_cj_fee_values(config, parser_options): + CONFIG_SECTION = 'POLICY' + CONFIG_OPTION = 'max_cj_fee_' + # rel, abs + fee_values = [None, None] + fee_types = [float, int] + + for i, option in enumerate(('rel', 'abs')): + value = getattr(parser_options, CONFIG_OPTION + option, None) + if value is not None: + fee_values[i] = fee_types[i](value) + continue + + try: + fee_values[i] = config.get(CONFIG_SECTION, CONFIG_OPTION + option) + except NoOptionError: + pass + + if len(filter(lambda x: x is None, fee_values)): + fee_values = prompt_user_for_cj_fee(*fee_values) + + return tuple(map(lambda j: fee_types[j](fee_values[j]), + range(len(fee_values)))) + + +def prompt_user_for_cj_fee(rel_val, abs_val): + msg = """Joinmarket will choose market makers randomly as long as their +fees are below a certain maximum value, or fraction. The suggested maximums are: + +X = {rel_val} +Y = {abs_val} satoshis + +Those values were chosen randomly for you (unless you already set one of them +in your joinmarket.cfg or via a CLI option). + +Since you are using N counterparties, if you agree to these values, your +**maximum** coinjoin fee will be either N*Y satoshis or N*X percent of your +coinjoin amount, depending on which is larger. The actual fee is likely to be +significantly less; perhaps half that amount, depending on which +counterparties are selected.""" + + def prompt_user_value(m, val, check): + while True: + data = raw_input(m) + if data == 'y': + return val + try: + val_user = float(data) + except ValueError: + print("Bad answer, try again.") + continue + if not check(val_user): + continue + return val_user + + rel_prompt = False + if rel_val is None: + rel_prompt = True + rel_val = 0.001 + + abs_prompt = False + if abs_val is None: + abs_prompt = True + abs_val = random.randint(1000, 10000) + + print(msg.format(rel_val=rel_val, abs_val=abs_val)) + if rel_prompt: + msg = ("\nIf you want to keep this relative limit, enter 'y';" + "\notherwise choose your own fraction (between 1 and 0): ") + + def rel_check(val): + if val >= 1: + print("Choose a number below 1! Else you will spend all your " + "bitcoins for fees!") + return False + return True + + rel_val = prompt_user_value(msg, rel_val, rel_check) + print("Success! Using relative fee limit of {:%}".format(rel_val)) + + if abs_prompt: + msg = ("\nIf you want to keep this absolute limit, enter 'y';" + "\notherwise choose your own limit in satoshi: ") + + def abs_check(val): + if val % 1 != 0: + print("You must choose a full number!") + return False + return True + + abs_val = int(prompt_user_value(msg, abs_val, abs_check)) + print("Success! Using absolute fee limit of {}".format(abs_val)) + + print("""\nIf you don't want to see this message again, make an entry like +this in the POLICY section of joinmarket.cfg: + +max_cj_fee_abs = {abs_val} +max_cj_fee_rel = {rel_val}\n""".format(rel_val=rel_val, abs_val=abs_val)) + + return rel_val, abs_val + def get_tumbler_parser(): parser = OptionParser( @@ -27,17 +215,6 @@ def get_tumbler_parser(): help= 'Mixing depth to start tumble process from. default=0.', default=0) - parser.add_option( - '-f', - '--txfee', - action='store', - type='int', - dest='txfee', - default=-1, - help='number of satoshis per participant to use as the initial estimate '+ - 'for the total transaction fee, default=dynamically estimated, note that this is adjusted '+ - 'based on the estimated fee calculated after tx construction, based on '+ - 'policy set in joinmarket.cfg.') parser.add_option('--restart', action='store_true', dest='restart', @@ -61,16 +238,6 @@ def get_tumbler_parser(): 'How many destination addresses in total should be used. If not enough are given' ' as command line arguments, the script will ask for more. This parameter is required' ' to stop amount correlation. default=3') - parser.add_option( - '-x', - '--maxcjfee', - type='float', - dest='maxcjfee', - nargs=2, - default=(0.01, 10000), - help='maximum coinjoin fee and bitcoin value the tumbler is ' - 'willing to pay to a single market maker. Both values need to be exceeded, so if ' - 'the fee is 30% but only 500satoshi is paid the tx will go ahead. default=0.01, 10000 (1%, 10000satoshi)') parser.add_option( '-N', '--makercountrange', @@ -174,14 +341,10 @@ def get_tumbler_parser(): default=9, help= 'maximum amount of times to re-create a transaction before giving up, default 9') - parser.add_option('--fast', - action='store_true', - dest='fastsync', - default=False, - help=('choose to do fast wallet sync, only for Core and ' - 'only for previously synced wallet')) + add_common_options(parser) return parser + def get_sendpayment_parser(): parser = OptionParser( usage= @@ -191,19 +354,6 @@ def get_sendpayment_parser(): 'wallet to an given address using coinjoin and then switches off. Also sends from bitcoinqt. ' + 'Setting amount to zero will do a sweep, where the entire mix depth is emptied') - parser.add_option( - '-f', - '--txfee', - action='store', - type='int', - dest='txfee', - default=-1, - help= - 'number of satoshis per participant to use as the initial estimate ' + - 'for the total transaction fee, default=dynamically estimated, note that this is adjusted ' - + - 'based on the estimated fee calculated after tx construction, based on ' - + 'policy set in joinmarket.cfg.') parser.add_option( '-w', '--wait-time', @@ -226,14 +376,6 @@ def get_sendpayment_parser(): dest='schedule', help='schedule file name; see file "sample-schedule-for-testnet" for explanation and example', default='') - parser.add_option( - '-C', - '--choose-cheapest', - action='store_true', - dest='choosecheapest', - default=False, - help= - 'override weightened offers picking and choose cheapest. this might reduce anonymity.') parser.add_option( '-P', '--pick-orders', @@ -268,12 +410,5 @@ def get_sendpayment_parser(): dest='answeryes', default=False, help='answer yes to everything') - parser.add_option('--fast', - action='store_true', - dest='fastsync', - default=False, - help=('choose to do fast wallet sync, only for Core and ' - 'only for previously synced wallet')) + add_common_options(parser) return parser - - diff --git a/scripts/sendpayment.py b/scripts/sendpayment.py index 6c2b282..03e0c1e 100644 --- a/scripts/sendpayment.py +++ b/scripts/sendpayment.py @@ -14,12 +14,11 @@ import pprint from jmclient import Taker, load_program_config, get_schedule,\ JMClientProtocolFactory, start_reactor, validate_address, jm_single,\ - cheapest_order_choose, weighted_order_choose, sync_wallet,\ - RegtestBitcoinCoreInterface, estimate_tx_fee, direct_send,\ + sync_wallet, RegtestBitcoinCoreInterface, estimate_tx_fee, direct_send,\ open_test_wallet_maybe, get_wallet_path from twisted.python.log import startLogging from jmbase.support import get_log -from cli_options import get_sendpayment_parser +from cli_options import get_sendpayment_parser, get_max_cj_fee_values log = get_log() @@ -91,16 +90,13 @@ def main(): jm_single().bc_interface.simulating = True jm_single().maker_timeout_sec = 15 - chooseOrdersFunc = None if options.pickorders: chooseOrdersFunc = pick_order if sweeping: print('WARNING: You may have to pick offers multiple times') print('WARNING: due to manual offer picking while sweeping') - elif options.choosecheapest: - chooseOrdersFunc = cheapest_order_choose - else: # choose randomly (weighted) - chooseOrdersFunc = weighted_order_choose + else: + chooseOrdersFunc = options.order_choose_fn # Dynamically estimate a realistic fee if it currently is the default value. # At this point we do not know even the number of our own inputs, so @@ -112,6 +108,11 @@ def main(): options.txfee)) assert (options.txfee >= 0) + if options.makercount != 0: + maxcjfee = get_max_cj_fee_values(jm_single().config, options) + log.info("Using maximum coinjoin fee limits per maker of {:.4%}, {} " + "sat".format(*maxcjfee)) + log.debug('starting sendpayment') max_mix_depth = max([mixdepth, options.amtmixdepths - 1]) @@ -222,6 +223,7 @@ def main(): taker = Taker(wallet, schedule, order_chooser=chooseOrdersFunc, + max_cj_fee=maxcjfee, callbacks=(filter_orders_callback, None, taker_finished)) clientfactory = JMClientProtocolFactory(taker) nodaemon = jm_single().config.getint("DAEMON", "no_daemon") diff --git a/scripts/tumbler.py b/scripts/tumbler.py index 5e7c54a..79c5813 100644 --- a/scripts/tumbler.py +++ b/scripts/tumbler.py @@ -7,13 +7,13 @@ import os import pprint from twisted.python.log import startLogging from jmclient import Taker, load_program_config, get_schedule,\ - weighted_order_choose, JMClientProtocolFactory, start_reactor, jm_single,\ - get_wallet_path, open_test_wallet_maybe, sync_wallet,\ - get_tumble_schedule, RegtestBitcoinCoreInterface, schedule_to_text,\ - restart_waiter, get_tumble_log, tumbler_taker_finished_update,\ + JMClientProtocolFactory, start_reactor, jm_single, get_wallet_path,\ + open_test_wallet_maybe, sync_wallet, get_tumble_schedule,\ + RegtestBitcoinCoreInterface, schedule_to_text, restart_waiter,\ + get_tumble_log, tumbler_taker_finished_update,\ tumbler_filter_orders_callback from jmbase.support import get_log -from cli_options import get_tumbler_parser +from cli_options import get_tumbler_parser, get_max_cj_fee_values log = get_log() logsdir = os.path.join(os.path.dirname( jm_single().config_location), "logs") @@ -22,11 +22,13 @@ logsdir = os.path.join(os.path.dirname( def main(): tumble_log = get_tumble_log(logsdir) (options, args) = get_tumbler_parser().parse_args() + options_org = options options = vars(options) if len(args) < 1: print('Error: Needs a wallet file') sys.exit(0) load_program_config() + #Load the wallet wallet_name = args[0] max_mix_depth = options['mixdepthsrc'] + options['mixdepthcount'] @@ -38,6 +40,10 @@ def main(): while not jm_single().bc_interface.wallet_synced: sync_wallet(wallet, fast=options['fastsync']) + maxcjfee = get_max_cj_fee_values(jm_single().config, options_org) + log.info("Using maximum coinjoin fee limits per maker of {:.4%}, {} sat" + .format(*maxcjfee)) + #Parse options and generate schedule #Output information to log files jm_single().mincjamount = options['mincjamount'] @@ -110,7 +116,8 @@ def main(): #instantiate Taker with given schedule and run taker = Taker(wallet, schedule, - order_chooser=weighted_order_choose, + order_chooser=options['order_choose_fn'], + max_cj_fee=maxcjfee, callbacks=(filter_orders_callback, None, taker_finished), tdestaddrs=destaddrs) clientfactory = JMClientProtocolFactory(taker)