Implemented tumbler

Refactor parsers into separate module (more code reuse to do).
Add a tumbler schedule generator in jmclient/schedule.py.
Alter syntax of schedules; taker interprets fractional amounts
as percentage of mixdepth (for tumbler), and integers as satoshi,
zero still sweep.
TODO tumbler is only a minor delta from sendpayment, fold them together.

jmclient/jmclient/__init__.py

@@ -28,7 +28,7 @@ from .podle import (set_commitment_file, get_commitment_file,
                     generate_podle_error_string, add_external_commitments,
                     PoDLE, generate_podle, get_podle_commitments,
                     update_commitments)
-from .schedule import get_schedule
+from .schedule import get_schedule, get_tumble_schedule
 from .commitment_utils import get_utxo_info, validate_utxo_data, quit
 # Set default logging handler to avoid "No handler found" warnings.
 

jmclient/jmclient/schedule.py

@@ -1,10 +1,14 @@
 #!/usr/bin/env python
 from __future__ import print_function
-from jmclient import validate_address
+from jmclient import (validate_address, rand_exp_array,
+                      rand_norm_array, rand_pow_array)
 """Utility functions for dealing with Taker schedules.
 
-- attempt to read the schedule from the provided file
+- get_schedule(filename):
-- (TODO) generate a schedule for e.g. tumbling from a given wallet, with parameters
+    attempt to read the schedule from the provided file
+- get_tumble_schedule(options, destaddrs):
+    generate a schedule for tumbling from a given wallet, using options dict
+    and specified destinations
 """
 
 def get_schedule(filename):
@@ -15,7 +19,7 @@ def get_schedule(filename):
             if sl.startswith("#"):
                 continue
             try:
-                mixdepth, amount, makercount, destaddr = sl.split(',')
+                mixdepth, amount, makercount, destaddr, waittime = sl.split(',')
             except ValueError as e:
                 return (False, "Failed to parse schedule line: " + sl)
             try:
@@ -23,6 +27,7 @@ def get_schedule(filename):
                 amount = int(amount)
                 makercount = int(makercount)
                 destaddr = destaddr.strip()
+                waittime = float(waittime)
             except ValueError as e:
                 return (False, "Failed to parse schedule line: " + sl)
             if destaddr != "INTERNAL":
@@ -31,3 +36,76 @@ def get_schedule(filename):
                     return (False, "Invalid address: " + destaddr + "," + errmsg)
             schedule.append((mixdepth, amount, makercount, destaddr))
     return (True, schedule)
+
+def get_tumble_schedule(options, destaddrs):
+    """for the general intent and design of the tumbler algo, see the docs in
+    joinmarket-org/joinmarket.
+    Alterations:
+    Donation removed for now.
+    Default final setting for "amount_fraction" is zero, for each mixdepth.
+    This is because we now use a general "schedule" syntax for both tumbler and
+    any other taker algo; it interprets floats as fractions and integers as satoshis,
+    and zero as sweep (as before).
+    This is a modified version of tumbler.py/generate_tumbler_tx()
+    """
+    def lower_bounded_int(thelist, lowerbound):
+        return [int(l) if int(l) >= lowerbound else lowerbound for l in thelist]
+
+    txcounts = rand_norm_array(options['txcountparams'][0],
+                               options['txcountparams'][1], options['mixdepthcount'])
+    txcounts = lower_bounded_int(txcounts, options['mintxcount'])
+    tx_list = []
+    for m, txcount in enumerate(txcounts):
+        if options['mixdepthcount'] - options['addrcount'] <= m and m < \
+                options['mixdepthcount'] - 1:
+            #these mixdepths send to a destination address, so their
+            # amount_fraction cant be 1.0, some coins must be left over
+            if txcount == 1:
+                txcount = 2
+        # assume that the sizes of outputs will follow a power law
+        amount_fractions = rand_pow_array(options['amountpower'], txcount)
+        amount_fractions = [1.0 - x for x in amount_fractions]
+        amount_fractions = [x / sum(amount_fractions) for x in amount_fractions]
+        # transaction times are uncorrelated
+        # time between events in a poisson process followed exp
+        waits = rand_exp_array(options['timelambda'], txcount)
+        # number of makers to use follows a normal distribution
+        makercounts = rand_norm_array(options['makercountrange'][0],
+                                      options['makercountrange'][1], txcount)
+        makercounts = lower_bounded_int(makercounts, options['minmakercount'])
+
+        for amount_fraction, wait, makercount in zip(amount_fractions, waits,
+                                                     makercounts):
+            tx = {'amount_fraction': amount_fraction,
+                  'wait': round(wait, 2),
+                  'srcmixdepth': m + options['mixdepthsrc'],
+                  'makercount': makercount,
+                  'destination': 'INTERNAL'}
+            tx_list.append(tx)
+        #reset the final amt_frac to zero, as it's the last one for this mixdepth:
+        tx_list[-1]['amount_fraction'] = 0
+
+    addrask = options['addrcount'] - len(destaddrs)
+    external_dest_addrs = ['addrask'] * addrask + destaddrs
+    for mix_offset in range(options['addrcount']):
+        srcmix = (options['mixdepthsrc'] + options['mixdepthcount'] -
+            mix_offset - 1)
+        for tx in reversed(tx_list):
+            if tx['srcmixdepth'] == srcmix:
+                tx['destination'] = external_dest_addrs[mix_offset]
+                break
+        if mix_offset == 0:
+            # setting last mixdepth to send all to dest
+            tx_list_remove = []
+            for tx in tx_list:
+                if tx['srcmixdepth'] == srcmix:
+                    if tx['destination'] == 'INTERNAL':
+                        tx_list_remove.append(tx)
+                    else:
+                        tx['amount_fraction'] = 0
+            [tx_list.remove(t) for t in tx_list_remove]
+    schedule = []
+    for t in tx_list:
+        schedule.append((t['srcmixdepth'], t['amount_fraction'],
+                  t['makercount'], t['destination'], t['wait']))
+    return schedule

jmclient/jmclient/taker.py

@@ -83,6 +83,14 @@ class Taker(object):
             si = self.schedule[self.schedule_index]
             self.mixdepth = si[0]
             self.cjamount = si[1]
+            #non-integer coinjoin amounts are treated as fractions
+            #this is currently used by the tumbler algo
+            if isinstance(self.cjamount, float):
+                mixdepthbal = self.wallet.get_balance_by_mixdepth()[self.mixdepth]
+                self.cjamount = int(self.cjamount * mixdepthbal)
+                if self.cjamount < jm_single().mincjamount:
+                    jlog.debug("Coinjoin amount too low, bringing up.")
+                    self.cjamount = jm_single().mincjamount
             self.n_counterparties = si[2]
             self.my_cj_addr = si[3]
             #if destination is flagged "INTERNAL", choose a destination
@@ -403,6 +411,7 @@ class Taker(object):
         assert not len(self.nonrespondants)
         jlog.debug('all makers have sent their signatures')
         self.taker_info_callback("INFO", "Transaction is valid, signing..")
+        jlog.debug("schedule item was: " + str(self.schedule[self.schedule_index]))
         self.self_sign_and_push()
         return True
 
@@ -567,4 +576,5 @@ class Taker(object):
     def confirm_callback(self, txd, txid, confirmations):
         jlog.debug("Confirmed callback in taker, confs: " + str(confirmations))
         fromtx=False if self.schedule_index + 1 == len(self.schedule) else True
-        self.on_finished_callback(True, fromtx=fromtx)
+        waittime = self.schedule[self.schedule_index][4]
+        self.on_finished_callback(True, fromtx=fromtx, waittime=waittime)

scripts/cli_options.py

@@ -0,0 +1,276 @@
+#! /usr/bin/env python
+from __future__ import absolute_import, print_function
+
+"""This exists as a separate module for two reasons:
+to reduce clutter in main scripts, and (TODO) refactor out
+options which are common to more than one script in a base class.
+"""
+
+from optparse import OptionParser
+
+def get_tumbler_parser():
+    parser = OptionParser(
+            usage='usage: %prog [options] [wallet file] [destaddr(s)...]',
+            description=
+            'Sends bitcoins to many different addresses using coinjoin in'
+            ' an attempt to break the link between them. Sending to multiple '
+            ' addresses is highly recommended for privacy. This tumbler can'
+            ' be configured to ask for more address mid-run, giving the user'
+            ' a chance to click `Generate New Deposit Address` on whatever service'
+            ' they are using.')
+    parser.add_option(
+            '-m',
+            '--mixdepthsource',
+            type='int',
+            dest='mixdepthsrc',
+            help=
+            'Mixing depth to spend from. Useful if a previous tumbler run prematurely ended with '
+            +
+            'coins being left in higher mixing levels, this option can be used to resume without needing'
+            + ' to send to another address. default=0',
+            default=0)
+    parser.add_option(
+            '-f',
+        '--txfee',
+        action='store',
+        type='int',
+        dest='txfee',
+        default=-1,
+        help='number of satoshis per participant to use as the initial estimate '+
+        'for the total transaction fee, default=dynamically estimated, note that this is adjusted '+
+        'based on the estimated fee calculated after tx construction, based on '+
+        'policy set in joinmarket.cfg.')
+    parser.add_option(
+            '-a',
+            '--addrcount',
+            type='int',
+            dest='addrcount',
+            default=3,
+            help=
+            'How many destination addresses in total should be used. If not enough are given'
+            ' as command line arguments, the script will ask for more. This parameter is required'
+            ' to stop amount correlation. default=3')
+    parser.add_option(
+            '-x',
+            '--maxcjfee',
+            type='float',
+            dest='maxcjfee',
+            nargs=2,
+            default=(0.01, 10000),
+            help='maximum coinjoin fee and bitcoin value the tumbler is '
+                 'willing to pay to a single market maker. Both values need to be exceeded, so if '
+                 'the fee is 30% but only 500satoshi is paid the tx will go ahead. default=0.01, 10000 (1%, 10000satoshi)')
+    parser.add_option(
+            '-N',
+            '--makercountrange',
+            type='float',
+            nargs=2,
+            action='store',
+            dest='makercountrange',
+            help=
+            'Input the mean and spread of number of makers to use. e.g. 6 1 will be a normal distribution '
+            'with mean 6 and standard deviation 1 inclusive, default=6 1 (floats are also OK)',
+            default=(6, 1))
+    parser.add_option(
+            '--minmakercount',
+            type='int',
+            dest='minmakercount',
+            default=4,
+            help=
+            'The minimum maker count in a transaction, random values below this are clamped at this number. default=4')
+    parser.add_option(
+            '-M',
+            '--mixdepthcount',
+            type='int',
+            dest='mixdepthcount',
+            help='How many mixing depths to mix through',
+            default=4)
+    parser.add_option(
+            '-c',
+            '--txcountparams',
+            type='float',
+            nargs=2,
+            dest='txcountparams',
+            default=(4, 1),
+            help=
+            'The number of transactions to take coins from one mixing depth to the next, it is'
+            ' randomly chosen following a normal distribution. Should be similar to --addrask. '
+            'This option controls the parameters of the normal distribution curve. (mean, standard deviation). default=4 1')
+    parser.add_option(
+            '--mintxcount',
+            type='int',
+            dest='mintxcount',
+            default=1,
+            help='The minimum transaction count per mixing level, default=1')
+    parser.add_option(
+            '--donateamount',
+            type='float',
+            dest='donateamount',
+            default=0,
+            help=
+            'percent of funds to donate to joinmarket development, or zero to opt out (default=0%)')
+    parser.add_option(
+            '--amountpower',
+            type='float',
+            dest='amountpower',
+            default=100.0,
+            help=
+            'The output amounts follow a power law distribution, this is the power, default=100.0')
+    parser.add_option(
+            '-l',
+            '--timelambda',
+            type='float',
+            dest='timelambda',
+            default=30,
+            help=
+            'Average the number of minutes to wait between transactions. Randomly chosen '
+            ' following an exponential distribution, which describes the time between uncorrelated'
+            ' events. default=30')
+    parser.add_option(
+            '-w',
+            '--wait-time',
+            action='store',
+            type='float',
+            dest='waittime',
+            help='wait time in seconds to allow orders to arrive, default=20',
+            default=20)
+    parser.add_option(
+            '-s',
+            '--mincjamount',
+            type='int',
+            dest='mincjamount',
+            default=100000,
+            help='minimum coinjoin amount in transaction in satoshi, default 100k')
+    parser.add_option(
+            '-q',
+            '--liquiditywait',
+            type='int',
+            dest='liquiditywait',
+            default=60,
+            help=
+            'amount of seconds to wait after failing to choose suitable orders before trying again, default 60')
+    parser.add_option(
+            '--maxbroadcasts',
+            type='int',
+            dest='maxbroadcasts',
+            default=4,
+            help=
+            'maximum amount of times to broadcast a transaction before giving up and re-creating it, default 4')
+    parser.add_option(
+            '--maxcreatetx',
+            type='int',
+            dest='maxcreatetx',
+            default=9,
+            help=
+            'maximum amount of times to re-create a transaction before giving up, default 9')
+    parser.add_option('--fast',
+                      action='store_true',
+                      dest='fastsync',
+                      default=False,
+                      help=('choose to do fast wallet sync, only for Core and '
+                      'only for previously synced wallet'))
+    return parser
+
+def get_sendpayment_parser():
+    parser = OptionParser(
+        usage=
+        'usage: %prog [options] [wallet file / fromaccount] [amount] [destaddr]',
+        description='Sends a single payment from a given mixing depth of your '
+        +
+        'wallet to an given address using coinjoin and then switches off. Also sends from bitcoinqt. '
+        +
+        'Setting amount to zero will do a sweep, where the entire mix depth is emptied')
+    parser.add_option(
+        '-f',
+        '--txfee',
+        action='store',
+        type='int',
+        dest='txfee',
+        default=-1,
+        help=
+        'number of satoshis per participant to use as the initial estimate ' +
+        'for the total transaction fee, default=dynamically estimated, note that this is adjusted '
+        +
+        'based on the estimated fee calculated after tx construction, based on '
+        + 'policy set in joinmarket.cfg.')
+    parser.add_option(
+        '-w',
+        '--wait-time',
+        action='store',
+        type='float',
+        dest='waittime',
+        help='wait time in seconds to allow orders to arrive, default=15',
+        default=15)
+    parser.add_option(
+        '-N',
+        '--makercount',
+        action='store',
+        type='int',
+        dest='makercount',
+        help='how many makers to coinjoin with, default random from 4 to 6',
+        default=random.randint(4, 6))
+    parser.add_option('-S',
+                      '--schedule-file',
+                      type='str',
+                      dest='schedule',
+                      help='schedule file name',
+                      default='')
+    parser.add_option(
+        '-C',
+        '--choose-cheapest',
+        action='store_true',
+        dest='choosecheapest',
+        default=False,
+        help=
+        'override weightened offers picking and choose cheapest. this might reduce anonymity.')
+    parser.add_option(
+        '-P',
+        '--pick-orders',
+        action='store_true',
+        dest='pickorders',
+        default=False,
+        help=
+        'manually pick which orders to take. doesn\'t work while sweeping.')
+    parser.add_option('-m',
+                      '--mixdepth',
+                      action='store',
+                      type='int',
+                      dest='mixdepth',
+                      help='mixing depth to spend from, default=0',
+                      default=0)
+    parser.add_option('-a',
+                      '--amtmixdepths',
+                      action='store',
+                      type='int',
+                      dest='amtmixdepths',
+                      help='number of mixdepths in wallet, default 5',
+                      default=5)
+    parser.add_option('-g',
+                      '--gap-limit',
+                      type="int",
+                      action='store',
+                      dest='gaplimit',
+                      help='gap limit for wallet, default=6',
+                      default=6)
+    parser.add_option('--yes',
+                      action='store_true',
+                      dest='answeryes',
+                      default=False,
+                      help='answer yes to everything')
+    parser.add_option(
+        '--rpcwallet',
+        action='store_true',
+        dest='userpcwallet',
+        default=False,
+        help=('Use the Bitcoin Core wallet through json rpc, instead '
+              'of the internal joinmarket wallet. Requires '
+              'blockchain_source=json-rpc'))
+    parser.add_option('--fast',
+                      action='store_true',
+                      dest='fastsync',
+                      default=False,
+                      help=('choose to do fast wallet sync, only for Core and '
+                            'only for previously synced wallet'))
+    return parser
+
+    

scripts/sample-schedule-for-testnet

@@ -1,3 +1,5 @@
 #sample for testing
-1, 110000000, 3, INTERNAL
+#fields: source mixdepth, amount (satoshis or fraction if non-int),
-0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw
+# makercount, destination, waittime (ignored if 0)
+1, 110000000, 3, INTERNAL, 0
+0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw, 0

scripts/sendpayment.py

@@ -60,6 +60,7 @@ from jmclient import (Taker, load_program_config, get_schedule,
                               RegtestBitcoinCoreInterface, estimate_tx_fee)
 
 from jmbase.support import get_log, debug_dump_object, get_password
+from cli_options import get_sendpayment_parser
 
 log = get_log()
 
@@ -85,106 +86,7 @@ def pick_order(orders, n): #pragma: no cover
         pickedOrderIndex = -1
 
 def main():
-    parser = OptionParser(
+    parser = get_sendpayment_parser()
-        usage=
-        'usage: %prog [options] [wallet file / fromaccount] [amount] [destaddr]',
-        description='Sends a single payment from a given mixing depth of your '
-        +
-        'wallet to an given address using coinjoin and then switches off. Also sends from bitcoinqt. '
-        +
-        'Setting amount to zero will do a sweep, where the entire mix depth is emptied')
-    parser.add_option(
-        '-f',
-        '--txfee',
-        action='store',
-        type='int',
-        dest='txfee',
-        default=-1,
-        help=
-        'number of satoshis per participant to use as the initial estimate ' +
-        'for the total transaction fee, default=dynamically estimated, note that this is adjusted '
-        +
-        'based on the estimated fee calculated after tx construction, based on '
-        + 'policy set in joinmarket.cfg.')
-    parser.add_option(
-        '-w',
-        '--wait-time',
-        action='store',
-        type='float',
-        dest='waittime',
-        help='wait time in seconds to allow orders to arrive, default=15',
-        default=15)
-    parser.add_option(
-        '-N',
-        '--makercount',
-        action='store',
-        type='int',
-        dest='makercount',
-        help='how many makers to coinjoin with, default random from 4 to 6',
-        default=random.randint(4, 6))
-    parser.add_option('-S',
-                      '--schedule-file',
-                      type='str',
-                      dest='schedule',
-                      help='schedule file name',
-                      default='')
-    parser.add_option(
-        '-C',
-        '--choose-cheapest',
-        action='store_true',
-        dest='choosecheapest',
-        default=False,
-        help=
-        'override weightened offers picking and choose cheapest. this might reduce anonymity.')
-    parser.add_option(
-        '-P',
-        '--pick-orders',
-        action='store_true',
-        dest='pickorders',
-        default=False,
-        help=
-        'manually pick which orders to take. doesn\'t work while sweeping.')
-    parser.add_option('-m',
-                      '--mixdepth',
-                      action='store',
-                      type='int',
-                      dest='mixdepth',
-                      help='mixing depth to spend from, default=0',
-                      default=0)
-    parser.add_option('-a',
-                      '--amtmixdepths',
-                      action='store',
-                      type='int',
-                      dest='amtmixdepths',
-                      help='number of mixdepths in wallet, default 5',
-                      default=5)
-    parser.add_option('-g',
-                      '--gap-limit',
-                      type="int",
-                      action='store',
-                      dest='gaplimit',
-                      help='gap limit for wallet, default=6',
-                      default=6)
-    parser.add_option('--yes',
-                      action='store_true',
-                      dest='answeryes',
-                      default=False,
-                      help='answer yes to everything')
-    parser.add_option(
-        '--rpcwallet',
-        action='store_true',
-        dest='userpcwallet',
-        default=False,
-        help=('Use the Bitcoin Core wallet through json rpc, instead '
-              'of the internal joinmarket wallet. Requires '
-              'blockchain_source=json-rpc'))
-    parser.add_option('--fast',
-                      action='store_true',
-                      dest='fastsync',
-                      default=False,
-                      help=('choose to do fast wallet sync, only for Core and '
-                            'only for previously synced wallet'))
-
     (options, args) = parser.parse_args()
     load_program_config()
 
@@ -287,11 +189,11 @@ def main():
                 return False
         return True
 
-    def taker_finished(res, fromtx=False):
+    def taker_finished(res, fromtx=False, waittime=0.0):
         if fromtx:
             if res:
                 sync_wallet(wallet, fast=options.fastsync)
-                clientfactory.getClient().clientStart()
+                reactor.callLater(waittime, clientfactory.getClient().clientStart)
             else:
                 #a transaction failed; just stop
                 reactor.stop()

scripts/tumbler.py

@@ -0,0 +1,103 @@
+from __future__ import absolute_import, print_function
+
+import random
+import sys
+import threading
+from optparse import OptionParser
+from twisted.internet import reactor
+import time
+import os
+import pprint
+import copy
+
+from jmclient import (Taker, load_program_config, get_schedule, weighted_order_choose,
+                              JMTakerClientProtocolFactory, start_reactor,
+                              validate_address, jm_single, WalletError,
+                              Wallet, sync_wallet, get_tumble_schedule,
+                              RegtestBitcoinCoreInterface, estimate_tx_fee)
+
+from jmbase.support import get_log, debug_dump_object, get_password
+from cli_options import get_tumbler_parser
+log = get_log()
+
+def main():
+    (options, args) = get_tumbler_parser().parse_args()
+    options = vars(options)
+
+    if len(args) < 1:
+        parser.error('Needs a wallet file')
+        sys.exit(0)
+
+    load_program_config()
+
+    #Load the wallet
+    wallet_name = args[0]
+    max_mix_depth = options['mixdepthsrc'] + options['mixdepthcount']
+    if not os.path.exists(os.path.join('wallets', wallet_name)):
+        wallet = Wallet(wallet_name, None, max_mix_depth)
+    else:
+        while True:
+            try:
+                pwd = get_password("Enter wallet decryption passphrase: ")
+                wallet = Wallet(wallet_name, pwd, max_mix_depth)
+            except WalletError:
+                print("Wrong password, try again.")
+                continue
+            except Exception as e:
+                print("Failed to load wallet, error message: " + repr(e))
+                sys.exit(0)
+            break
+    sync_wallet(wallet, fast=options['fastsync'])
+
+    #Parse options and generate schedule
+    
+    #for testing, TODO remove
+    jm_single().maker_timeout_sec = 5 
+    
+    jm_single().mincjamount = options['mincjamount']
+    destaddrs = args[1:]
+    print(destaddrs)
+    schedule = get_tumble_schedule(options, destaddrs)
+    print("got schedule:")
+    print(pprint.pformat(schedule))
+
+    #callback for order checking; dummy/passthrough
+    def filter_orders_callback(orders_fees, cjamount):
+        return True
+    #callback between transactions
+    def taker_finished(res, fromtx=False, waittime=0.0):
+        if fromtx:
+            if res:
+                sync_wallet(wallet, fast=options['fastsync'])
+                log.info("Waiting for: " + str(waittime) + " seconds.")
+                reactor.callLater(waittime, clientfactory.getClient().clientStart)
+            else:
+                #a transaction failed; just stop
+                reactor.stop()
+        else:
+            if not res:
+                log.info("Did not complete successfully, shutting down")
+            else:
+                log.info("All transactions completed correctly")
+            reactor.stop()
+
+    #to allow testing of confirm/unconfirm callback for multiple txs
+    if isinstance(jm_single().bc_interface, RegtestBitcoinCoreInterface):
+        jm_single().bc_interface.tick_forward_chain_interval = 10
+
+    #instantiate Taker with given schedule and run
+    taker = Taker(wallet,
+                  schedule,
+                  order_chooser=weighted_order_choose,
+                  callbacks=(filter_orders_callback, None, taker_finished))
+    clientfactory = JMTakerClientProtocolFactory(taker)
+    nodaemon = jm_single().config.getint("DAEMON", "no_daemon")
+    daemon = True if nodaemon == 1 else False
+    start_reactor(jm_single().config.get("DAEMON", "daemon_host"),
+                  jm_single().config.getint("DAEMON", "daemon_port"),
+                  clientfactory, daemon=daemon)
+
+
+if __name__ == "__main__":
+    main()
+    print('done')