Merge #727: Move YG settings to a config file

a490ddf Move YG settings to a config file (dchoe)

jmclient/jmclient/configure.py

@@ -322,6 +322,10 @@ accept_commitment_broadcasts = 1
 #and those you want to use in future), relative to the scripts directory.
 commit_file_location = cmtdata/commitments.json
 
+##############################
+# END OF ANTI-SNOOPING SETTINGS
+##############################
+
 [PAYJOIN]
 # for the majority of situations, the defaults
 # need not be altered - they will ensure you don't pay
@@ -365,6 +369,33 @@ tor_control_port = 9051
 # this feature is not yet implemented in code, but here for the
 # future:
 hidden_service_ssl = false
+
+[YIELDGENERATOR]
+# [string, 'reloffer' or 'absoffer'], which fee type to actually use
+ordertype = reloffer
+
+# [satoshis, any integer] / absolute offer fee you wish to receive for coinjoins (cj)
+cjfee_a = 500
+
+# [fraction, any str between 0-1] / relative offer fee you wish to receive based on a cj's amount
+cjfee_r = 0.00002
+
+# [fraction, 0-1] / variance around the average fee. Ex: 200 fee, 0.2 var = fee is btw 160-240
+cjfee_factor = 0.1
+
+# [satoshis, any integer] / the average transaction fee you're adding to coinjoin transactions
+txfee = 100
+
+# [fraction, 0-1] / variance around the average fee. Ex: 1000 fee, 0.2 var = fee is btw 800-1200
+txfee_factor = 0.3
+
+# [satoshis, any integer] / minimum size of your cj offer. Lower cj amounts will be disregarded
+minsize = 100000
+
+# [fraction, 0-1] / variance around all offer sizes. Ex: 500k minsize, 0.1 var = 450k-550k
+size_factor = 0.1
+
+gaplimit = 6
 """
 
 #This allows use of the jmclient package with a

jmclient/jmclient/yieldgenerator.py

@@ -71,8 +71,9 @@ class YieldGeneratorBasic(YieldGenerator):
     thus is somewhat suboptimal in giving more information to spies.
     """
     def __init__(self, wallet_service, offerconfig):
-        self.txfee, self.cjfee_a, self.cjfee_r, self.ordertype, self.minsize \
+        # note the randomizing entries are ignored in this base class:
-             = offerconfig
+        self.txfee, self.cjfee_a, self.cjfee_r, self.ordertype, self.minsize, \
+            self.txfee_factor, self.cjfee_factor, self.size_factor = offerconfig
         super().__init__(wallet_service)
 
     def create_my_orders(self):
@@ -188,27 +189,50 @@ class YieldGeneratorBasic(YieldGenerator):
         return self.wallet_service.get_internal_addr(cjoutmix)
 
 
-def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='reloffer',
+def ygmain(ygclass, nickserv_password='', gaplimit=6):
-           nickserv_password='', minsize=100000, gaplimit=6):
     import sys
 
     parser = OptionParser(usage='usage: %prog [options] [wallet file]')
     add_base_options(parser)
+    # A note about defaults:
+    # We want command line settings to override config settings.
+    # This would naturally mean setting `default=` arguments here, to the
+    # values in the config.
+    # However, we cannot load the config until we know the datadir.
+    # The datadir is a setting in the command line options, so we have to
+    # call parser.parse_args() before we know the datadir.
+    # Hence we do the following: set all modifyable-by-config arguments to
+    # default "None" initially; call parse_args(); then call load_program_config
+    # and override values of "None" with what is set in the config.
+    # (remember, the joinmarket defaultconfig always sets every value, even if
+    # the user doesn't).
     parser.add_option('-o', '--ordertype', action='store', type='string',
-                      dest='ordertype', default=ordertype,
+                      dest='ordertype', default=None,
                       help='type of order; can be either reloffer or absoffer')
     parser.add_option('-t', '--txfee', action='store', type='int',
-                      dest='txfee', default=txfee,
+                      dest='txfee', default=None,
                       help='minimum miner fee in satoshis')
-    parser.add_option('-c', '--cjfee', action='store', type='string',
+    parser.add_option('-f', '--txfee-factor', action='store', type='float',
-                      dest='cjfee', default='',
+                      dest='txfee_factor', default=None,
-                      help='requested coinjoin fee in satoshis or proportion')
+                      help='variance around the average fee, decimal fraction')
+    parser.add_option('-a', '--cjfee-a', action='store', type='string',
+                      dest='cjfee_a', default=None,
+                      help='requested coinjoin fee (absolute) in satoshis')
+    parser.add_option('-r', '--cjfee-r', action='store', type='string',
+                      dest='cjfee_r', default=None,
+                      help='requested coinjoin fee (relative) as a decimal')
+    parser.add_option('-j', '--cjfee-factor', action='store', type='float',
+                      dest='cjfee_factor', default=None,
+                      help='variance around the average fee, decimal fraction')
     parser.add_option('-p', '--password', action='store', type='string',
                       dest='password', default=nickserv_password,
                       help='irc nickserv password')
     parser.add_option('-s', '--minsize', action='store', type='int',
-                      dest='minsize', default=minsize,
+                      dest='minsize', default=None,
                       help='minimum coinjoin size in satoshis')
+    parser.add_option('-z', '--size-factor', action='store', type='float',
+                      dest='size_factor', default=None,
+                      help='variance around all offer sizes, decimal fraction')
     parser.add_option('-g', '--gap-limit', action='store', type="int",
                       dest='gaplimit', default=gaplimit,
                       help='gap limit for wallet, default='+str(gaplimit))
@@ -216,29 +240,40 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='reloffer'
                       dest='mixdepth', default=None,
                       help="highest mixdepth to use")
     (options, args) = parser.parse_args()
+    # for string access, convert to dict:
+    options = vars(options)
     if len(args) < 1:
         parser.error('Needs a wallet')
         sys.exit(EXIT_ARGERROR)
+
+    load_program_config(config_path=options["datadir"])
+
+    # As per previous note, override non-default command line settings:
+    for x in ["ordertype", "txfee", "txfee_factor", "cjfee_a", "cjfee_r",
+              "cjfee_factor", "minsize", "size_factor"]:
+        if options[x] is None:
+            options[x] = jm_single().config.get("YIELDGENERATOR", x)
     wallet_name = args[0]
-    ordertype = options.ordertype
+    ordertype = options["ordertype"]
-    txfee = options.txfee
+    txfee = int(options["txfee"])
+    txfee_factor = float(options["txfee_factor"])
+    cjfee_factor = float(options["cjfee_factor"])
+    size_factor = float(options["size_factor"])
     if ordertype == 'reloffer':
-        if options.cjfee != '':
+        cjfee_r = options["cjfee_r"]
-            cjfee_r = options.cjfee
         # minimum size is such that you always net profit at least 20%
         #of the miner fee
-        minsize = max(int(1.2 * txfee / float(cjfee_r)), options.minsize)
+        minsize = max(int(1.2 * txfee / float(cjfee_r)), int(options["minsize"]))
+        cjfee_a = None
     elif ordertype == 'absoffer':
-        if options.cjfee != '':
+        cjfee_a = int(options["cjfee_a"])
-            cjfee_a = int(options.cjfee)
+        minsize = int(options["minsize"])
-        minsize = options.minsize
+        cjfee_r = None
     else:
         parser.error('You specified an incorrect offer type which ' +\
                      'can be either reloffer or absoffer')
         sys.exit(EXIT_ARGERROR)
-    nickserv_password = options.password
+    nickserv_password = options["password"]
-
-    load_program_config(config_path=options.datadir)
 
     if jm_single().bc_interface is None:
         jlog.error("Running yield generator requires configured " +
@@ -247,13 +282,13 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='reloffer'
 
     wallet_path = get_wallet_path(wallet_name, None)
     wallet = open_test_wallet_maybe(
-        wallet_path, wallet_name, options.mixdepth,
+        wallet_path, wallet_name, options["mixdepth"],
-        wallet_password_stdin=options.wallet_password_stdin,
+        wallet_password_stdin=options["wallet_password_stdin"],
-        gap_limit=options.gaplimit)
+        gap_limit=options["gaplimit"])
 
     wallet_service = WalletService(wallet)
     while not wallet_service.synced:
-        wallet_service.sync_wallet(fast=not options.recoversync)
+        wallet_service.sync_wallet(fast=not options["recoversync"])
     wallet_service.startService()
 
     txtype = wallet_service.get_txtype()
@@ -270,8 +305,9 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='reloffer'
     ordertype = prefix + ordertype
     jlog.debug("Set the offer type string to: " + ordertype)
 
-    maker = ygclass(wallet_service, [options.txfee, cjfee_a, cjfee_r,
+    maker = ygclass(wallet_service, [txfee, cjfee_a, cjfee_r,
-                             ordertype, minsize])
+                             ordertype, minsize, txfee_factor,
+                             cjfee_factor, size_factor])
     jlog.info('starting yield generator')
     clientfactory = JMClientProtocolFactory(maker, proto_type="MAKER")
 

jmclient/test/test_coinjoin.py

@@ -141,7 +141,7 @@ def test_simple_coinjoin(monkeypatch, tmpdir, setup_cj, wallet_cls):
 
     makers = [YieldGeneratorBasic(
         wallet_services[i],
-        [0, 2000, 0, absoffer_type_map[wallet_cls], 10**7]) for i in range(MAKER_NUM)]
+        [0, 2000, 0, absoffer_type_map[wallet_cls], 10**7, None, None, None]) for i in range(MAKER_NUM)]
     create_orders(makers)
 
     orderbook = create_orderbook(makers)
@@ -186,7 +186,7 @@ def test_coinjoin_mixdepth_wrap_taker(monkeypatch, tmpdir, setup_cj):
     cj_fee = 2000
     makers = [YieldGeneratorBasic(
         wallet_services[i],
-        [0, cj_fee, 0, absoffer_type_map[SegwitWallet], 10**7]) for i in range(MAKER_NUM)]
+        [0, cj_fee, 0, absoffer_type_map[SegwitWallet], 10**7, None, None, None]) for i in range(MAKER_NUM)]
     create_orders(makers)
 
     orderbook = create_orderbook(makers)
@@ -242,7 +242,7 @@ def test_coinjoin_mixdepth_wrap_maker(monkeypatch, tmpdir, setup_cj):
     cj_fee = 2000
     makers = [YieldGeneratorBasic(
         wallet_services[i],
-        [0, cj_fee, 0, absoffer_type_map[SegwitWallet], 10**7]) for i in range(MAKER_NUM)]
+        [0, cj_fee, 0, absoffer_type_map[SegwitWallet], 10**7, None, None, None]) for i in range(MAKER_NUM)]
     create_orders(makers)
     orderbook = create_orderbook(makers)
     assert len(orderbook) == MAKER_NUM

jmclient/test/test_yieldgenerator.py

@@ -49,7 +49,7 @@ def create_yg_basic(balances, txfee=0, cjfee_a=0, cjfee_r=0,
     will be set as given here."""
 
     wallet = CustomUtxoWallet(balances)
-    offerconfig = (txfee, cjfee_a, cjfee_r, ordertype, minsize)
+    offerconfig = (txfee, cjfee_a, cjfee_r, ordertype, minsize, None, None, None)
 
     yg = YieldGeneratorBasic(WalletService(wallet), offerconfig)
 

scripts/yg-privacyenhanced.py

@@ -4,28 +4,16 @@ from future.utils import iteritems
 import random
 
 from jmbase import get_log, jmprint
+from jmbase.support import lookup_appdata_folder
 from jmclient import YieldGeneratorBasic, ygmain, jm_single
 
-
 # This is a maker for the purposes of generating a yield from held bitcoins
 # while maximising the difficulty of spying on blockchain activity.
 # This is primarily attempted by randomizing all aspects of orders
 # after transactions wherever possible.
 
-"""THESE SETTINGS CAN SIMPLY BE EDITED BY HAND IN THIS FILE:
+# YIELD GENERATOR SETTINGS ARE NOW IN YOUR joinmarket.cfg CONFIG FILE
-"""
+# (You can also use command line flags; see --help for this script).
-
-ordertype = 'reloffer'  # [string, 'reloffer' or 'absoffer'], which fee type to actually use
-cjfee_a = 500             # [satoshis, any integer] / absolute offer fee you wish to receive for coinjoins (cj)
-cjfee_r = '0.00002'       # [fraction, any str between 0-1] / relative offer fee you wish to receive based on a cj's amount
-cjfee_factor = 0.1        # [fraction, 0-1] / variance around the average fee. Ex: 200 fee, 0.2 var = fee is btw 160-240
-txfee = 0                 # [satoshis, any integer] / the average transaction fee contribution you're adding to coinjoin transactions
-txfee_factor = 0.3        # [fraction, 0-1] / variance around the average fee contribution. Ex: 1000 fee, 0.2 var = fee is btw 800-1200
-minsize = 100000          # [satoshis, any integer] / minimum size of your cj offer. Lower cj amounts will be disregarded
-size_factor = 0.1         # [fraction, 0-1] / variance around all offer sizes. Ex: 500k minsize, 0.1 var = 450k-550k
-gaplimit = 6
-
-# end of settings customization
 
 jlog = get_log()
 
@@ -53,21 +41,21 @@ class YieldGeneratorPrivacyEnhanced(YieldGeneratorBasic):
         max_mix = max(mix_balance, key=mix_balance.get)
 
         # randomizing the different values
-        randomize_txfee = int(random.uniform(txfee * (1 - float(txfee_factor)),
+        randomize_txfee = int(random.uniform(self.txfee * (1 - float(self.txfee_factor)),
-                                             txfee * (1 + float(txfee_factor))))
+                                             self.txfee * (1 + float(self.txfee_factor))))
-        randomize_minsize = int(random.uniform(self.minsize * (1 - float(size_factor)),
+        randomize_minsize = int(random.uniform(self.minsize * (1 - float(self.size_factor)),
-                                               self.minsize * (1 + float(size_factor))))
+                                               self.minsize * (1 + float(self.size_factor))))
         possible_maxsize = mix_balance[max_mix] - max(jm_single().DUST_THRESHOLD, randomize_txfee)
-        randomize_maxsize = int(random.uniform(possible_maxsize * (1 - float(size_factor)),
+        randomize_maxsize = int(random.uniform(possible_maxsize * (1 - float(self.size_factor)),
                                                possible_maxsize))
 
         if self.ordertype in ['swabsoffer', 'sw0absoffer']:
-            randomize_cjfee = int(random.uniform(float(cjfee_a) * (1 - float(cjfee_factor)),
+            randomize_cjfee = int(random.uniform(float(self.cjfee_a) * (1 - float(self.cjfee_factor)),
-                                                 float(cjfee_a) * (1 + float(cjfee_factor))))
+                                                 float(self.cjfee_a) * (1 + float(self.cjfee_factor))))
             randomize_cjfee = randomize_cjfee + randomize_txfee
         else:
-            randomize_cjfee = random.uniform(float(f) * (1 - float(cjfee_factor)),
+            randomize_cjfee = random.uniform(float(f) * (1 - float(self.cjfee_factor)),
-                                             float(f) * (1 + float(cjfee_factor)))
+                                             float(f) * (1 + float(self.cjfee_factor)))
             randomize_cjfee = "{0:.6f}".format(randomize_cjfee)  # round to 6 decimals
 
         order = {'oid': 0,
@@ -90,8 +78,5 @@ class YieldGeneratorPrivacyEnhanced(YieldGeneratorBasic):
 
 
 if __name__ == "__main__":
-    ygmain(YieldGeneratorPrivacyEnhanced, txfee=txfee, cjfee_a=cjfee_a,
+    ygmain(YieldGeneratorPrivacyEnhanced, nickserv_password='')
-           cjfee_r=cjfee_r, ordertype=ordertype,
-           nickserv_password='',
-           minsize=minsize, gaplimit=gaplimit)
     jmprint('done', "success")

scripts/yield-generator-basic.py

@@ -3,20 +3,9 @@
 from jmbase import jmprint
 from jmclient import YieldGeneratorBasic, ygmain
 
-"""THESE SETTINGS CAN SIMPLY BE EDITED BY HAND IN THIS FILE:
+# YIELD GENERATOR SETTINGS ARE NOW IN YOUR joinmarket.cfg CONFIG FILE
-"""
+# (You can also use command line flags; see --help for this script).
-
-ordertype = 'reloffer' # [string, 'reloffer' or 'absoffer'], which fee type to actually use
-cjfee_a = 500 # [satoshis, any integer] / absolute offer fee you wish to receive for coinjoins (cj)
-cjfee_r = '0.00002' # [fraction, any str between 0-1] / relative offer fee you wish to receive based on a cj's amount
-txfee = 0 # [satoshis, any integer] / the transaction fee contribution you're adding to coinjoin transactions
-nickserv_password = ''
-minsize = 100000 # [satoshis, any integer] / minimum size of your cj offer. Lower cj amounts will be disregarded
-gaplimit = 6
 
 if __name__ == "__main__":
-    ygmain(YieldGeneratorBasic, txfee=txfee, cjfee_a=cjfee_a,
+    ygmain(YieldGeneratorBasic, nickserv_password='')
-           cjfee_r=cjfee_r, ordertype=ordertype,
-           nickserv_password=nickserv_password,
-           minsize=minsize, gaplimit=gaplimit)
     jmprint('done', "success")

test/ygrunner.py

@@ -117,12 +117,43 @@ def test_start_ygs(setup_ygrunner, num_ygs, wallet_structures, mean_amt,
         jmprint("Maker seed: " + wallet_services[i]['seed'])
     jmprint("\n")
     wallet_service.sync_wallet(fast=True)
-    txfee = 1000
-    cjfee_a = 4200
-    cjfee_r = '0.001'
-    ordertype = 'sw0reloffer'
-    minsize = 100000
     ygclass = YieldGeneratorBasic
+
+    # As per previous note, override non-default command line settings:
+    options = {}
+    for x in ["ordertype", "txfee", "txfee_factor", "cjfee_a", "cjfee_r",
+              "cjfee_factor", "minsize", "size_factor"]:
+        options[x] = jm_single().config.get("YIELDGENERATOR", x)
+    ordertype = options["ordertype"]
+    txfee = int(options["txfee"])
+    txfee_factor = float(options["txfee_factor"])
+    cjfee_factor = float(options["cjfee_factor"])
+    size_factor = float(options["size_factor"])
+    if ordertype == 'reloffer':
+        cjfee_r = options["cjfee_r"]
+        # minimum size is such that you always net profit at least 20%
+        #of the miner fee
+        minsize = max(int(1.2 * txfee / float(cjfee_r)), int(options["minsize"]))
+        cjfee_a = None
+    elif ordertype == 'absoffer':
+        cjfee_a = int(options["cjfee_a"])
+        minsize = int(options["minsize"])
+        cjfee_r = None
+    else:
+        assert False, "incorrect offertype config for yieldgenerator."
+
+    txtype = wallet_service.get_txtype()
+    if txtype == "p2wpkh":
+        prefix = "sw0"
+    elif txtype == "p2sh-p2wpkh":
+        prefix = "sw"
+    elif txtype == "p2pkh":
+        prefix = ""
+    else:
+        assert False, "Unsupported wallet type for yieldgenerator: " + txtype
+
+    ordertype = prefix + ordertype
+
     if malicious:
         if deterministic:
             ygclass = DeterministicMaliciousYieldGenerator
@@ -130,7 +161,8 @@ def test_start_ygs(setup_ygrunner, num_ygs, wallet_structures, mean_amt,
             ygclass = MaliciousYieldGenerator
     for i in range(num_ygs):
         
-        cfg = [txfee, cjfee_a, cjfee_r, ordertype, minsize]
+        cfg = [txfee, cjfee_a, cjfee_r, ordertype, minsize, txfee_factor,
+               cjfee_factor, size_factor]
         wallet_service_yg = wallet_services[i]["wallet"]
         wallet_service_yg.startService()
         yg = ygclass(wallet_service_yg, cfg)