Move YG settings to a config file

Allow YG settings to be saved to joinmarket.cfg.
Before this commit, yield generator settings were
set inside the python script, which is not good.
After this commit, the order of precedence for the
settings for a yield generator is:
* Command line arguments, or
* Settings in [YIELDGENERATOR] in joinmarket.cfg, or
* default config settings in jmclient.configure.py

jmclient/jmclient/configure.py

@@ -306,6 +306,10 @@ accept_commitment_broadcasts = 1
 #and those you want to use in future), relative to the scripts directory.
 commit_file_location = cmtdata/commitments.json
 
+##############################
+# END OF ANTI-SNOOPING SETTINGS
+##############################
+
 [PAYJOIN]
 # for the majority of situations, the defaults
 # need not be altered - they will ensure you don't pay
@@ -349,6 +353,33 @@ tor_control_port = 9051
 # this feature is not yet implemented in code, but here for the
 # future:
 hidden_service_ssl = false
+
+[YIELDGENERATOR]
+# [string, 'reloffer' or 'absoffer'], which fee type to actually use
+ordertype = reloffer
+
+# [satoshis, any integer] / absolute offer fee you wish to receive for coinjoins (cj)
+cjfee_a = 500
+
+# [fraction, any str between 0-1] / relative offer fee you wish to receive based on a cj's amount
+cjfee_r = 0.00002
+
+# [fraction, 0-1] / variance around the average fee. Ex: 200 fee, 0.2 var = fee is btw 160-240
+cjfee_factor = 0.1
+
+# [satoshis, any integer] / the average transaction fee you're adding to coinjoin transactions
+txfee = 100
+
+# [fraction, 0-1] / variance around the average fee. Ex: 1000 fee, 0.2 var = fee is btw 800-1200
+txfee_factor = 0.3
+
+# [satoshis, any integer] / minimum size of your cj offer. Lower cj amounts will be disregarded
+minsize = 100000
+
+# [fraction, 0-1] / variance around all offer sizes. Ex: 500k minsize, 0.1 var = 450k-550k
+size_factor = 0.1
+
+gaplimit = 6
 """
 
 #This allows use of the jmclient package with a

jmclient/jmclient/yieldgenerator.py

@@ -71,8 +71,9 @@ class YieldGeneratorBasic(YieldGenerator):
     thus is somewhat suboptimal in giving more information to spies.
     """
     def __init__(self, wallet_service, offerconfig):
-        self.txfee, self.cjfee_a, self.cjfee_r, self.ordertype, self.minsize \
+        # note the randomizing entries are ignored in this base class:
-             = offerconfig
+        self.txfee, self.cjfee_a, self.cjfee_r, self.ordertype, self.minsize, \
+            self.txfee_factor, self.cjfee_factor, self.size_factor = offerconfig
         super().__init__(wallet_service)
 
     def create_my_orders(self):
@@ -188,27 +189,50 @@ class YieldGeneratorBasic(YieldGenerator):
         return self.wallet_service.get_internal_addr(cjoutmix)
 
 
-def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='reloffer',
+def ygmain(ygclass, nickserv_password='', gaplimit=6):
-           nickserv_password='', minsize=100000, gaplimit=6):
     import sys
 
     parser = OptionParser(usage='usage: %prog [options] [wallet file]')
     add_base_options(parser)
+    # A note about defaults:
+    # We want command line settings to override config settings.
+    # This would naturally mean setting `default=` arguments here, to the
+    # values in the config.
+    # However, we cannot load the config until we know the datadir.
+    # The datadir is a setting in the command line options, so we have to
+    # call parser.parse_args() before we know the datadir.
+    # Hence we do the following: set all modifyable-by-config arguments to
+    # default "None" initially; call parse_args(); then call load_program_config
+    # and override values of "None" with what is set in the config.
+    # (remember, the joinmarket defaultconfig always sets every value, even if
+    # the user doesn't).
     parser.add_option('-o', '--ordertype', action='store', type='string',
-                      dest='ordertype', default=ordertype,
+                      dest='ordertype', default=None,
                       help='type of order; can be either reloffer or absoffer')
     parser.add_option('-t', '--txfee', action='store', type='int',
-                      dest='txfee', default=txfee,
+                      dest='txfee', default=None,
                       help='minimum miner fee in satoshis')
-    parser.add_option('-c', '--cjfee', action='store', type='string',
+    parser.add_option('-f', '--txfee-factor', action='store', type='float',
-                      dest='cjfee', default='',
+                      dest='txfee_factor', default=None,
-                      help='requested coinjoin fee in satoshis or proportion')
+                      help='variance around the average fee, decimal fraction')
+    parser.add_option('-a', '--cjfee-a', action='store', type='string',
+                      dest='cjfee_a', default=None,
+                      help='requested coinjoin fee (absolute) in satoshis')
+    parser.add_option('-r', '--cjfee-r', action='store', type='string',
+                      dest='cjfee_r', default=None,
+                      help='requested coinjoin fee (relative) as a decimal')
+    parser.add_option('-j', '--cjfee-factor', action='store', type='float',
+                      dest='cjfee_factor', default=None,
+                      help='variance around the average fee, decimal fraction')
     parser.add_option('-p', '--password', action='store', type='string',
                       dest='password', default=nickserv_password,
                       help='irc nickserv password')
     parser.add_option('-s', '--minsize', action='store', type='int',
-                      dest='minsize', default=minsize,
+                      dest='minsize', default=None,
                       help='minimum coinjoin size in satoshis')
+    parser.add_option('-z', '--size-factor', action='store', type='float',
+                      dest='size_factor', default=None,
+                      help='variance around all offer sizes, decimal fraction')
     parser.add_option('-g', '--gap-limit', action='store', type="int",
                       dest='gaplimit', default=gaplimit,
                       help='gap limit for wallet, default='+str(gaplimit))
@@ -216,29 +240,40 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='reloffer'
                       dest='mixdepth', default=None,
                       help="highest mixdepth to use")
     (options, args) = parser.parse_args()
+    # for string access, convert to dict:
+    options = vars(options)
     if len(args) < 1:
         parser.error('Needs a wallet')
         sys.exit(EXIT_ARGERROR)
+
+    load_program_config(config_path=options["datadir"])
+
+    # As per previous note, override non-default command line settings:
+    for x in ["ordertype", "txfee", "txfee_factor", "cjfee_a", "cjfee_r",
+              "cjfee_factor", "minsize", "size_factor"]:
+        if options[x] is None:
+            options[x] = jm_single().config.get("YIELDGENERATOR", x)
     wallet_name = args[0]
-    ordertype = options.ordertype
+    ordertype = options["ordertype"]
-    txfee = options.txfee
+    txfee = int(options["txfee"])
+    txfee_factor = float(options["txfee_factor"])
+    cjfee_factor = float(options["cjfee_factor"])
+    size_factor = float(options["size_factor"])
     if ordertype == 'reloffer':
-        if options.cjfee != '':
+        cjfee_r = options["cjfee_r"]
-            cjfee_r = options.cjfee
         # minimum size is such that you always net profit at least 20%
         #of the miner fee
-        minsize = max(int(1.2 * txfee / float(cjfee_r)), options.minsize)
+        minsize = max(int(1.2 * txfee / float(cjfee_r)), int(options["minsize"]))
+        cjfee_a = None
     elif ordertype == 'absoffer':
-        if options.cjfee != '':
+        cjfee_a = int(options["cjfee_a"])
-            cjfee_a = int(options.cjfee)
+        minsize = int(options["minsize"])
-        minsize = options.minsize
+        cjfee_r = None
     else:
         parser.error('You specified an incorrect offer type which ' +\
                      'can be either reloffer or absoffer')
         sys.exit(EXIT_ARGERROR)
-    nickserv_password = options.password
+    nickserv_password = options["password"]
-
-    load_program_config(config_path=options.datadir)
 
     if jm_single().bc_interface is None:
         jlog.error("Running yield generator requires configured " +
@@ -247,13 +282,13 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='reloffer'
 
     wallet_path = get_wallet_path(wallet_name, None)
     wallet = open_test_wallet_maybe(
-        wallet_path, wallet_name, options.mixdepth,
+        wallet_path, wallet_name, options["mixdepth"],
-        wallet_password_stdin=options.wallet_password_stdin,
+        wallet_password_stdin=options["wallet_password_stdin"],
-        gap_limit=options.gaplimit)
+        gap_limit=options["gaplimit"])
 
     wallet_service = WalletService(wallet)
     while not wallet_service.synced:
-        wallet_service.sync_wallet(fast=not options.recoversync)
+        wallet_service.sync_wallet(fast=not options["recoversync"])
     wallet_service.startService()
 
     txtype = wallet_service.get_txtype()
@@ -270,8 +305,9 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='reloffer'
     ordertype = prefix + ordertype
     jlog.debug("Set the offer type string to: " + ordertype)
 
-    maker = ygclass(wallet_service, [options.txfee, cjfee_a, cjfee_r,
+    maker = ygclass(wallet_service, [txfee, cjfee_a, cjfee_r,
-                             ordertype, minsize])
+                             ordertype, minsize, txfee_factor,
+                             cjfee_factor, size_factor])
     jlog.info('starting yield generator')
     clientfactory = JMClientProtocolFactory(maker, proto_type="MAKER")
 

jmclient/test/test_coinjoin.py

@@ -141,7 +141,7 @@ def test_simple_coinjoin(monkeypatch, tmpdir, setup_cj, wallet_cls):
 
     makers = [YieldGeneratorBasic(
         wallet_services[i],
-        [0, 2000, 0, absoffer_type_map[wallet_cls], 10**7]) for i in range(MAKER_NUM)]
+        [0, 2000, 0, absoffer_type_map[wallet_cls], 10**7, None, None, None]) for i in range(MAKER_NUM)]
     create_orders(makers)
 
     orderbook = create_orderbook(makers)
@@ -186,7 +186,7 @@ def test_coinjoin_mixdepth_wrap_taker(monkeypatch, tmpdir, setup_cj):
     cj_fee = 2000
     makers = [YieldGeneratorBasic(
         wallet_services[i],
-        [0, cj_fee, 0, absoffer_type_map[SegwitWallet], 10**7]) for i in range(MAKER_NUM)]
+        [0, cj_fee, 0, absoffer_type_map[SegwitWallet], 10**7, None, None, None]) for i in range(MAKER_NUM)]
     create_orders(makers)
 
     orderbook = create_orderbook(makers)
@@ -242,7 +242,7 @@ def test_coinjoin_mixdepth_wrap_maker(monkeypatch, tmpdir, setup_cj):
     cj_fee = 2000
     makers = [YieldGeneratorBasic(
         wallet_services[i],
-        [0, cj_fee, 0, absoffer_type_map[SegwitWallet], 10**7]) for i in range(MAKER_NUM)]
+        [0, cj_fee, 0, absoffer_type_map[SegwitWallet], 10**7, None, None, None]) for i in range(MAKER_NUM)]
     create_orders(makers)
     orderbook = create_orderbook(makers)
     assert len(orderbook) == MAKER_NUM

jmclient/test/test_yieldgenerator.py

@@ -49,7 +49,7 @@ def create_yg_basic(balances, txfee=0, cjfee_a=0, cjfee_r=0,
     will be set as given here."""
 
     wallet = CustomUtxoWallet(balances)
-    offerconfig = (txfee, cjfee_a, cjfee_r, ordertype, minsize)
+    offerconfig = (txfee, cjfee_a, cjfee_r, ordertype, minsize, None, None, None)
 
     yg = YieldGeneratorBasic(WalletService(wallet), offerconfig)
 

scripts/yg-privacyenhanced.py

@@ -4,28 +4,16 @@ from future.utils import iteritems
 import random
 
 from jmbase import get_log, jmprint
+from jmbase.support import lookup_appdata_folder
 from jmclient import YieldGeneratorBasic, ygmain, jm_single
 
-
 # This is a maker for the purposes of generating a yield from held bitcoins
 # while maximising the difficulty of spying on blockchain activity.
 # This is primarily attempted by randomizing all aspects of orders
 # after transactions wherever possible.
 
-"""THESE SETTINGS CAN SIMPLY BE EDITED BY HAND IN THIS FILE:
+# YIELD GENERATOR SETTINGS ARE NOW IN YOUR joinmarket.cfg CONFIG FILE
-"""
+# (You can also use command line flags; see --help for this script).
-
-ordertype = 'reloffer'  # [string, 'reloffer' or 'absoffer'], which fee type to actually use
-cjfee_a = 500             # [satoshis, any integer] / absolute offer fee you wish to receive for coinjoins (cj)
-cjfee_r = '0.00002'       # [fraction, any str between 0-1] / relative offer fee you wish to receive based on a cj's amount
-cjfee_factor = 0.1        # [fraction, 0-1] / variance around the average fee. Ex: 200 fee, 0.2 var = fee is btw 160-240
-txfee = 0                 # [satoshis, any integer] / the average transaction fee contribution you're adding to coinjoin transactions
-txfee_factor = 0.3        # [fraction, 0-1] / variance around the average fee contribution. Ex: 1000 fee, 0.2 var = fee is btw 800-1200
-minsize = 100000          # [satoshis, any integer] / minimum size of your cj offer. Lower cj amounts will be disregarded
-size_factor = 0.1         # [fraction, 0-1] / variance around all offer sizes. Ex: 500k minsize, 0.1 var = 450k-550k
-gaplimit = 6
-
-# end of settings customization
 
 jlog = get_log()
 
@@ -53,21 +41,21 @@ class YieldGeneratorPrivacyEnhanced(YieldGeneratorBasic):
         max_mix = max(mix_balance, key=mix_balance.get)
 
         # randomizing the different values
-        randomize_txfee = int(random.uniform(txfee * (1 - float(txfee_factor)),
+        randomize_txfee = int(random.uniform(self.txfee * (1 - float(self.txfee_factor)),
-                                             txfee * (1 + float(txfee_factor))))
+                                             self.txfee * (1 + float(self.txfee_factor))))
-        randomize_minsize = int(random.uniform(self.minsize * (1 - float(size_factor)),
+        randomize_minsize = int(random.uniform(self.minsize * (1 - float(self.size_factor)),
-                                               self.minsize * (1 + float(size_factor))))
+                                               self.minsize * (1 + float(self.size_factor))))
         possible_maxsize = mix_balance[max_mix] - max(jm_single().DUST_THRESHOLD, randomize_txfee)
-        randomize_maxsize = int(random.uniform(possible_maxsize * (1 - float(size_factor)),
+        randomize_maxsize = int(random.uniform(possible_maxsize * (1 - float(self.size_factor)),
                                                possible_maxsize))
 
         if self.ordertype in ['swabsoffer', 'sw0absoffer']:
-            randomize_cjfee = int(random.uniform(float(cjfee_a) * (1 - float(cjfee_factor)),
+            randomize_cjfee = int(random.uniform(float(self.cjfee_a) * (1 - float(self.cjfee_factor)),
-                                                 float(cjfee_a) * (1 + float(cjfee_factor))))
+                                                 float(self.cjfee_a) * (1 + float(self.cjfee_factor))))
             randomize_cjfee = randomize_cjfee + randomize_txfee
         else:
-            randomize_cjfee = random.uniform(float(f) * (1 - float(cjfee_factor)),
+            randomize_cjfee = random.uniform(float(f) * (1 - float(self.cjfee_factor)),
-                                             float(f) * (1 + float(cjfee_factor)))
+                                             float(f) * (1 + float(self.cjfee_factor)))
             randomize_cjfee = "{0:.6f}".format(randomize_cjfee)  # round to 6 decimals
 
         order = {'oid': 0,
@@ -90,8 +78,5 @@ class YieldGeneratorPrivacyEnhanced(YieldGeneratorBasic):
 
 
 if __name__ == "__main__":
-    ygmain(YieldGeneratorPrivacyEnhanced, txfee=txfee, cjfee_a=cjfee_a,
+    ygmain(YieldGeneratorPrivacyEnhanced, nickserv_password='')
-           cjfee_r=cjfee_r, ordertype=ordertype,
-           nickserv_password='',
-           minsize=minsize, gaplimit=gaplimit)
     jmprint('done', "success")

scripts/yield-generator-basic.py

@@ -3,20 +3,9 @@
 from jmbase import jmprint
 from jmclient import YieldGeneratorBasic, ygmain
 
-"""THESE SETTINGS CAN SIMPLY BE EDITED BY HAND IN THIS FILE:
+# YIELD GENERATOR SETTINGS ARE NOW IN YOUR joinmarket.cfg CONFIG FILE
-"""
+# (You can also use command line flags; see --help for this script).
-
-ordertype = 'reloffer' # [string, 'reloffer' or 'absoffer'], which fee type to actually use
-cjfee_a = 500 # [satoshis, any integer] / absolute offer fee you wish to receive for coinjoins (cj)
-cjfee_r = '0.00002' # [fraction, any str between 0-1] / relative offer fee you wish to receive based on a cj's amount
-txfee = 0 # [satoshis, any integer] / the transaction fee contribution you're adding to coinjoin transactions
-nickserv_password = ''
-minsize = 100000 # [satoshis, any integer] / minimum size of your cj offer. Lower cj amounts will be disregarded
-gaplimit = 6
 
 if __name__ == "__main__":
-    ygmain(YieldGeneratorBasic, txfee=txfee, cjfee_a=cjfee_a,
+    ygmain(YieldGeneratorBasic, nickserv_password='')
-           cjfee_r=cjfee_r, ordertype=ordertype,
-           nickserv_password=nickserv_password,
-           minsize=minsize, gaplimit=gaplimit)
     jmprint('done', "success")

test/ygrunner.py

@@ -117,12 +117,43 @@ def test_start_ygs(setup_ygrunner, num_ygs, wallet_structures, mean_amt,
         jmprint("Maker seed: " + wallet_services[i]['seed'])
     jmprint("\n")
     wallet_service.sync_wallet(fast=True)
-    txfee = 1000
-    cjfee_a = 4200
-    cjfee_r = '0.001'
-    ordertype = 'sw0reloffer'
-    minsize = 100000
     ygclass = YieldGeneratorBasic
+
+    # As per previous note, override non-default command line settings:
+    options = {}
+    for x in ["ordertype", "txfee", "txfee_factor", "cjfee_a", "cjfee_r",
+              "cjfee_factor", "minsize", "size_factor"]:
+        options[x] = jm_single().config.get("YIELDGENERATOR", x)
+    ordertype = options["ordertype"]
+    txfee = int(options["txfee"])
+    txfee_factor = float(options["txfee_factor"])
+    cjfee_factor = float(options["cjfee_factor"])
+    size_factor = float(options["size_factor"])
+    if ordertype == 'reloffer':
+        cjfee_r = options["cjfee_r"]
+        # minimum size is such that you always net profit at least 20%
+        #of the miner fee
+        minsize = max(int(1.2 * txfee / float(cjfee_r)), int(options["minsize"]))
+        cjfee_a = None
+    elif ordertype == 'absoffer':
+        cjfee_a = int(options["cjfee_a"])
+        minsize = int(options["minsize"])
+        cjfee_r = None
+    else:
+        assert False, "incorrect offertype config for yieldgenerator."
+
+    txtype = wallet_service.get_txtype()
+    if txtype == "p2wpkh":
+        prefix = "sw0"
+    elif txtype == "p2sh-p2wpkh":
+        prefix = "sw"
+    elif txtype == "p2pkh":
+        prefix = ""
+    else:
+        assert False, "Unsupported wallet type for yieldgenerator: " + txtype
+
+    ordertype = prefix + ordertype
+
     if malicious:
         if deterministic:
             ygclass = DeterministicMaliciousYieldGenerator
@@ -130,7 +161,8 @@ def test_start_ygs(setup_ygrunner, num_ygs, wallet_structures, mean_amt,
             ygclass = MaliciousYieldGenerator
     for i in range(num_ygs):
         
-        cfg = [txfee, cjfee_a, cjfee_r, ordertype, minsize]
+        cfg = [txfee, cjfee_a, cjfee_r, ordertype, minsize, txfee_factor,
+               cjfee_factor, size_factor]
         wallet_service_yg = wallet_services[i]["wallet"]
         wallet_service_yg.startService()
         yg = ygclass(wallet_service_yg, cfg)