Test coverage of support.py in jmclient

Also added some blockr tests (not complete). Minor packaging
changes, in particular pick_orders removed from package as
requires user intervention, moved to sendpayment.py for now.

jmclient/jmclient/__init__.py

@@ -9,7 +9,7 @@ import logging
 from btc import *
 
 from .support import (calc_cj_fee, choose_sweep_orders, choose_orders,
-                      pick_order, cheapest_order_choose, weighted_order_choose,
+                      cheapest_order_choose, weighted_order_choose,
                       rand_norm_array, rand_pow_array, rand_exp_array, select,
                       select_gradual, select_greedy, select_greediest)
 from .jsonrpc import JsonRpcError, JsonRpcConnectionError, JsonRpc

jmclient/jmclient/support.py

@@ -109,6 +109,7 @@ def select_greedy(unspent, value):
     UTXO selection algorithm for greedy dust reduction, but leaves out
     extraneous utxos, preferring to keep multiple small ones.
     """
+    original_value = value
     value, key, cursor = int(value), lambda u: u['value'], 0
     utxos, picked = sorted(unspent, key=key), []
     for utxo in utxos:  # find the smallest consecutive sum >= value
@@ -119,13 +120,15 @@ def select_greedy(unspent, value):
             picked += [utxo]  # definitely need this utxo
             break  # proceed to dilution
         cursor += 1
-    for utxo in utxos[cursor - 1::-1]:  # dilution loop
+    for utxo in utxos[max(cursor-1, 0)::-1]:  # dilution loop
         value += key(utxo)  # see if we can skip this one
         if value > 0:  # no, that drops us below the target
             picked += [utxo]  # so we need this one too
             value -= key(utxo)  # 'backtrack' the counter
     if len(picked) > 0:
-        return picked
+        if len(picked) < len(utxos) or sum(
+            key(u) for u in picked) >= original_value:
+            return picked
     raise Exception('Not enough funds')  # if all else fails, we do too
 
 
@@ -202,29 +205,8 @@ def cheapest_order_choose(orders, n):
     return orders[0]
 
 
-def pick_order(orders, n):
+def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None,
-    print("Considered orders:")
+                  pick=False):
-    for i, o in enumerate(orders):
-        print("    %2d. %20s, CJ fee: %6s, tx fee: %6d" %
-              (i, o[0]['counterparty'], str(o[0]['cjfee']), o[0]['txfee']))
-    pickedOrderIndex = -1
-    if i == 0:
-        print("Only one possible pick, picking it.")
-        return orders[0]
-    while pickedOrderIndex == -1:
-        try:
-            pickedOrderIndex = int(raw_input('Pick an order between 0 and ' +
-                                             str(i) + ': '))
-        except ValueError:
-            pickedOrderIndex = -1
-            continue
-
-        if 0 <= pickedOrderIndex < len(orders):
-            return orders[pickedOrderIndex]
-        pickedOrderIndex = -1
-
-
-def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None):
     if ignored_makers is None:
         ignored_makers = []
     #Filter ignored makers and inappropriate amounts
@@ -248,7 +230,7 @@ def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None):
     all of them. however, if orders are picked manually, allow duplicates.
     """
     feekey = lambda x: x[1]
-    if chooseOrdersBy != pick_order:
+    if not pick:
         orders_fees = sorted(
             dict((v[0]['counterparty'], v)
                  for v in sorted(orders_fees,
@@ -256,8 +238,7 @@ def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None):
                                  reverse=True)).values(),
             key=feekey)
     else:
-        orders_fees = sorted(orders_fees, key=feekey)  #sort by ascending cjfee
+        orders_fees = sorted(orders_fees, key=feekey)  #pragma: no cover
-
     log.debug('considered orders = \n' + '\n'.join([str(o) for o in orders_fees
                                                    ]))
     total_cj_fee = 0
@@ -305,7 +286,8 @@ def choose_sweep_orders(offers,
                 sumabsfee += int(order['cjfee'])
             elif order['ordertype'] == 'reloffer':
                 sumrelfee += Decimal(order['cjfee'])
-            else:
+            #this is unreachable since calc_cj_fee must already have been called
+            else: #pragma: no cover
                 raise RuntimeError('unknown order type: {}'.format(order[
                     'ordertype']))
 

jmclient/test/test_blockr.py

@@ -0,0 +1,133 @@
+#! /usr/bin/env python
+from __future__ import absolute_import
+'''Blockchain access via blockr tests.'''
+
+import sys
+import os
+import time
+import binascii
+from mock import patch
+import json
+
+import jmbitcoin as btc
+import pytest
+from jmclient import (load_program_config, jm_single, sync_wallet, BlockrInterface,
+                      get_p2pk_vbyte, get_log, Wallet)
+log = get_log()
+
+#TODO: some kind of mainnet testing, harder.
+blockr_root_url = "https://tbtc.blockr.io/api/v1/"
+
+def test_blockr_bad_request():
+    with pytest.raises(Exception) as e_info:
+        btc.make_request_blockr(blockr_root_url+"address/txs/", "0000")
+
+def test_blockr_bad_pushtx():
+    inps = [("00000000", "btc"), ("00000000", "testnet"),
+            ('\x00'*8, "testnet"), ('\x00'*8, "x")]
+    for i in inps:
+        with pytest.raises(Exception) as e_info:
+            btc.blockr_pushtx(i[0],i[1])
+
+def test_bci_bad_pushtx():
+    inps = [("00000000"), ('\x00'*8)]
+    for i in inps:
+        with pytest.raises(Exception) as e_info:
+            btc.bci_pushtx(i[0])
+
+def test_blockr_estimate_fee(setup_blockr):
+    res = []
+    for N in [1,3,6]:
+        res.append(jm_single().bc_interface.estimate_fee_per_kb(N))
+    assert res[0] >= res[2]
+    #Note this can fail, it isn't very accurate.
+    #assert res[1] >= res[2]
+    #sanity checks:
+    assert res[0] < 200000
+    assert res[2] < 150000
+
+@pytest.mark.parametrize(
+    "net, seed, gaplimit, showprivkey, method",
+    [
+        ("testnet",
+         #Dont take these testnet coins, itll botch up our tests!!
+         "I think i did pretty good with Christmas",
+         6,
+         True,
+         #option "displayall" here will show all addresses from beginning
+         "display"),
+    ])
+def test_blockr_sync(setup_blockr, net, seed, gaplimit, showprivkey, method):
+    jm_single().config.set("BLOCKCHAIN", "network", net)
+    wallet = Wallet(seed, max_mix_depth = 5)
+    sync_wallet(wallet)
+
+    #copy pasted from wallet-tool; some boiled down form of
+    #this should really be in wallet.py in the joinmarket module.
+    def cus_print(s):
+            print s
+
+    total_balance = 0
+    for m in range(wallet.max_mix_depth):
+        cus_print('mixing depth %d m/0/%d/' % (m, m))
+        balance_depth = 0
+        for forchange in [0, 1]:
+            cus_print(' ' + ('external' if forchange == 0 else 'internal') +
+                      ' addresses m/0/%d/%d/' % (m, forchange))
+
+            for k in range(wallet.index[m][forchange] + gaplimit):
+                addr = wallet.get_addr(m, forchange, k)
+                balance = 0.0
+                for addrvalue in wallet.unspent.values():
+                    if addr == addrvalue['address']:
+                        balance += addrvalue['value']
+                balance_depth += balance
+                used = ('used' if k < wallet.index[m][forchange] else ' new')
+                if showprivkey:
+                    privkey = btc.wif_compressed_privkey(
+                    wallet.get_key(m, forchange, k), get_p2pk_vbyte())
+                else:
+                    privkey = ''
+                if (method == 'displayall' or balance > 0 or
+                    (used == ' new' and forchange == 0)):
+                    cus_print('  m/0/%d/%d/%03d %-35s%s %.8f btc %s' %
+                              (m, forchange, k, addr, used, balance / 1e8,
+                               privkey))
+        total_balance += balance_depth
+        print('for mixdepth=%d balance=%.8fbtc' % (m, balance_depth / 1e8))
+    assert total_balance == 96085297
+
+@patch('jmbitcoin.bci.make_request')
+def test_blockr_error_429(make_request):
+    error = {u'code': 429,
+             u'data': None,
+             u'message': u'Too many requests. Wait a bit...',
+             u'status': u'error'}
+    success = {u'code': 200,
+               u'data': {u'address': u'mqG1k82TDWfxSYFyDRkomjYonDUYjPRbsb',
+                         u'limit_txs': 200,
+                         u'nb_txs': 1,
+                         u'nb_txs_displayed': 1,
+                         u'txs': [{u'amount': 1,
+                                   u'amount_multisig': 0,
+                                   u'confirmations': 400,
+                                   u'time_utc': u'2016-09-15T19:46:14Z',
+                                   u'tx': u'6a1bfbdd011cbb2ab2a000d477bd6372150238b4c24e43a850220dba4dbf2c0d'}]},
+               u'message': u'',
+               u'status': u'success'}
+    make_request.side_effect = map(json.dumps, [error]*3 + [success])
+
+    d = btc.make_request_blockr(blockr_root_url + "address/txs/", "mqG1k82TDWfxSYFyDRkomjYonDUYjPRbsb")
+    assert d['code'] == 200
+    assert d['data'] is not None
+
+
+@pytest.fixture(scope="module")
+def setup_blockr(request):
+    def blockr_teardown():
+        jm_single().config.set("BLOCKCHAIN", "blockchain_source", "regtest")
+        jm_single().config.set("BLOCKCHAIN", "network", "testnet")
+    request.addfinalizer(blockr_teardown)    
+    load_program_config()
+    jm_single().config.set("BLOCKCHAIN", "blockchain_source", "blockr")
+    jm_single().bc_interface = BlockrInterface(True)

jmclient/test/test_support.py

@@ -0,0 +1,137 @@
+#! /usr/bin/env python
+from __future__ import absolute_import
+'''support functions for jmclient tests.'''
+
+import pytest
+from jmclient import (select, select_gradual, select_greedy, select_greediest,
+                      choose_orders, choose_sweep_orders, weighted_order_choose)
+from jmclient.support import (calc_cj_fee, rand_exp_array, rand_pow_array,
+                              rand_norm_array, rand_weighted_choice,
+                              cheapest_order_choose)
+from taker_test_data import t_orderbook
+import copy
+
+def test_utxo_selection():
+    """Check that all the utxo selection algorithms work with a random
+    variety of wallet contents.
+    """
+    unspent = [{'utxo':'a', 'value': 10000000},
+               {'utxo':'b', 'value': 20000000},
+               {'utxo':'c', 'value': 50000000},
+               {'utxo':'d', 'value': 50000000}]
+    for selector in [select, select_gradual, select_greedy, select_greediest]:
+        for amt in [9999999, 10000000, 110000000, 19999999, 20000000,
+                49999999, 50000000, 99999999, 100000000]:
+            selector(unspent, amt)
+        for amt in [1300000010, 2000000000]:
+            with pytest.raises(Exception) as e_info:
+                x = selector(unspent, amt)
+                print(x)
+            assert e_info.match("Not enough funds")
+
+def test_random_funcs():
+    x1 = rand_norm_array(5, 2, 10)
+    assert len(x1) == 10
+    for x in x1:
+        assert x > -7 #6 sigma!
+    x2 = rand_exp_array(100, 10)
+    assert len(x2) == 10
+    for x in x2:
+        assert x > 0
+    x3 = rand_pow_array(100, 10)
+    assert len(x3) == 10
+    for x in x3:
+        assert x > 0
+        assert x < 1
+    x4 = rand_weighted_choice(5, [0.2, 0.1, 0.3, 0.15, 0.25])
+    assert x4 in range(5)
+    #test weighted choice fails with invalid inputs
+    with pytest.raises(ValueError) as e_info:
+        x = rand_weighted_choice(5, [0.2, 0.1, 0.3, 0.15, 0.26])
+    assert e_info.match("Sum of probabilities")
+    with pytest.raises(ValueError) as e_info:
+        x = rand_weighted_choice(5, [0.25, 0.25, 0.25, 0.25])
+    assert e_info.match("Need: 5 probabilities.")
+
+def test_calc_cjfee():
+    assert calc_cj_fee("absoffer", 3000, 200000000) == 3000
+    assert calc_cj_fee("reloffer", "0.01", 100000000) == 1000000
+    with pytest.raises(RuntimeError) as e_info:
+        calc_cj_fee("dummyoffer", 2, 3)
+
+def test_choose_orders():
+    orderbook = copy.deepcopy(t_orderbook)
+    #test not enough liquidity
+    orders_fees = choose_orders(orderbook, 10000000, 7, weighted_order_choose)
+    assert orders_fees == (None, 0)
+    orders_fees = choose_orders(orderbook, 10000000, 3, weighted_order_choose)
+    #need variable fee sizes
+    for i, o in enumerate(orderbook):
+        o['cjfee'] = str(float(o['cjfee']) + 0.0001*i)
+    #test phi not zero
+    orders_fees = choose_orders(orderbook, 10000000, 3, weighted_order_choose)
+    assert len(orders_fees[0]) == 3
+    #test M < orderbook size for weighted
+    orders_fees = choose_orders(orderbook, 10000000, 1, weighted_order_choose)
+    assert len(orders_fees[0]) == 1
+    #test the hated 'cheapest'
+    orders_fees = choose_orders(orderbook, 100000000, 3, cheapest_order_choose)
+    assert len(orders_fees[0]) == 3
+    #test sweep
+    result, cjamount, total_fee = choose_sweep_orders(orderbook, 50000000,
+                                                      30000,
+                                                      3,
+                                                      weighted_order_choose,
+                                                      None)
+    assert cjamount >= 49800000
+    assert cjamount <= 50000000
+    assert total_fee >= 30000
+    assert total_fee <= 100000
+    assert len(result) == 3
+    #test not enough liquidity
+    result, cjamount, total_fee = choose_sweep_orders(orderbook, 50000000,
+                                                      30000, 7,
+                                                      weighted_order_choose,
+                                                      None)
+    assert result == None
+    assert cjamount == 0
+    assert total_fee == 0
+    
+    #here we doctor the orderbook; (a) include an absfee
+    #(b) add an unrecognized ordertype
+    #(c) put an order with wrong minsize
+    orderbook.append({u'counterparty': u'fake',
+                      u'ordertype': u'absoffer', u'oid': 0,
+                      u'minsize': 7500000, u'txfee': 1000,
+                      u'maxsize': 599972700, u'cjfee': 9000})
+    result, cjamount, total_fee = choose_sweep_orders(orderbook, 50000000,
+                                                30000, 7,
+                                                cheapest_order_choose,
+                                                None)
+    assert total_fee > 0
+    #(b)
+    orderbook.append({u'counterparty': u'fake2',
+                         u'ordertype': u'dummyoffer', u'oid': 0,
+                         u'minsize': 7500000, u'txfee': 1000,
+                         u'maxsize': 599972700, u'cjfee': 9000})
+    with pytest.raises(RuntimeError) as e_info:
+        result, cjamount, total_fee = choose_sweep_orders(orderbook,
+                                                      50000000,
+                                                      30000,
+                                                      8,
+                                                      weighted_order_choose,
+                                                      None)
+    #(c)
+    #remove bad offer
+    orderbook = orderbook[:-1]
+    for i in range(7):
+        orderbook[i]['minsize'] = 49999999
+    result, cjamount, total_fee = choose_sweep_orders(orderbook,
+                                                      50000000,
+                                                      30000,
+                                                      4,
+                                                      weighted_order_choose,
+                                                      None)
+    assert result == None
+    assert cjamount == 0
+    assert total_fee == 0

scripts/sendpayment.py

@@ -52,7 +52,7 @@ import time
 from jmclient import (Taker, load_program_config, get_schedule,
                               JMTakerClientProtocolFactory, start_reactor,
                               validate_address, jm_single,
-                              choose_orders, choose_sweep_orders, pick_order,
+                              choose_orders, choose_sweep_orders,
                               cheapest_order_choose, weighted_order_choose,
                               Wallet, BitcoinCoreWallet, sync_wallet,
                               RegtestBitcoinCoreInterface, estimate_tx_fee)
@@ -61,6 +61,27 @@ from jmbase.support import get_log, debug_dump_object
 
 log = get_log()
 
+def pick_order(orders, n): #pragma: no cover
+    print("Considered orders:")
+    for i, o in enumerate(orders):
+        print("    %2d. %20s, CJ fee: %6s, tx fee: %6d" %
+              (i, o[0]['counterparty'], str(o[0]['cjfee']), o[0]['txfee']))
+    pickedOrderIndex = -1
+    if i == 0:
+        print("Only one possible pick, picking it.")
+        return orders[0]
+    while pickedOrderIndex == -1:
+        try:
+            pickedOrderIndex = int(raw_input('Pick an order between 0 and ' +
+                                             str(i) + ': '))
+        except ValueError:
+            pickedOrderIndex = -1
+            continue
+
+        if 0 <= pickedOrderIndex < len(orders):
+            return orders[pickedOrderIndex]
+        pickedOrderIndex = -1
+
 def main():
     parser = OptionParser(
         usage=