diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index 3132a39..dc87b61 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -9,7 +9,7 @@ import logging from btc import * from .support import (calc_cj_fee, choose_sweep_orders, choose_orders, - pick_order, cheapest_order_choose, weighted_order_choose, + cheapest_order_choose, weighted_order_choose, rand_norm_array, rand_pow_array, rand_exp_array, select, select_gradual, select_greedy, select_greediest) from .jsonrpc import JsonRpcError, JsonRpcConnectionError, JsonRpc diff --git a/jmclient/jmclient/support.py b/jmclient/jmclient/support.py index 76f6180..5666b65 100644 --- a/jmclient/jmclient/support.py +++ b/jmclient/jmclient/support.py @@ -109,6 +109,7 @@ def select_greedy(unspent, value): UTXO selection algorithm for greedy dust reduction, but leaves out extraneous utxos, preferring to keep multiple small ones. """ + original_value = value value, key, cursor = int(value), lambda u: u['value'], 0 utxos, picked = sorted(unspent, key=key), [] for utxo in utxos: # find the smallest consecutive sum >= value @@ -119,13 +120,15 @@ def select_greedy(unspent, value): picked += [utxo] # definitely need this utxo break # proceed to dilution cursor += 1 - for utxo in utxos[cursor - 1::-1]: # dilution loop + for utxo in utxos[max(cursor-1, 0)::-1]: # dilution loop value += key(utxo) # see if we can skip this one if value > 0: # no, that drops us below the target picked += [utxo] # so we need this one too value -= key(utxo) # 'backtrack' the counter if len(picked) > 0: - return picked + if len(picked) < len(utxos) or sum( + key(u) for u in picked) >= original_value: + return picked raise Exception('Not enough funds') # if all else fails, we do too @@ -202,29 +205,8 @@ def cheapest_order_choose(orders, n): return orders[0] -def pick_order(orders, n): - print("Considered orders:") - for i, o in enumerate(orders): - print(" %2d. %20s, CJ fee: %6s, tx fee: %6d" % - (i, o[0]['counterparty'], str(o[0]['cjfee']), o[0]['txfee'])) - pickedOrderIndex = -1 - if i == 0: - print("Only one possible pick, picking it.") - return orders[0] - while pickedOrderIndex == -1: - try: - pickedOrderIndex = int(raw_input('Pick an order between 0 and ' + - str(i) + ': ')) - except ValueError: - pickedOrderIndex = -1 - continue - - if 0 <= pickedOrderIndex < len(orders): - return orders[pickedOrderIndex] - pickedOrderIndex = -1 - - -def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None): +def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None, + pick=False): if ignored_makers is None: ignored_makers = [] #Filter ignored makers and inappropriate amounts @@ -248,7 +230,7 @@ def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None): all of them. however, if orders are picked manually, allow duplicates. """ feekey = lambda x: x[1] - if chooseOrdersBy != pick_order: + if not pick: orders_fees = sorted( dict((v[0]['counterparty'], v) for v in sorted(orders_fees, @@ -256,8 +238,7 @@ def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None): reverse=True)).values(), key=feekey) else: - orders_fees = sorted(orders_fees, key=feekey) #sort by ascending cjfee - + orders_fees = sorted(orders_fees, key=feekey) #pragma: no cover log.debug('considered orders = \n' + '\n'.join([str(o) for o in orders_fees ])) total_cj_fee = 0 @@ -305,7 +286,8 @@ def choose_sweep_orders(offers, sumabsfee += int(order['cjfee']) elif order['ordertype'] == 'reloffer': sumrelfee += Decimal(order['cjfee']) - else: + #this is unreachable since calc_cj_fee must already have been called + else: #pragma: no cover raise RuntimeError('unknown order type: {}'.format(order[ 'ordertype'])) diff --git a/jmclient/test/test_blockr.py b/jmclient/test/test_blockr.py new file mode 100644 index 0000000..a56d4fa --- /dev/null +++ b/jmclient/test/test_blockr.py @@ -0,0 +1,133 @@ +#! /usr/bin/env python +from __future__ import absolute_import +'''Blockchain access via blockr tests.''' + +import sys +import os +import time +import binascii +from mock import patch +import json + +import jmbitcoin as btc +import pytest +from jmclient import (load_program_config, jm_single, sync_wallet, BlockrInterface, + get_p2pk_vbyte, get_log, Wallet) +log = get_log() + +#TODO: some kind of mainnet testing, harder. +blockr_root_url = "https://tbtc.blockr.io/api/v1/" + +def test_blockr_bad_request(): + with pytest.raises(Exception) as e_info: + btc.make_request_blockr(blockr_root_url+"address/txs/", "0000") + +def test_blockr_bad_pushtx(): + inps = [("00000000", "btc"), ("00000000", "testnet"), + ('\x00'*8, "testnet"), ('\x00'*8, "x")] + for i in inps: + with pytest.raises(Exception) as e_info: + btc.blockr_pushtx(i[0],i[1]) + +def test_bci_bad_pushtx(): + inps = [("00000000"), ('\x00'*8)] + for i in inps: + with pytest.raises(Exception) as e_info: + btc.bci_pushtx(i[0]) + +def test_blockr_estimate_fee(setup_blockr): + res = [] + for N in [1,3,6]: + res.append(jm_single().bc_interface.estimate_fee_per_kb(N)) + assert res[0] >= res[2] + #Note this can fail, it isn't very accurate. + #assert res[1] >= res[2] + #sanity checks: + assert res[0] < 200000 + assert res[2] < 150000 + +@pytest.mark.parametrize( + "net, seed, gaplimit, showprivkey, method", + [ + ("testnet", + #Dont take these testnet coins, itll botch up our tests!! + "I think i did pretty good with Christmas", + 6, + True, + #option "displayall" here will show all addresses from beginning + "display"), + ]) +def test_blockr_sync(setup_blockr, net, seed, gaplimit, showprivkey, method): + jm_single().config.set("BLOCKCHAIN", "network", net) + wallet = Wallet(seed, max_mix_depth = 5) + sync_wallet(wallet) + + #copy pasted from wallet-tool; some boiled down form of + #this should really be in wallet.py in the joinmarket module. + def cus_print(s): + print s + + total_balance = 0 + for m in range(wallet.max_mix_depth): + cus_print('mixing depth %d m/0/%d/' % (m, m)) + balance_depth = 0 + for forchange in [0, 1]: + cus_print(' ' + ('external' if forchange == 0 else 'internal') + + ' addresses m/0/%d/%d/' % (m, forchange)) + + for k in range(wallet.index[m][forchange] + gaplimit): + addr = wallet.get_addr(m, forchange, k) + balance = 0.0 + for addrvalue in wallet.unspent.values(): + if addr == addrvalue['address']: + balance += addrvalue['value'] + balance_depth += balance + used = ('used' if k < wallet.index[m][forchange] else ' new') + if showprivkey: + privkey = btc.wif_compressed_privkey( + wallet.get_key(m, forchange, k), get_p2pk_vbyte()) + else: + privkey = '' + if (method == 'displayall' or balance > 0 or + (used == ' new' and forchange == 0)): + cus_print(' m/0/%d/%d/%03d %-35s%s %.8f btc %s' % + (m, forchange, k, addr, used, balance / 1e8, + privkey)) + total_balance += balance_depth + print('for mixdepth=%d balance=%.8fbtc' % (m, balance_depth / 1e8)) + assert total_balance == 96085297 + +@patch('jmbitcoin.bci.make_request') +def test_blockr_error_429(make_request): + error = {u'code': 429, + u'data': None, + u'message': u'Too many requests. Wait a bit...', + u'status': u'error'} + success = {u'code': 200, + u'data': {u'address': u'mqG1k82TDWfxSYFyDRkomjYonDUYjPRbsb', + u'limit_txs': 200, + u'nb_txs': 1, + u'nb_txs_displayed': 1, + u'txs': [{u'amount': 1, + u'amount_multisig': 0, + u'confirmations': 400, + u'time_utc': u'2016-09-15T19:46:14Z', + u'tx': u'6a1bfbdd011cbb2ab2a000d477bd6372150238b4c24e43a850220dba4dbf2c0d'}]}, + u'message': u'', + u'status': u'success'} + make_request.side_effect = map(json.dumps, [error]*3 + [success]) + + d = btc.make_request_blockr(blockr_root_url + "address/txs/", "mqG1k82TDWfxSYFyDRkomjYonDUYjPRbsb") + assert d['code'] == 200 + assert d['data'] is not None + + +@pytest.fixture(scope="module") +def setup_blockr(request): + def blockr_teardown(): + jm_single().config.set("BLOCKCHAIN", "blockchain_source", "regtest") + jm_single().config.set("BLOCKCHAIN", "network", "testnet") + request.addfinalizer(blockr_teardown) + load_program_config() + jm_single().config.set("BLOCKCHAIN", "blockchain_source", "blockr") + jm_single().bc_interface = BlockrInterface(True) diff --git a/jmclient/test/test_support.py b/jmclient/test/test_support.py new file mode 100644 index 0000000..a82ecf1 --- /dev/null +++ b/jmclient/test/test_support.py @@ -0,0 +1,137 @@ +#! /usr/bin/env python +from __future__ import absolute_import +'''support functions for jmclient tests.''' + +import pytest +from jmclient import (select, select_gradual, select_greedy, select_greediest, + choose_orders, choose_sweep_orders, weighted_order_choose) +from jmclient.support import (calc_cj_fee, rand_exp_array, rand_pow_array, + rand_norm_array, rand_weighted_choice, + cheapest_order_choose) +from taker_test_data import t_orderbook +import copy + +def test_utxo_selection(): + """Check that all the utxo selection algorithms work with a random + variety of wallet contents. + """ + unspent = [{'utxo':'a', 'value': 10000000}, + {'utxo':'b', 'value': 20000000}, + {'utxo':'c', 'value': 50000000}, + {'utxo':'d', 'value': 50000000}] + for selector in [select, select_gradual, select_greedy, select_greediest]: + for amt in [9999999, 10000000, 110000000, 19999999, 20000000, + 49999999, 50000000, 99999999, 100000000]: + selector(unspent, amt) + for amt in [1300000010, 2000000000]: + with pytest.raises(Exception) as e_info: + x = selector(unspent, amt) + print(x) + assert e_info.match("Not enough funds") + +def test_random_funcs(): + x1 = rand_norm_array(5, 2, 10) + assert len(x1) == 10 + for x in x1: + assert x > -7 #6 sigma! + x2 = rand_exp_array(100, 10) + assert len(x2) == 10 + for x in x2: + assert x > 0 + x3 = rand_pow_array(100, 10) + assert len(x3) == 10 + for x in x3: + assert x > 0 + assert x < 1 + x4 = rand_weighted_choice(5, [0.2, 0.1, 0.3, 0.15, 0.25]) + assert x4 in range(5) + #test weighted choice fails with invalid inputs + with pytest.raises(ValueError) as e_info: + x = rand_weighted_choice(5, [0.2, 0.1, 0.3, 0.15, 0.26]) + assert e_info.match("Sum of probabilities") + with pytest.raises(ValueError) as e_info: + x = rand_weighted_choice(5, [0.25, 0.25, 0.25, 0.25]) + assert e_info.match("Need: 5 probabilities.") + +def test_calc_cjfee(): + assert calc_cj_fee("absoffer", 3000, 200000000) == 3000 + assert calc_cj_fee("reloffer", "0.01", 100000000) == 1000000 + with pytest.raises(RuntimeError) as e_info: + calc_cj_fee("dummyoffer", 2, 3) + +def test_choose_orders(): + orderbook = copy.deepcopy(t_orderbook) + #test not enough liquidity + orders_fees = choose_orders(orderbook, 10000000, 7, weighted_order_choose) + assert orders_fees == (None, 0) + orders_fees = choose_orders(orderbook, 10000000, 3, weighted_order_choose) + #need variable fee sizes + for i, o in enumerate(orderbook): + o['cjfee'] = str(float(o['cjfee']) + 0.0001*i) + #test phi not zero + orders_fees = choose_orders(orderbook, 10000000, 3, weighted_order_choose) + assert len(orders_fees[0]) == 3 + #test M < orderbook size for weighted + orders_fees = choose_orders(orderbook, 10000000, 1, weighted_order_choose) + assert len(orders_fees[0]) == 1 + #test the hated 'cheapest' + orders_fees = choose_orders(orderbook, 100000000, 3, cheapest_order_choose) + assert len(orders_fees[0]) == 3 + #test sweep + result, cjamount, total_fee = choose_sweep_orders(orderbook, 50000000, + 30000, + 3, + weighted_order_choose, + None) + assert cjamount >= 49800000 + assert cjamount <= 50000000 + assert total_fee >= 30000 + assert total_fee <= 100000 + assert len(result) == 3 + #test not enough liquidity + result, cjamount, total_fee = choose_sweep_orders(orderbook, 50000000, + 30000, 7, + weighted_order_choose, + None) + assert result == None + assert cjamount == 0 + assert total_fee == 0 + + #here we doctor the orderbook; (a) include an absfee + #(b) add an unrecognized ordertype + #(c) put an order with wrong minsize + orderbook.append({u'counterparty': u'fake', + u'ordertype': u'absoffer', u'oid': 0, + u'minsize': 7500000, u'txfee': 1000, + u'maxsize': 599972700, u'cjfee': 9000}) + result, cjamount, total_fee = choose_sweep_orders(orderbook, 50000000, + 30000, 7, + cheapest_order_choose, + None) + assert total_fee > 0 + #(b) + orderbook.append({u'counterparty': u'fake2', + u'ordertype': u'dummyoffer', u'oid': 0, + u'minsize': 7500000, u'txfee': 1000, + u'maxsize': 599972700, u'cjfee': 9000}) + with pytest.raises(RuntimeError) as e_info: + result, cjamount, total_fee = choose_sweep_orders(orderbook, + 50000000, + 30000, + 8, + weighted_order_choose, + None) + #(c) + #remove bad offer + orderbook = orderbook[:-1] + for i in range(7): + orderbook[i]['minsize'] = 49999999 + result, cjamount, total_fee = choose_sweep_orders(orderbook, + 50000000, + 30000, + 4, + weighted_order_choose, + None) + assert result == None + assert cjamount == 0 + assert total_fee == 0 \ No newline at end of file diff --git a/scripts/sendpayment.py b/scripts/sendpayment.py index 5f16f94..50cd689 100644 --- a/scripts/sendpayment.py +++ b/scripts/sendpayment.py @@ -52,7 +52,7 @@ import time from jmclient import (Taker, load_program_config, get_schedule, JMTakerClientProtocolFactory, start_reactor, validate_address, jm_single, - choose_orders, choose_sweep_orders, pick_order, + choose_orders, choose_sweep_orders, cheapest_order_choose, weighted_order_choose, Wallet, BitcoinCoreWallet, sync_wallet, RegtestBitcoinCoreInterface, estimate_tx_fee) @@ -61,6 +61,27 @@ from jmbase.support import get_log, debug_dump_object log = get_log() +def pick_order(orders, n): #pragma: no cover + print("Considered orders:") + for i, o in enumerate(orders): + print(" %2d. %20s, CJ fee: %6s, tx fee: %6d" % + (i, o[0]['counterparty'], str(o[0]['cjfee']), o[0]['txfee'])) + pickedOrderIndex = -1 + if i == 0: + print("Only one possible pick, picking it.") + return orders[0] + while pickedOrderIndex == -1: + try: + pickedOrderIndex = int(raw_input('Pick an order between 0 and ' + + str(i) + ': ')) + except ValueError: + pickedOrderIndex = -1 + continue + + if 0 <= pickedOrderIndex < len(orders): + return orders[pickedOrderIndex] + pickedOrderIndex = -1 + def main(): parser = OptionParser( usage=