Browse Source

Include ygrunner for multiple yieldgens in test

Update blockchain interface notifier thread to account
for multiple concurrent notification functions from
multiple yieldgens in one process instance.
master
Adam Gibson 9 years ago
parent
commit
252d189423
No known key found for this signature in database
GPG Key ID: B3AE09F1E9A3197A
  1. 78
      jmclient/jmclient/blockchaininterface.py
  2. 5
      jmclient/jmclient/client_protocol.py
  3. 2
      jmclient/jmclient/wallet.py
  4. 3
      jmclient/jmclient/yieldgenerator.py
  5. 109
      test/commontest.py
  6. 66
      test/ygrunner.py

78
jmclient/jmclient/blockchaininterface.py

@ -859,48 +859,50 @@ class NotifyRequestHeader(BaseHTTPServer.BaseHTTPRequestHandler):
txnotify_tuple = None
unconfirmfun, confirmfun, timeoutfun, uc_called = (None, None, None,
None)
to_be_updated = []
to_be_removed = []
for tnf in self.btcinterface.txnotify_fun:
tx_out = tnf[0]
if tx_out == tx_output_set:
txnotify_tuple = tnf
tx_out, unconfirmfun, confirmfun, timeoutfun, uc_called = tnf
break
if unconfirmfun is None:
log.debug('txid=' + txid + ' not being listened for')
else:
# on rare occasions people spend their output without waiting
# for a confirm
txdata = None
for n in range(len(txd['outs'])):
txdata = self.btcinterface.rpc('gettxout', [txid, n, True])
if txdata is not None:
break
assert txdata is not None
if txdata['confirmations'] == 0:
reactor.callFromThread(unconfirmfun, txd, txid)
# TODO pass the total transfered amount value here somehow
# wallet_name = self.get_wallet_name()
# amount =
# bitcoin-cli move wallet_name "" amount
self.btcinterface.txnotify_fun.remove(txnotify_tuple)
self.btcinterface.txnotify_fun.append(txnotify_tuple[:-1] +
(True,))
log.debug('ran unconfirmfun')
if timeoutfun:
threading.Timer(jm_single().config.getfloat(
'TIMEOUT', 'confirm_timeout_hours') * 60 * 60,
bitcoincore_timeout_callback,
args=(True, tx_output_set,
self.btcinterface.txnotify_fun,
timeoutfun)).start()
else:
if not uc_called:
reactor.callFromThread(unconfirmfun, txd, txid)
log.debug('saw confirmed tx before unconfirmed, ' +
'running unconfirmfun first')
reactor.callFromThread(confirmfun, txd, txid, txdata['confirmations'])
self.btcinterface.txnotify_fun.remove(txnotify_tuple)
log.debug('ran confirmfun')
if unconfirmfun is None:
log.debug('txid=' + txid + ' not being listened for')
else:
# on rare occasions people spend their output without waiting
# for a confirm
txdata = None
for n in range(len(txd['outs'])):
txdata = self.btcinterface.rpc('gettxout', [txid, n, True])
if txdata is not None:
break
assert txdata is not None
if txdata['confirmations'] == 0:
reactor.callFromThread(unconfirmfun, txd, txid)
to_be_updated.append(txnotify_tuple)
log.debug('ran unconfirmfun')
if timeoutfun:
threading.Timer(jm_single().config.getfloat(
'TIMEOUT', 'confirm_timeout_hours') * 60 * 60,
bitcoincore_timeout_callback,
args=(True, tx_output_set,
self.btcinterface.txnotify_fun,
timeoutfun)).start()
else:
if not uc_called:
reactor.callFromThread(unconfirmfun, txd, txid)
log.debug('saw confirmed tx before unconfirmed, ' +
'running unconfirmfun first')
reactor.callFromThread(confirmfun, txd, txid, txdata['confirmations'])
to_be_removed.append(txnotify_tuple)
log.debug('ran confirmfun')
#If any notifyfun tuples need to update from unconfirm to confirm state:
for tbu in to_be_updated:
self.btcinterface.txnotify_fun.remove(tbu)
self.btcinterface.txnotify_fun.append(tbu[:-1] + (True,))
#If any notifyfun tuples need to be removed as confirmed
for tbr in to_be_removed:
self.btcinterface.txnotify_fun.remove(tbr)
elif self.path.startswith('/alertnotify?'):
jm_single().core_alert[0] = urllib.unquote(self.path[len(pages[
@ -916,7 +918,7 @@ class NotifyRequestHeader(BaseHTTPServer.BaseHTTPRequestHandler):
request.get_method = lambda: 'HEAD'
try:
urllib2.urlopen(request)
except urllib2.URLError:
except:
pass
self.send_response(200)
# self.send_header('Connection', 'close')

5
jmclient/jmclient/client_protocol.py

@ -435,7 +435,7 @@ class JMClientProtocolFactory(protocol.ClientFactory):
def buildProtocol(self, addr):
return self.protocol(self, self.client)
def start_reactor(host, port, factory, ish=True, daemon=False): #pragma: no cover
def start_reactor(host, port, factory, ish=True, daemon=False, rs=True): #pragma: no cover
#(Cannot start the reactor in tests)
#Not used in prod (twisted logging):
#startLogging(stdout)
@ -473,4 +473,5 @@ def start_reactor(host, port, factory, ish=True, daemon=False): #pragma: no cove
reactor.connectSSL(host, port, factory, ctx)
else:
reactor.connectTCP(host, port, factory)
reactor.run(installSignalHandlers=ish)
if rs:
reactor.run(installSignalHandlers=ish)

2
jmclient/jmclient/wallet.py

@ -345,7 +345,7 @@ class Wallet(AbstractWallet):
removed_utxos[utxo] = self.unspent[utxo]
del self.unspent[utxo]
log.debug('removed utxos, wallet now is \n' + pprint.pformat(
self.get_utxos_by_mixdepth()))
self.get_utxos_by_mixdepth(verbose=False)))
self.spent_utxos += removed_utxos.keys()
return removed_utxos

3
jmclient/jmclient/yieldgenerator.py

@ -78,7 +78,7 @@ class YieldGeneratorBasic(YieldGenerator):
super(YieldGeneratorBasic,self).__init__(wallet)
def create_my_orders(self):
mix_balance = self.wallet.get_balance_by_mixdepth()
mix_balance = self.wallet.get_balance_by_mixdepth(verbose=False)
if len([b for m, b in mix_balance.iteritems() if b > 0]) == 0:
jlog.error('do not have any coins left')
return []
@ -254,6 +254,7 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='swreloffe
options.ordertype, options.minsize])
jlog.info('starting yield generator')
clientfactory = JMClientProtocolFactory(maker, proto_type="MAKER")
nodaemon = jm_single().config.getint("DAEMON", "no_daemon")
daemon = True if nodaemon == 1 else False
start_reactor(jm_single().config.get("DAEMON", "daemon_host"),

109
test/commontest.py

@ -0,0 +1,109 @@
#! /usr/bin/env python
from __future__ import absolute_import
'''Some helper functions for testing'''
import sys
import os
import time
import binascii
import random
from decimal import Decimal
data_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
sys.path.insert(0, os.path.join(data_dir))
from jmclient import SegwitWallet, Wallet, get_log, estimate_tx_fee, jm_single
import jmbitcoin as btc
from jmbase import chunks
log = get_log()
def make_sign_and_push(ins_full,
wallet,
amount,
output_addr=None,
change_addr=None,
hashcode=btc.SIGHASH_ALL,
estimate_fee = False):
"""Utility function for easily building transactions
from wallets
"""
total = sum(x['value'] for x in ins_full.values())
ins = ins_full.keys()
#random output address and change addr
output_addr = wallet.get_new_addr(1, 1) if not output_addr else output_addr
change_addr = wallet.get_new_addr(1, 0) if not change_addr else change_addr
fee_est = estimate_tx_fee(len(ins), 2) if estimate_fee else 10000
outs = [{'value': amount,
'address': output_addr}, {'value': total - amount - fee_est,
'address': change_addr}]
tx = btc.mktx(ins, outs)
de_tx = btc.deserialize(tx)
for index, ins in enumerate(de_tx['ins']):
utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
addr = ins_full[utxo]['address']
priv = wallet.get_key_from_addr(addr)
if index % 2:
priv = binascii.unhexlify(priv)
tx = btc.sign(tx, index, priv, hashcode=hashcode)
#pushtx returns False on any error
print btc.deserialize(tx)
push_succeed = jm_single().bc_interface.pushtx(tx)
if push_succeed:
return btc.txhash(tx)
else:
return False
def make_wallets(n,
wallet_structures=None,
mean_amt=1,
sdev_amt=0,
start_index=0,
fixed_seeds=None,
test_wallet=False,
passwords=None):
'''n: number of wallets to be created
wallet_structure: array of n arrays , each subarray
specifying the number of addresses to be populated with coins
at each depth (for now, this will only populate coins into 'receive' addresses)
mean_amt: the number of coins (in btc units) in each address as above
sdev_amt: if randomness in amouts is desired, specify here.
Returns: a dict of dicts of form {0:{'seed':seed,'wallet':Wallet object},1:..,}
Default Wallet constructor is joinmarket.Wallet, else use TestWallet,
which takes a password parameter as in the list passwords.
'''
if len(wallet_structures) != n:
raise Exception("Number of wallets doesn't match wallet structures")
if not fixed_seeds:
seeds = chunks(binascii.hexlify(os.urandom(15 * n)), 15 * 2)
else:
seeds = fixed_seeds
wallets = {}
for i in range(n):
if test_wallet:
w = TestWallet(seeds[i], max_mix_depth=5, pwd=passwords[i])
else:
w = SegwitWallet(seeds[i], pwd=None, max_mix_depth=5)
wallets[i + start_index] = {'seed': seeds[i],
'wallet': w}
for j in range(5):
for k in range(wallet_structures[i][j]):
deviation = sdev_amt * random.random()
amt = mean_amt - sdev_amt / 2.0 + deviation
if amt < 0: amt = 0.001
amt = float(Decimal(amt).quantize(Decimal(10)**-8))
jm_single().bc_interface.grab_coins(
wallets[i + start_index]['wallet'].get_external_addr(j),
amt)
#reset the index so the coins can be seen if running in same script
wallets[i + start_index]['wallet'].index[j][0] -= wallet_structures[i][j]
return wallets
def interact(process, inputs, expected):
if len(inputs) != len(expected):
raise Exception("Invalid inputs to interact()")
for i, inp in enumerate(inputs):
process.expect(expected[i])
process.sendline(inp)

66
test/ygrunner.py

@ -0,0 +1,66 @@
#! /usr/bin/env python
from __future__ import absolute_import
'''Creates wallets and yield generators in regtest.
Provides seed for joinmarket-qt test.
This should be run via pytest, even though
it's NOT part of the test-suite, because that
makes it much easier to handle start up and
shut down of the environment.
Run it like:
PYTHONPATH=.:$PYTHONPATH py.test \
--btcroot=/path/to/bitcoin/bin/ \
--btcpwd=123456abcdef --btcconf=/blah/bitcoin.conf \
--nirc=2 -s test/ygrunner.py
'''
from commontest import make_wallets
import os
import pytest
import sys
import time
from jmclient import (YieldGeneratorBasic, ygmain, load_program_config,
jm_single, sync_wallet, JMClientProtocolFactory,
start_reactor)
@pytest.mark.parametrize(
"num_ygs, wallet_structures, mean_amt",
[
# 1sp 3yg, 2 mixdepths, sweep from depth1
(2, [[1, 3, 0, 0, 0]] * 3, 2),
])
def test_start_ygs(setup_ygrunner, num_ygs, wallet_structures, mean_amt):
"""Set up some wallets, for the ygs and 1 sp.
Then start the ygs in background and publish
the seed of the sp wallet for easy import into -qt
"""
wallets = make_wallets(num_ygs + 1,
wallet_structures=wallet_structures,
mean_amt=mean_amt)
#the sendpayment bot uses the last wallet in the list
wallet = wallets[num_ygs]['wallet']
print "Seed : " + wallets[num_ygs]['seed']
#useful to see the utxos on screen sometimes
sync_wallet(wallet)
print wallet.unspent
txfee = 1000
cjfee_a = 4200
cjfee_r = '0.001'
ordertype = 'swreloffer'
minsize = 100000
for i in range(num_ygs):
cfg = [txfee, cjfee_a, cjfee_r, ordertype, minsize]
sync_wallet(wallets[i]["wallet"])
yg = YieldGeneratorBasic(wallets[i]["wallet"], cfg)
clientfactory = JMClientProtocolFactory(yg, proto_type="MAKER")
nodaemon = jm_single().config.getint("DAEMON", "no_daemon")
daemon = True if nodaemon == 1 else False
rs = True if i == num_ygs - 1 else False
start_reactor(jm_single().config.get("DAEMON", "daemon_host"),
jm_single().config.getint("DAEMON", "daemon_port"),
clientfactory, daemon=daemon, rs=rs)
time.sleep(2) #give it a chance
@pytest.fixture(scope="module")
def setup_ygrunner():
load_program_config()
Loading…
Cancel
Save