From d34c53bc05200da9edb49230a9990415d0aed9ac Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Tue, 9 Jun 2020 20:27:57 +0100 Subject: [PATCH] Various fixups: Upgrade python-bitcointx to 1.1.0: Address requirements of python-bitcointx 1.1.0: Specifically, the witness `utxo` field can no longer be assumed to be of type CTxOut, so we should access the CTxOut with the field witness_utxo and also when updating the `utxo` field we now use `set_utxo()`. Use PartiallySignedTransaction.get_fee() method. Use PartiallySignedTransaction.set_utxo. Additionally some minor typos/comment corrections and removal of the now defunct `apply_freeze_signature`. Add custom load location for libsecp where needed; falls back to system installation if Joinmarket custom installation is not found. Decode error msg from server in payjoin Cleanup test file test_proposals.txt (delete after test) Human readable function names (names for human readable conversions are now themselves human readable). Remove unused get_*_vbyte functions and cleanup Removes old unused files (electrum*.py). Fixes core nohistory sync test to use both standard wallet types, and fixes address import counter. Fixes that test to use the right chain params so that native segwit wallets can work in regtest with nohistory mode. Removes some now unneeded imports. Fixes commontest.create_wallet_for_sync to hash all parameters, including optional ones. Replaces usage of binascii.hexlify with bintohex. --- .gitignore | 1 + jmbitcoin/jmbitcoin/__init__.py | 11 + jmbitcoin/jmbitcoin/secp256k1_transaction.py | 22 +- jmbitcoin/setup.py | 2 +- jmbitcoin/test/test_tx_signing.py | 3 +- jmclient/jmclient/__init__.py | 5 +- jmclient/jmclient/configure.py | 12 +- jmclient/jmclient/electrum_data.py | 261 --------- jmclient/jmclient/electruminterface.py | 549 ------------------- jmclient/jmclient/maker.py | 2 +- jmclient/jmclient/payjoin.py | 34 +- jmclient/jmclient/taker.py | 6 +- jmclient/jmclient/taker_utils.py | 11 +- jmclient/jmclient/wallet.py | 34 +- jmclient/test/commontest.py | 13 +- jmclient/test/test_configure.py | 10 +- jmclient/test/test_core_nohistory_sync.py | 26 +- jmclient/test/test_psbt_wallet.py | 16 +- jmclient/test/test_snicker.py | 18 +- test/payjoinserver.py | 6 +- 20 files changed, 122 insertions(+), 920 deletions(-) delete mode 100644 jmclient/jmclient/electrum_data.py delete mode 100644 jmclient/jmclient/electruminterface.py diff --git a/.gitignore b/.gitignore index ce381d7..6352064 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ miniircd/ miniircd.tar.gz nums_basepoints.txt schedulefortesting +test_proposals.txt scripts/commitmentlist tmp/ wallets/ diff --git a/jmbitcoin/jmbitcoin/__init__.py b/jmbitcoin/jmbitcoin/__init__.py index 6ab579f..2a19f7f 100644 --- a/jmbitcoin/jmbitcoin/__init__.py +++ b/jmbitcoin/jmbitcoin/__init__.py @@ -1,4 +1,15 @@ import coincurve as secp256k1 + +# If user has compiled and installed libsecp256k1 via +# JM installation script install.sh, use that; +# if not, it is assumed to be present at the system level +# See: https://github.com/Simplexum/python-bitcointx/commit/79333106eeb55841df2935781646369b186d99f7#diff-1ea6586127522e62d109ec5893a18850R301-R310 +import os, sys +expected_secp_location = os.path.join(sys.prefix, "lib", "libsecp256k1.so") +if os.path.exists(expected_secp_location): + import bitcointx + bitcointx.set_custom_secp256k1_path(expected_secp_location) + from jmbitcoin.secp256k1_main import * from jmbitcoin.secp256k1_transaction import * from jmbitcoin.secp256k1_deterministic import * diff --git a/jmbitcoin/jmbitcoin/secp256k1_transaction.py b/jmbitcoin/jmbitcoin/secp256k1_transaction.py index 4b5f65b..7cf9fb5 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_transaction.py +++ b/jmbitcoin/jmbitcoin/secp256k1_transaction.py @@ -18,7 +18,7 @@ from bitcointx.wallet import (P2WPKHCoinAddress, CCoinAddress, P2PKHCoinAddress, from bitcointx.core.scripteval import (VerifyScript, SCRIPT_VERIFY_WITNESS, SCRIPT_VERIFY_P2SH, SIGVERSION_WITNESS_V0) -def hrt(tx, jsonified=True): +def human_readable_transaction(tx, jsonified=True): """ Given a CTransaction object, output a human readable json-formatted string (suitable for terminal output or large GUI textbox display) containing @@ -40,14 +40,14 @@ def hrt(tx, jsonified=True): witarg = None else: witarg = tx.wit.vtxinwit[i] - outdict["inputs"].append(hrinp(inp, witarg)) + outdict["inputs"].append(human_readable_input(inp, witarg)) for i, out in enumerate(tx.vout): - outdict["outputs"].append(hrout(out)) + outdict["outputs"].append(human_readable_output(out)) if not jsonified: return outdict return json.dumps(outdict, indent=4) -def hrinp(txinput, txinput_witness): +def human_readable_input(txinput, txinput_witness): """ Pass objects of type CTxIn and CTxInWitness (or None) and a dict of human-readable entries for this input is returned. @@ -66,7 +66,7 @@ def hrinp(txinput, txinput_witness): txinput_witness.scriptWitness.serialize()) return outdict -def hrout(txoutput): +def human_readable_output(txoutput): """ Returns a dict of human-readable entries for this output. """ @@ -232,7 +232,7 @@ def sign(tx, i, priv, hashcode=SIGHASH_ALL, amount=None, native=False): else: # segwit case; we currently support p2wpkh native or under p2sh. - # see line 1256 of bitcointx.core.scripteval.py: + # https://github.com/Simplexum/python-bitcointx/blob/648ad8f45ff853bf9923c6498bfa0648b3d7bcbd/bitcointx/core/scripteval.py#L1250-L1252 flags.add(SCRIPT_VERIFY_P2SH) if native and native != "p2wpkh": @@ -273,16 +273,6 @@ def sign(tx, i, priv, hashcode=SIGHASH_ALL, amount=None, native=False): return sig, "signing succeeded" -def apply_freeze_signature(tx, i, redeem_script, sig): - if isinstance(redeem_script, str): - redeem_script = binascii.unhexlify(redeem_script) - if isinstance(sig, str): - sig = binascii.unhexlify(sig) - txobj = deserialize(tx) - txobj["ins"][i]["script"] = "" - txobj["ins"][i]["txinwitness"] = [sig, redeem_script] - return serialize(txobj) - def mktx(ins, outs, version=1, locktime=0): """ Given a list of input tuples (txid(bytes), n(int)), and a list of outputs which are dicts with diff --git a/jmbitcoin/setup.py b/jmbitcoin/setup.py index 9330fc0..00adc0a 100644 --- a/jmbitcoin/setup.py +++ b/jmbitcoin/setup.py @@ -10,5 +10,5 @@ setup(name='joinmarketbitcoin', license='GPL', packages=['jmbitcoin'], python_requires='>=3.6', - install_requires=['coincurve', 'python-bitcointx>=1.0.5', 'pyaes', 'urldecode'], + install_requires=['coincurve', 'python-bitcointx>=1.1.0', 'pyaes', 'urldecode'], zip_safe=False) diff --git a/jmbitcoin/test/test_tx_signing.py b/jmbitcoin/test/test_tx_signing.py index e2ccf21..28ca2ba 100644 --- a/jmbitcoin/test/test_tx_signing.py +++ b/jmbitcoin/test/test_tx_signing.py @@ -61,7 +61,8 @@ def test_sign_standard_txs(addrtype): raise print("created signature: ", bintohex(sig)) print("serialized transaction: {}".format(bintohex(tx.serialize()))) - print("deserialized transaction: {}\n".format(btc.hrt(tx))) + print("deserialized transaction: {}\n".format( + btc.human_readable_transaction(tx))) def test_mk_shuffled_tx(): # prepare two addresses for the outputs diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index d222478..0a48658 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -21,13 +21,12 @@ from .storage import (Argon2Hash, Storage, StorageError, RetryableStorageError, from .cryptoengine import (BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, EngineError, TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH) from .configure import (load_test_config, - load_program_config, get_p2pk_vbyte, jm_single, get_network, update_persist_config, + load_program_config, jm_single, get_network, update_persist_config, validate_address, is_burn_destination, get_irc_mchannels, - get_blockchain_interface_instance, get_p2sh_vbyte, set_config, is_segwit_mode, + get_blockchain_interface_instance, set_config, is_segwit_mode, is_native_segwit_mode) from .blockchaininterface import (BlockchainInterface, RegtestBitcoinCoreInterface, BitcoinCoreInterface) -from .electruminterface import ElectrumInterface from .client_protocol import (JMTakerClientProtocol, JMClientProtocolFactory, start_reactor) from .podle import (set_commitment_file, get_commitment_file, diff --git a/jmclient/jmclient/configure.py b/jmclient/jmclient/configure.py index 17c885c..c2cb920 100644 --- a/jmclient/jmclient/configure.py +++ b/jmclient/jmclient/configure.py @@ -366,15 +366,6 @@ def get_network(): """Returns network name""" return global_singleton.config.get("BLOCKCHAIN", "network") - -def get_p2sh_vbyte(): - return btc.BTC_P2SH_VBYTE[get_network()] - - -def get_p2pk_vbyte(): - return btc.BTC_P2PK_VBYTE[get_network()] - - def validate_address(addr): try: # automatically respects the network @@ -570,7 +561,8 @@ def get_blockchain_interface_instance(_config): elif source == "bitcoin-rpc-no-history": bc_interface = BitcoinCoreNoHistoryInterface(rpc, network) if testnet or network == "regtest": - # TODO will not work for bech32 regtest addresses: + # in tests, for bech32 regtest addresses, for bc-no-history, + # this will have to be reset manually: btc.select_chain_params("bitcoin/testnet") else: btc.select_chain_params("bitcoin") diff --git a/jmclient/jmclient/electrum_data.py b/jmclient/jmclient/electrum_data.py deleted file mode 100644 index 393f49c..0000000 --- a/jmclient/jmclient/electrum_data.py +++ /dev/null @@ -1,261 +0,0 @@ -# Default server list from electrum client -# https://github.com/spesmilo/electrum, file https://github.com/spesmilo/electrum/blob/7dbd612d5dad13cd6f1c0df32534a578bad331ad/lib/servers.json - -#Edit this to 't' instead of 's' to use TCP; -#This is specifically not exposed in joinmarket.cfg -#since there is no good reason to prefer TCP over SSL -#unless the latter simply doesn't work. -DEFAULT_PROTO = 's' - -DEFAULT_PORTS = {'t':'50001', 's':'50002'} - -DEFAULT_SERVERS = { - "E-X.not.fyi": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "ELECTRUMX.not.fyi": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "ELEX01.blackpole.online": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "VPS.hsmiths.com": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "bitcoin.freedomnode.com": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "btc.smsys.me": { - "pruning": "-", - "s": "995", - "version": "1.1" - }, - "currentlane.lovebitco.in": { - "pruning": "-", - "t": "50001", - "version": "1.1" - }, - "daedalus.bauerj.eu": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "de01.hamster.science": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "ecdsa.net": { - "pruning": "-", - "s": "110", - "t": "50001", - "version": "1.1" - }, - "elec.luggs.co": { - "pruning": "-", - "s": "443", - "version": "1.1" - }, - "electrum.akinbo.org": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrum.antumbra.se": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrum.be": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrum.coinucopia.io": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrum.cutie.ga": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrum.festivaldelhumor.org": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrum.hsmiths.com": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrum.qtornado.com": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrum.vom-stausee.de": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrum3.hachre.de": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrumx.bot.nu": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrumx.westeurope.cloudapp.azure.com": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "elx01.knas.systems": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "ex-btc.server-on.net": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "helicarrier.bauerj.eu": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "mooo.not.fyi": { - "pruning": "-", - "s": "50012", - "t": "50011", - "version": "1.1" - }, - "ndnd.selfhost.eu": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "node.arihanc.com": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "node.xbt.eu": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "node1.volatilevictory.com": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "noserver4u.de": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "qmebr.spdns.org": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "raspi.hsmiths.com": { - "pruning": "-", - "s": "51002", - "t": "51001", - "version": "1.1" - }, - "s2.noip.pl": { - "pruning": "-", - "s": "50102", - "version": "1.1" - }, - "s5.noip.pl": { - "pruning": "-", - "s": "50105", - "version": "1.1" - }, - "songbird.bauerj.eu": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "us.electrum.be": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "us01.hamster.science": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - } -} - -def set_electrum_testnet(): - global DEFAULT_PORTS, DEFAULT_SERVERS - DEFAULT_PORTS = {'t':'51001', 's':'51002'} - DEFAULT_SERVERS = { - 'testnetnode.arihanc.com': {'t':'51001', 's':'51002'}, - 'testnet1.bauerj.eu': {'t':'51001', 's':'51002'}, - #'14.3.140.101': {'t':'51001', 's':'51002'}, #non-responsive? - 'testnet.hsmiths.com': {'t':'53011', 's':'53012'}, - 'electrum.akinbo.org': {'t':'51001', 's':'51002'}, - 'ELEX05.blackpole.online': {'t':'52011', 's':'52002'},} - #Replace with for regtest: - #'localhost': {'t': '50001', 's': '51002'},} - -def get_default_servers(): - return DEFAULT_SERVERS - -def get_default_ports(): - return DEFAULT_PORTS \ No newline at end of file diff --git a/jmclient/jmclient/electruminterface.py b/jmclient/jmclient/electruminterface.py deleted file mode 100644 index 773d830..0000000 --- a/jmclient/jmclient/electruminterface.py +++ /dev/null @@ -1,549 +0,0 @@ -import jmbitcoin as btc -import json -import queue as Queue -import os -import pprint -import random -import socket -import threading -import ssl -import binascii -from twisted.internet.protocol import ClientFactory -from twisted.internet.ssl import ClientContextFactory -from twisted.protocols.basic import LineReceiver -from twisted.internet import reactor, task, defer -from .blockchaininterface import BlockchainInterface -from .configure import get_p2sh_vbyte -from jmbase import get_log, jmprint -from .electrum_data import get_default_servers, set_electrum_testnet,\ - DEFAULT_PROTO - -log = get_log() - -class ElectrumConnectionError(Exception): - pass - -class TxElectrumClientProtocol(LineReceiver): - #map deferreds to msgids to correctly link response with request - deferreds = {} - delimiter = b"\n" - - def __init__(self, factory): - self.factory = factory - - def connectionMade(self): - log.debug('connection to Electrum succesful') - self.msg_id = 0 - if self.factory.bci.wallet: - #Use connectionMade as a trigger to start wallet sync, - #if the reactor start happened after the call to wallet sync - #(in Qt, the reactor starts before wallet sync, so we make - #this call manually instead). - self.factory.bci.sync_addresses(self.factory.bci.wallet) - #these server calls must always be done to keep the connection open - self.start_ping() - self.call_server_method('blockchain.numblocks.subscribe') - - def start_ping(self): - pingloop = task.LoopingCall(self.ping) - pingloop.start(60.0) - - def ping(self): - #We dont bother tracking response to this; - #just for keeping connection active - self.call_server_method('server.version') - - def send_json(self, json_data): - data = json.dumps(json_data).encode() - self.sendLine(data) - - def call_server_method(self, method, params=[]): - self.msg_id = self.msg_id + 1 - current_id = self.msg_id - self.deferreds[current_id] = defer.Deferred() - method_dict = { - 'id': current_id, - 'method': method, - 'params': params - } - self.send_json(method_dict) - return self.deferreds[current_id] - - def lineReceived(self, line): - try: - parsed = json.loads(line.decode()) - msgid = parsed['id'] - linked_deferred = self.deferreds[msgid] - except: - log.debug("Ignored response from Electrum server: " + str(line)) - return - linked_deferred.callback(parsed) - -class TxElectrumClientProtocolFactory(ClientFactory): - - def __init__(self, bci): - self.bci = bci - def buildProtocol(self,addr): - self.client = TxElectrumClientProtocol(self) - return self.client - - def clientConnectionLost(self, connector, reason): - log.debug('Electrum connection lost, reason: ' + str(reason)) - self.bci.start_electrum_proto(None) - - def clientConnectionFailed(self, connector, reason): - jmprint('connection failed', "warning") - self.bci.start_electrum_proto(None) - -class ElectrumConn(threading.Thread): - - def __init__(self, server, port, proto): - threading.Thread.__init__(self) - self.daemon = True - self.msg_id = 0 - self.RetQueue = Queue.Queue() - try: - if proto == 't': - self.s = socket.create_connection((server,int(port))) - elif proto == 's': - self.raw_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - #reads are sometimes quite slow, so conservative, but we must - #time out a completely hanging connection. - self.raw_socket.settimeout(60) - self.raw_socket.connect((server, int(port))) - self.s = ssl.wrap_socket(self.raw_socket) - else: - #Wrong proto is not accepted for restarts - log.error("Failure to connect to Electrum, " - "protocol must be TCP or SSL.") - os._exit(1) - except Exception as e: - log.error("Error connecting to electrum server; trying again.") - raise ElectrumConnectionError - self.ping() - - def run(self): - while True: - all_data = None - while True: - data = self.s.recv(1024) - if data is None: - continue - if all_data is None: - all_data = data - else: - all_data = all_data + data - if b'\n' in all_data: - break - data_json = json.loads(all_data[:-1].decode()) - self.RetQueue.put(data_json) - - def ping(self): - log.debug('Sending Electrum server ping') - self.send_json({'id':0,'method':'server.version','params':[]}) - t = threading.Timer(60, self.ping) - t.daemon = True - t.start() - - def send_json(self, json_data): - data = json.dumps(json_data).encode() - self.s.send(data + b'\n') - - def call_server_method(self, method, params=[]): - self.msg_id = self.msg_id + 1 - current_id = self.msg_id - method_dict = { - 'id': current_id, - 'method': method, - 'params': params - } - self.send_json(method_dict) - while True: - ret_data = self.RetQueue.get() - if ret_data.get('id', None) == current_id: - return ret_data - else: - log.debug(json.dumps(ret_data)) - -class ElectrumInterface(BlockchainInterface): - BATCH_SIZE = 8 - def __init__(self, testnet=False, electrum_server=None): - self.synctype = "sync-only" - if testnet: - set_electrum_testnet() - self.start_electrum_proto() - self.electrum_conn = None - self.start_connection_thread() - #task.LoopingCall objects that track transactions, keyed by txids. - #Format: {"txid": (loop, unconfirmed true/false, confirmed true/false, - #spent true/false), ..} - self.tx_watcher_loops = {} - self.wallet = None - self.wallet_synced = False - - def start_electrum_proto(self, electrum_server=None): - self.server, self.port = self.get_server(electrum_server) - self.factory = TxElectrumClientProtocolFactory(self) - if DEFAULT_PROTO == 's': - ctx = ClientContextFactory() - reactor.connectSSL(self.server, self.port, self.factory, ctx) - elif DEFAULT_PROTO == 't': - reactor.connectTCP(self.server, self.port, self.factory) - else: - raise Exception("Unrecognized connection protocol to Electrum, " - "should be one of 't' or 's' (TCP or SSL), " - "critical error, quitting.") - - def start_connection_thread(self): - """Initiate a thread that serves blocking, single - calls to an Electrum server. This won't usually be the - same server that's used to do sync (which, confusingly, - is asynchronous). - """ - try: - s, p = self.get_server(None) - self.electrum_conn = ElectrumConn(s, p, DEFAULT_PROTO) - except ElectrumConnectionError: - reactor.callLater(1.0, self.start_connection_thread) - return - self.electrum_conn.start() - #used to hold open server conn - self.electrum_conn.call_server_method('blockchain.numblocks.subscribe') - - def sync_wallet(self, wallet, fast=False, restart_cb=False): - """This triggers the start of syncing, wiping temporary state - and starting the reactor for wallet-tool runs. The 'fast' - and 'restart_cb' parameters are ignored and included only - for compatibility; they are both only used by Core. - """ - self.wallet = wallet - #wipe the temporary cache of address histories - self.temp_addr_history = {} - #mark as not currently synced - self.wallet_synced = False - if self.synctype == "sync-only": - if not reactor.running: - reactor.run() - - def get_server(self, electrum_server): - if not electrum_server: - while True: - electrum_server = random.choice(list(get_default_servers().keys())) - if DEFAULT_PROTO in get_default_servers()[electrum_server]: - break - s = electrum_server - p = int(get_default_servers()[electrum_server][DEFAULT_PROTO]) - log.debug('Trying to connect to Electrum server: ' + str(electrum_server)) - return (s, p) - - def get_from_electrum(self, method, params=[], blocking=False): - params = [params] if type(params) is not list else params - if blocking: - return self.electrum_conn.call_server_method(method, params) - else: - return self.factory.client.call_server_method(method, params) - - def sync_addresses(self, wallet, restart_cb=None): - if not self.electrum_conn: - #wait until we have some connection up before starting - reactor.callLater(0.2, self.sync_addresses, wallet, restart_cb) - return - log.debug("downloading wallet history from Electrum server ...") - for mixdepth in range(wallet.max_mixdepth + 1): - for forchange in [0, 1]: - #start from a clean index - wallet.set_next_index(mixdepth, forchange, 0) - self.synchronize_batch(wallet, mixdepth, forchange, 0) - - def synchronize_batch(self, wallet, mixdepth, forchange, start_index): - #for debugging only: - #log.debug("Syncing address batch, m, fc, i: " + ",".join( - # [str(x) for x in [mixdepth, forchange, start_index]])) - if mixdepth not in self.temp_addr_history: - self.temp_addr_history[mixdepth] = {} - if forchange not in self.temp_addr_history[mixdepth]: - self.temp_addr_history[mixdepth][forchange] = {"finished": False} - for i in range(start_index, start_index + self.BATCH_SIZE): - #get_new_addr is OK here, as guaranteed to be sequential *on this branch* - a = wallet.get_new_addr(mixdepth, forchange) - d = self.get_from_electrum('blockchain.address.get_history', a) - #makes sure entries in temporary address history are ready - #to be accessed. - if i not in self.temp_addr_history[mixdepth][forchange]: - self.temp_addr_history[mixdepth][forchange][i] = {'synced': False, - 'addr': a, - 'used': False} - d.addCallback(self.process_address_history, wallet, - mixdepth, forchange, i, a, start_index) - - def process_address_history(self, history, wallet, mixdepth, forchange, i, - addr, start_index): - """Given the history data for an address from Electrum, update the current view - of the wallet's usage at mixdepth mixdepth and account forchange, address addr at - index i. Once all addresses from index start_index to start_index + self.BATCH_SIZE - have been thus updated, trigger either continuation to the next batch, or, if - conditions are fulfilled, end syncing for this (mixdepth, forchange) branch, and - if all such branches are finished, proceed to the sync_unspent step. - """ - tah = self.temp_addr_history[mixdepth][forchange] - if len(history['result']) > 0: - tah[i]['used'] = True - tah[i]['synced'] = True - #Having updated this specific record, check if the entire batch from start_index - #has been synchronized - if all([tah[j]['synced'] for j in range(start_index, start_index + self.BATCH_SIZE)]): - #check if unused goes back as much as gaplimit *and* we are ahead of any - #existing index_cache from the wallet file; if both true, end, else, continue - #to next batch - if all([tah[j]['used'] is False for j in range( - start_index + self.BATCH_SIZE - wallet.gap_limit, - start_index + self.BATCH_SIZE)]): - last_used_addr = None - #to find last used, note that it may be in the *previous* batch; - #may as well just search from the start, since it takes no time. - for j in range(start_index + self.BATCH_SIZE): - if tah[j]['used']: - last_used_addr = tah[j]['addr'] - if last_used_addr: - wallet.set_next_index( - mixdepth, forchange, - wallet.get_next_unused_index(mixdepth, forchange)) - else: - wallet.set_next_index(mixdepth, forchange, 0) - tah["finished"] = True - #check if all branches are finished to trigger next stage of sync. - addr_sync_complete = True - for m in range(wallet.max_mix_depth): - for fc in [0, 1]: - if not self.temp_addr_history[m][fc]["finished"]: - addr_sync_complete = False - if addr_sync_complete: - self.sync_unspent(wallet) - else: - #continue search forwards on this branch - self.synchronize_batch(wallet, mixdepth, forchange, start_index + self.BATCH_SIZE) - - def sync_unspent(self, wallet): - # finds utxos in the wallet - wallet.reset_utxos() - #Prepare list of all used addresses - addrs = set() - for m in range(wallet.max_mixdepth): - for fc in [0, 1]: - branch_list = [] - for k, v in self.temp_addr_history[m][fc].items(): - if k == "finished": - continue - if v["used"]: - branch_list.append(v["addr"]) - addrs.update(branch_list) - if len(addrs) == 0: - log.debug('no tx used') - self.wallet_synced = True - if self.synctype == 'sync-only': - reactor.stop() - return - #make sure to add any addresses during the run (a subset of those - #added to the address cache) - for md in range(wallet.max_mixdepth): - for internal in (True, False): - for index in range(wallet.get_next_unused_index(md, internal)): - addrs.add(wallet.get_addr(md, internal, index)) - for path in wallet.yield_imported_paths(md): - addrs.add(wallet.get_address_from_path(path)) - - self.listunspent_calls = len(addrs) - for a in addrs: - # FIXME: update to protocol version 1.1 and use scripthash instead - script = wallet.addr_to_script(a) - d = self.get_from_electrum('blockchain.address.listunspent', a) - d.addCallback(self.process_listunspent_data, wallet, script) - - def process_listunspent_data(self, unspent_info, wallet, script): - res = unspent_info['result'] - for u in res: - txid = binascii.unhexlify(u['tx_hash']) - wallet.add_utxo(txid, int(u['tx_pos']), script, int(u['value'])) - - self.listunspent_calls -= 1 - if self.listunspent_calls == 0: - self.wallet_synced = True - if self.synctype == "sync-only": - reactor.stop() - - def pushtx(self, txhex): - brcst_res = self.get_from_electrum('blockchain.transaction.broadcast', - txhex, blocking=True) - brcst_status = brcst_res['result'] - if isinstance(brcst_status, str) and len(brcst_status) == 64: - return (True, brcst_status) - log.debug(brcst_status) - return (False, None) - - def query_utxo_set(self, txout, includeconf=False): - self.current_height = self.get_from_electrum( - "blockchain.numblocks.subscribe", blocking=True)['result'] - if not isinstance(txout, list): - txout = [txout] - utxos = [[t[:64],int(t[65:])] for t in txout] - result = [] - for ut in utxos: - address = self.get_from_electrum("blockchain.utxo.get_address", - ut, blocking=True)['result'] - utxo_info = self.get_from_electrum("blockchain.address.listunspent", - address, blocking=True)['result'] - utxo = None - for u in utxo_info: - if u['tx_hash'] == ut[0] and u['tx_pos'] == ut[1]: - utxo = u - if utxo is None: - result.append(None) - else: - r = { - 'value': utxo['value'], - 'address': address, - 'script': btc.address_to_script(address) - } - if includeconf: - if int(utxo['height']) in [0, -1]: - #-1 means unconfirmed inputs - r['confirms'] = 0 - else: - #+1 because if current height = tx height, that's 1 conf - r['confirms'] = int(self.current_height) - int( - utxo['height']) + 1 - result.append(r) - return result - - def estimate_fee_per_kb(self, N): - if super(ElectrumInterface, self).fee_per_kb_has_been_manually_set(N): - return int(random.uniform(N * float(0.8), N * float(1.2))) - fee_info = self.get_from_electrum('blockchain.estimatefee', N, blocking=True) - jmprint('got fee info result: ' + str(fee_info), "debug") - fee = fee_info.get('result') - fee_per_kb_sat = int(float(fee) * 100000000) - return fee_per_kb_sat - - def outputs_watcher(self, wallet_name, notifyaddr, tx_output_set, - unconfirmfun, confirmfun, timeoutfun): - """Given a key for the watcher loop (notifyaddr), a wallet name (account), - a set of outputs, and unconfirm, confirm and timeout callbacks, - check to see if a transaction matching that output set has appeared in - the wallet. Call the callbacks and update the watcher loop state. - End the loop when the confirmation has been seen (no spent monitoring here). - """ - wl = self.tx_watcher_loops[notifyaddr] - jmprint('txoutset=' + pprint.pformat(tx_output_set), "debug") - unconftx = self.get_from_electrum('blockchain.address.get_mempool', - notifyaddr, blocking=True).get('result') - unconftxs = set([str(t['tx_hash']) for t in unconftx]) - if len(unconftxs): - txdatas = [] - for txid in unconftxs: - txdatas.append({'id': txid, - 'hex':str(self.get_from_electrum( - 'blockchain.transaction.get',txid, - blocking=True).get('result'))}) - unconfirmed_txid = None - for txdata in txdatas: - txhex = txdata['hex'] - outs = set([(sv['script'], sv['value']) for sv in btc.deserialize( - txhex)['outs']]) - jmprint('unconfirm query outs = ' + str(outs), "debug") - if outs == tx_output_set: - unconfirmed_txid = txdata['id'] - unconfirmed_txhex = txhex - break - #call unconf callback if it was found in the mempool - if unconfirmed_txid and not wl[1]: - jmprint("Tx: " + str(unconfirmed_txid) + " seen on network.", "info") - unconfirmfun(btc.deserialize(unconfirmed_txhex), unconfirmed_txid) - wl[1] = True - return - - conftx = self.get_from_electrum('blockchain.address.listunspent', - notifyaddr, blocking=True).get('result') - conftxs = set([str(t['tx_hash']) for t in conftx]) - if len(conftxs): - txdatas = [] - for txid in conftxs: - txdata = str(self.get_from_electrum('blockchain.transaction.get', - txid, blocking=True).get('result')) - txdatas.append({'hex':txdata,'id':txid}) - confirmed_txid = None - for txdata in txdatas: - txhex = txdata['hex'] - outs = set([(sv['script'], sv['value']) for sv in btc.deserialize( - txhex)['outs']]) - jmprint('confirm query outs = ' + str(outs), "info") - if outs == tx_output_set: - confirmed_txid = txdata['id'] - confirmed_txhex = txhex - break - if confirmed_txid and not wl[2]: - confirmfun(btc.deserialize(confirmed_txhex), confirmed_txid, 1) - wl[2] = True - wl[0].stop() - return - - def tx_watcher(self, txd, unconfirmfun, confirmfun, spentfun, c, n): - """Called at a polling interval, checks if the given deserialized - transaction (which must be fully signed) is (a) broadcast, (b) confirmed - and (c) spent from. (c, n ignored in electrum version, just supports - registering first confirmation). - TODO: There is no handling of conflicts here. - """ - txid = btc.txhash(btc.serialize(txd)) - wl = self.tx_watcher_loops[txid] - #first check if in mempool (unconfirmed) - #choose an output address for the query. Filter out - #p2pkh addresses, assume p2sh (thus would fail to find tx on - #some nonstandard script type) - addr = None - for i in range(len(txd['outs'])): - if not btc.is_p2pkh_script(txd['outs'][i]['script']): - addr = btc.script_to_address(txd['outs'][i]['script'], get_p2sh_vbyte()) - break - if not addr: - log.error("Failed to find any p2sh output, cannot be a standard " - "joinmarket transaction, fatal error!") - reactor.stop() - return - unconftxs_res = self.get_from_electrum('blockchain.address.get_mempool', - addr, blocking=True).get('result') - unconftxs = [str(t['tx_hash']) for t in unconftxs_res] - - if not wl[1] and txid in unconftxs: - jmprint("Tx: " + str(txid) + " seen on network.", "info") - unconfirmfun(txd, txid) - wl[1] = True - return - conftx = self.get_from_electrum('blockchain.address.listunspent', - addr, blocking=True).get('result') - conftxs = [str(t['tx_hash']) for t in conftx] - if not wl[2] and len(conftxs) and txid in conftxs: - jmprint("Tx: " + str(txid) + " is confirmed.", "info") - confirmfun(txd, txid, 1) - wl[2] = True - #Note we do not stop the monitoring loop when - #confirmations occur, since we are also monitoring for spending. - return - if not spentfun or wl[3]: - return - - def rpc(self, method, args): - # FIXME: this is very poorly written code - if method == 'gettransaction': - assert len(args) == 1 - return self._gettransaction(args[0]) - else: - raise NotImplementedError(method) - - def _gettransaction(self, txid): - # FIXME: this is not complete and only implemented to work with - # wallet_utils - return { - 'hex': str(self.get_from_electrum('blockchain.transaction.get', - txid, blocking=True) - .get('result')) - } diff --git a/jmclient/jmclient/maker.py b/jmclient/jmclient/maker.py index 40ffc3a..3838ec3 100644 --- a/jmclient/jmclient/maker.py +++ b/jmclient/jmclient/maker.py @@ -132,7 +132,7 @@ class Maker(object): return (False, 'malformed txhex. ' + repr(e)) # if the above deserialization was successful, the human readable # parsing will be also: - jlog.info('obtained tx\n' + btc.hrt(tx)) + jlog.info('obtained tx\n' + btc.human_readable_transaction(tx)) goodtx, errmsg = self.verify_unsigned_tx(tx, offerinfo) if not goodtx: jlog.info('not a good tx, reason=' + errmsg) diff --git a/jmclient/jmclient/payjoin.py b/jmclient/jmclient/payjoin.py index 9c116dd..37bb4da 100644 --- a/jmclient/jmclient/payjoin.py +++ b/jmclient/jmclient/payjoin.py @@ -116,7 +116,7 @@ class JMPayjoinManager(object): # inputs must all have witness utxo populated for inp in self.initial_psbt.inputs: - if not inp.utxo and isinstance(inp.utxo, btc.CTxOut): + if not isinstance(inp.witness_utxo, btc.CTxOut): return False # check that there is no xpub or derivation info @@ -191,7 +191,7 @@ class JMPayjoinManager(object): else: receiver_input_indices.append(i) - if any([found[i] != 1 for i in range(len(found))]): + if any([f != 1 for f in found]): return (False, "Receiver proposed PSBT does not contain our inputs.") # 3 found = 0 @@ -229,9 +229,11 @@ class JMPayjoinManager(object): # version (so all witnesses filled in) to calculate its size, # then compare that with the fee, and do the same for the # pre-existing non-payjoin. - gffp = PSBTWalletMixin.get_fee_from_psbt - proposed_tx_fee = gffp(signed_psbt_for_fees) - nonpayjoin_tx_fee = gffp(self.initial_psbt) + try: + proposed_tx_fee = signed_psbt_for_fees.get_fee() + except ValueError: + return (False, "receiver proposed tx has negative fee.") + nonpayjoin_tx_fee = self.initial_psbt.get_fee() proposed_tx_size = signed_psbt_for_fees.extract_transaction( ).get_virtual_size() nonpayjoin_tx_size = self.initial_psbt.extract_transaction( @@ -329,16 +331,16 @@ class JMPayjoinManager(object): reportdict = {"name:", "PAYJOIN STATUS REPORT"} reportdict["status"] = self.pj_state # TODO: string if self.payment_tx: - txdata = btc.hrt(self.payment_tx) + txdata = btc.human_readable_transaction(self.payment_tx) if verbose: txdata = txdata["hex"] reportdict["payment-tx"] = txdata if self.payjoin_psbt: - psbtdata = PSBTWalletMixin.hr_psbt( + psbtdata = PSBTWalletMixin.human_readable_psbt( self.payjoin_psbt) if verbose else self.payjoin_psbt.to_base64() reportdict["payjoin-proposed"] = psbtdata if self.final_psbt: - finaldata = PSBTWalletMixin.hr_psbt( + finaldata = PSBTWalletMixin.human_readable_psbt( self.final_psbt) if verbose else self.final_psbt.to_base64() reportdict["payjoin-final"] = finaldata if jsonified: @@ -427,12 +429,12 @@ def send_payjoin(manager, accept_callback=None, def fallback_nonpayjoin_broadcast(manager, err): assert isinstance(manager, JMPayjoinManager) log.warn("Payjoin did not succeed, falling back to non-payjoin payment.") - log.warn("Error message was: " + str(err)) + log.warn("Error message was: " + err.decode("utf-8")) original_tx = manager.initial_psbt.extract_transaction() if not jm_single().bc_interface.pushtx(original_tx.serialize()): log.error("Unable to broadcast original payment. The payment is NOT made.") log.info("We paid without coinjoin. Transaction: ") - log.info(btc.hrt(original_tx)) + log.info(btc.human_readable_transaction(original_tx)) reactor.stop() def receive_payjoin_proposal_from_server(response, manager): @@ -463,15 +465,15 @@ def process_payjoin_proposal_from_server(response_body, manager): return log.debug("Receiver sent us this PSBT: ") - log.debug(manager.wallet_service.hr_psbt(payjoin_proposal_psbt)) + log.debug(manager.wallet_service.human_readable_psbt(payjoin_proposal_psbt)) # we need to add back in our utxo information to the received psbt, # since the servers remove it (not sure why?) for i, inp in enumerate(payjoin_proposal_psbt.unsigned_tx.vin): for j, inp2 in enumerate(manager.initial_psbt.unsigned_tx.vin): if (inp.prevout.hash, inp.prevout.n) == ( inp2.prevout.hash, inp2.prevout.n): - payjoin_proposal_psbt.inputs[i].utxo = \ - manager.initial_psbt.inputs[j].utxo + payjoin_proposal_psbt.set_utxo( + manager.initial_psbt.inputs[j].utxo, i) signresultandpsbt, err = manager.wallet_service.sign_psbt( payjoin_proposal_psbt.serialize(), with_sign_result=True) if err: @@ -489,15 +491,15 @@ def process_payjoin_proposal_from_server(response_body, manager): # All checks have passed. We can use the already signed transaction in # sender_signed_psbt. log.info("Our final signed PSBT is:\n{}".format( - manager.wallet_service.hr_psbt(sender_signed_psbt))) + manager.wallet_service.human_readable_psbt(sender_signed_psbt))) manager.set_final_payjoin_psbt(sender_signed_psbt) # broadcast the tx extracted_tx = sender_signed_psbt.extract_transaction() log.info("Here is the final payjoin transaction:") - log.info(btc.hrt(extracted_tx)) + log.info(btc.human_readable_transaction(extracted_tx)) if not jm_single().bc_interface.pushtx(extracted_tx.serialize()): log.info("The above transaction failed to broadcast.") else: - log.info("Payjoin transactoin broadcast successfully.") + log.info("Payjoin transaction broadcast successfully.") reactor.stop() diff --git a/jmclient/jmclient/taker.py b/jmclient/jmclient/taker.py index 25d5d1a..3650923 100644 --- a/jmclient/jmclient/taker.py +++ b/jmclient/jmclient/taker.py @@ -499,7 +499,8 @@ class Taker(object): self.outputs.append({'address': self.coinjoin_address(), 'value': self.cjamount}) self.latest_tx = btc.make_shuffled_tx(self.utxo_tx, self.outputs) - jlog.info('obtained tx\n' + btc.hrt(self.latest_tx)) + jlog.info('obtained tx\n' + btc.human_readable_transaction( + self.latest_tx)) for index, ins in enumerate(self.latest_tx.vin): utxo = (ins.prevout.hash[::-1], ins.prevout.n) @@ -1008,7 +1009,8 @@ class P2EPTaker(Taker): # contains only those. tx = btc.make_shuffled_tx(self.input_utxos, self.outputs, version=2, locktime=compute_tx_locktime()) - jlog.info('Created proposed fallback tx:\n' + btc.hrt(tx)) + jlog.info('Created proposed fallback tx:\n' + \ + btc.human_readable_transaction(tx)) # We now sign as a courtesy, because if we disappear the recipient # can still claim his coins with this. # sign our inputs before transfer diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index f579356..1752385 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -11,7 +11,8 @@ from .schedule import human_readable_schedule_entry, tweak_tumble_schedule,\ from .wallet import BaseWallet, estimate_tx_fee, compute_tx_locktime, \ FidelityBondMixin from jmbitcoin import make_shuffled_tx, amount_to_str, mk_burn_script,\ - PartiallySignedTransaction, CMutableTxOut, hrt, Hash160 + PartiallySignedTransaction, CMutableTxOut,\ + human_readable_transaction, Hash160 from jmbase.support import EXIT_SUCCESS log = get_log() @@ -165,7 +166,7 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, return False new_psbt_signed = PartiallySignedTransaction.deserialize(serialized_psbt) print("Completed PSBT created: ") - print(wallet_service.hr_psbt(new_psbt_signed)) + print(wallet_service.human_readable_psbt(new_psbt_signed)) return new_psbt_signed else: success, msg = wallet_service.sign_tx(tx, inscripts) @@ -173,7 +174,7 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, log.error("Failed to sign transaction, quitting. Error msg: " + msg) return log.info("Got signed transaction:\n") - log.info(hrt(tx)) + log.info(human_readable_transaction(tx)) actual_amount = amount if amount != 0 else total_inputs_val - fee_est log.info("Sends: " + amount_to_str(actual_amount) + " to destination: " + destination) if not answeryes: @@ -182,8 +183,8 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, log.info("You chose not to broadcast the transaction, quitting.") return False else: - accepted = accept_callback(hrt(tx), destination, actual_amount, - fee_est) + accepted = accept_callback(human_readable_transaction(tx), + destination, actual_amount, fee_est) if not accepted: return False jm_single().bc_interface.pushtx(tx.serialize()) diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index 2797e3c..c013a89 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -1007,13 +1007,6 @@ class PSBTWalletMixin(object): def __init__(self, storage, **kwargs): super(PSBTWalletMixin, self).__init__(storage, **kwargs) - @staticmethod - def get_fee_from_psbt(in_psbt): - assert isinstance(in_psbt, btc.PartiallySignedTransaction) - spent = sum(in_psbt.get_input_amounts()) - paid = sum((x.nValue for x in in_psbt.unsigned_tx.vout)) - return spent - paid - def is_input_finalized(self, psbt_input): """ This should be a convenience method in python-bitcointx. However note: this is not a static method and tacitly @@ -1032,7 +1025,7 @@ class PSBTWalletMixin(object): return True @staticmethod - def hr_psbt(in_psbt): + def human_readable_psbt(in_psbt): """ Returns a jsonified indented string with all relevant information, in human readable form, contained in a PSBT. Warning: the output can be very verbose in certain cases. @@ -1054,17 +1047,20 @@ class PSBTWalletMixin(object): if in_psbt.unknown_fields: outdict["unknown-fields"] = str(in_psbt.unknown_fields) - outdict["unsigned-tx"] = btc.hrt(in_psbt.unsigned_tx, jsonified=False) + outdict["unsigned-tx"] = btc.human_readable_transaction( + in_psbt.unsigned_tx, jsonified=False) outdict["psbt-inputs"] = [] for inp in in_psbt.inputs: - outdict["psbt-inputs"].append(PSBTWalletMixin.hr_psbt_in(inp)) + outdict["psbt-inputs"].append( + PSBTWalletMixin.human_readable_psbt_in(inp)) outdict["psbt-outputs"] = [] for out in in_psbt.outputs: - outdict["psbt-outputs"].append(PSBTWalletMixin.hr_psbt_out(out)) + outdict["psbt-outputs"].append( + PSBTWalletMixin.human_readable_psbt_out(out)) return json.dumps(outdict, indent=4) @staticmethod - def hr_psbt_in(psbt_input): + def human_readable_psbt_in(psbt_input): """ Returns a dict containing human readable information about a bitcointx.core.psbt.PSBT_Input object. """ @@ -1074,7 +1070,7 @@ class PSBTWalletMixin(object): outdict["input-index"] = psbt_input.index if psbt_input.utxo: if isinstance(psbt_input.utxo, btc.CTxOut): - outdict["utxo"] = btc.hrout(psbt_input.utxo) + outdict["utxo"] = btc.human_readable_output(psbt_input.utxo) elif isinstance(psbt_input.utxo, btc.CTransaction): # human readable full transaction is *too* verbose: outdict["utxo"] = bintohex(psbt_input.utxo.serialize()) @@ -1116,7 +1112,7 @@ class PSBTWalletMixin(object): return outdict @staticmethod - def hr_psbt_out(psbt_output): + def human_readable_psbt_out(psbt_output): """ Returns a dict containing human readable information about a PSBT_Output object. """ @@ -1175,13 +1171,15 @@ class PSBTWalletMixin(object): continue if isinstance(spent_outs[i], (btc.CTransaction, btc.CTxOut)): # note that we trust the caller to choose Tx vs TxOut as according - # to non-witness/witness: - txinput.utxo = spent_outs[i] + # to non-witness/witness. Note also that for now this mixin does + # not attempt to provide unsigned-tx(second argument) for witness + # case. + txinput.set_utxo(spent_outs[i], None) else: assert False, "invalid spent output type passed into PSBT creator" # we now insert redeemscripts where that is possible and necessary: for i, txinput in enumerate(new_psbt.inputs): - if isinstance(txinput.utxo, btc.CTxOut): + if isinstance(txinput.witness_utxo, btc.CTxOut): # witness if txinput.utxo.scriptPubKey.is_witness_scriptpubkey(): # nothing needs inserting; the scriptSig is empty. @@ -1228,7 +1226,7 @@ class PSBTWalletMixin(object): # then overwriting it is harmless (preimage resistance). if isinstance(self, SegwitLegacyWallet): for i, txinput in enumerate(new_psbt.inputs): - tu = txinput.utxo + tu = txinput.witness_utxo if isinstance(tu, btc.CTxOut): # witness if tu.scriptPubKey.is_witness_scriptpubkey(): diff --git a/jmclient/test/commontest.py b/jmclient/test/commontest.py index e06eaf0..f4c849c 100644 --- a/jmclient/test/commontest.py +++ b/jmclient/test/commontest.py @@ -6,7 +6,7 @@ import binascii import random from decimal import Decimal -from jmbase import (get_log, hextobin, dictchanger) +from jmbase import (get_log, hextobin, bintohex, dictchanger) from jmclient import ( jm_single, open_test_wallet_maybe, estimate_tx_fee, @@ -120,8 +120,10 @@ class DummyBlockchainInterface(BlockchainInterface): def create_wallet_for_sync(wallet_structure, a, **kwargs): #We need a distinct seed for each run so as not to step over each other; - #make it through a deterministic hash - seedh = btc.b2x(btc.Hash("".join([str(x) for x in a]).encode("utf-8")))[:32] + #make it through a deterministic hash of all parameters including optionals. + preimage = "".join([str(x) for x in a] + [str(y) for y in kwargs.values()]).encode("utf-8") + print("using preimage: ", preimage) + seedh = bintohex(btc.Hash(preimage))[:32] return make_wallets( 1, [wallet_structure], fixed_seeds=[seedh], **kwargs)[0]['wallet'] @@ -191,8 +193,9 @@ def make_wallets(n, if len(wallet_structures) != n: raise Exception("Number of wallets doesn't match wallet structures") if not fixed_seeds: - seeds = chunks(binascii.hexlify(os.urandom(BIP32Wallet.ENTROPY_BYTES * n)).decode('ascii'), - BIP32Wallet.ENTROPY_BYTES * 2) + seeds = chunks(bintohex(os.urandom( + BIP32Wallet.ENTROPY_BYTES * n)), + BIP32Wallet.ENTROPY_BYTES * 2) else: seeds = fixed_seeds wallets = {} diff --git a/jmclient/test/test_configure.py b/jmclient/test/test_configure.py index cb488d2..2c5eb03 100644 --- a/jmclient/test/test_configure.py +++ b/jmclient/test/test_configure.py @@ -3,8 +3,8 @@ import pytest import struct from jmclient import load_test_config, jm_single, get_irc_mchannels -from jmclient.configure import (get_config_irc_channel, get_p2sh_vbyte, - get_p2pk_vbyte, get_blockchain_interface_instance) +from jmclient.configure import (get_config_irc_channel, + get_blockchain_interface_instance) def test_attribute_dict(): @@ -35,12 +35,6 @@ def test_config_get_irc_channel(): load_test_config() -def test_net_byte(): - load_test_config() - assert struct.unpack(b'B', get_p2pk_vbyte())[0] == 0x6f - assert struct.unpack(b'B', get_p2sh_vbyte())[0] == 196 - - def test_blockchain_sources(): load_test_config() for src in ["electrum", "dummy"]: diff --git a/jmclient/test/test_core_nohistory_sync.py b/jmclient/test/test_core_nohistory_sync.py index ded8eb1..7c8c083 100644 --- a/jmclient/test/test_core_nohistory_sync.py +++ b/jmclient/test/test_core_nohistory_sync.py @@ -7,29 +7,35 @@ from commontest import create_wallet_for_sync import pytest from jmbase import get_log -from jmclient import load_test_config +from jmclient import (load_test_config, SegwitLegacyWallet, + SegwitWallet, jm_single) +from jmbitcoin import select_chain_params log = get_log() def test_fast_sync_unavailable(setup_sync): - load_test_config(bs="bitcoin-rpc-no-history") wallet_service = create_wallet_for_sync([0, 0, 0, 0, 0], ['test_fast_sync_unavailable']) with pytest.raises(RuntimeError) as e_info: wallet_service.sync_wallet(fast=True) -@pytest.mark.parametrize('internal', (False, True)) -def test_sync(setup_sync, internal): - load_test_config(bs="bitcoin-rpc-no-history") +@pytest.mark.parametrize('internal, wallet_cls', [(False, SegwitLegacyWallet), + (True, SegwitLegacyWallet), + (False, SegwitWallet), + (True, SegwitWallet)]) +def test_sync(setup_sync, internal, wallet_cls): used_count = [1, 3, 6, 2, 23] wallet_service = create_wallet_for_sync(used_count, ['test_sync'], - populate_internal=internal) + populate_internal=internal, wallet_cls=wallet_cls) ##the gap limit should be not zero before sync assert wallet_service.gap_limit > 0 for md in range(len(used_count)): ##obtaining an address should be possible without error before sync wallet_service.get_new_script(md, internal) + # TODO bci should probably not store this state globally, + # in case syncing is needed for multiple wallets (as in this test): + jm_single().bc_interface.import_addresses_call_count = 0 wallet_service.sync_wallet(fast=False) for md in range(len(used_count)): @@ -45,4 +51,10 @@ def test_sync(setup_sync, internal): @pytest.fixture(scope='module') def setup_sync(): - pass + load_test_config(bs="bitcoin-rpc-no-history") + # a special case needed for the bitcoin core + # no history interface: it does not use + # 'blockchain_source' to distinguish regtest, + # so it must be set specifically for the test + # here: + select_chain_params("bitcoin/regtest") diff --git a/jmclient/test/test_psbt_wallet.py b/jmclient/test/test_psbt_wallet.py index 18f88d7..5c59e00 100644 --- a/jmclient/test/test_psbt_wallet.py +++ b/jmclient/test/test_psbt_wallet.py @@ -131,7 +131,7 @@ def test_create_psbt_and_sign(setup_psbt_wallet, unowned_utxo, wallet_cls): newpsbt.inputs[-1].redeem_script = redeem_script print(bintohex(newpsbt.serialize())) print("human readable: ") - print(wallet_service.hr_psbt(newpsbt)) + print(wallet_service.human_readable_psbt(newpsbt)) # we cannot compare with a fixed expected result due to wallet randomization, but we can # check psbt structure: expected_inputs_length = 3 if unowned_utxo else 2 @@ -213,8 +213,8 @@ def test_payjoin_workflow(setup_psbt_wallet, payment_amt, wallet_cls_sender, info_callback=dummy_info_callback, with_final_psbt=True) - print("Initial payment PSBT created:\n{}".format(wallet_s.hr_psbt( - payment_psbt))) + print("Initial payment PSBT created:\n{}".format( + wallet_s.human_readable_psbt(payment_psbt))) # ensure that the payemnt amount is what was intended: out_amts = [x.nValue for x in payment_psbt.unsigned_tx.vout] # NOTE this would have to change for more than 2 outputs: @@ -278,7 +278,7 @@ def test_payjoin_workflow(setup_psbt_wallet, payment_amt, wallet_cls_sender, version=payment_psbt.unsigned_tx.nVersion, locktime=payment_psbt.unsigned_tx.nLockTime) print("we created this unsigned tx: ") - print(bitcoin.hrt(unsigned_payjoin_tx)) + print(bitcoin.human_readable_transaction(unsigned_payjoin_tx)) # to create the PSBT we need the spent_outs for each input, # in the right order: spent_outs = [] @@ -306,7 +306,7 @@ def test_payjoin_workflow(setup_psbt_wallet, payment_amt, wallet_cls_sender, r_payjoin_psbt = wallet_r.create_psbt_from_tx(unsigned_payjoin_tx, spent_outs=spent_outs) print("Receiver created payjoin PSBT:\n{}".format( - wallet_r.hr_psbt(r_payjoin_psbt))) + wallet_r.human_readable_psbt(r_payjoin_psbt))) signresultandpsbt, err = wallet_r.sign_psbt(r_payjoin_psbt.serialize(), with_sign_result=True) @@ -316,7 +316,7 @@ def test_payjoin_workflow(setup_psbt_wallet, payment_amt, wallet_cls_sender, assert not signresult.is_final print("Receiver signing successful. Payjoin PSBT is now:\n{}".format( - wallet_r.hr_psbt(receiver_signed_psbt))) + wallet_r.human_readable_psbt(receiver_signed_psbt))) # *** STEP 3 *** # ************** @@ -328,7 +328,7 @@ def test_payjoin_workflow(setup_psbt_wallet, payment_amt, wallet_cls_sender, assert not err, err signresult, sender_signed_psbt = signresultandpsbt print("Sender's final signed PSBT is:\n{}".format( - wallet_s.hr_psbt(sender_signed_psbt))) + wallet_s.human_readable_psbt(sender_signed_psbt))) assert signresult.is_final # broadcast the tx @@ -367,7 +367,7 @@ hr_test_vectors = { def test_hr_psbt(setup_psbt_wallet): bitcoin.select_chain_params("bitcoin") for k, v in hr_test_vectors.items(): - print(PSBTWalletMixin.hr_psbt( + print(PSBTWalletMixin.human_readable_psbt( bitcoin.PartiallySignedTransaction.from_binary(hextobin(v)))) bitcoin.select_chain_params("bitcoin/regtest") diff --git a/jmclient/test/test_snicker.py b/jmclient/test/test_snicker.py index 7d81e33..7eeca55 100644 --- a/jmclient/test/test_snicker.py +++ b/jmclient/test/test_snicker.py @@ -2,14 +2,16 @@ '''Test of SNICKER functionality using Joinmarket wallets as defined in jmclient.wallet.''' -from commontest import make_wallets, dummy_accept_callback, dummy_info_callback +import pytest +import os +from commontest import make_wallets, dummy_accept_callback, dummy_info_callback import jmbitcoin as btc -import pytest from jmbase import get_log, bintohex from jmclient import (load_test_config, estimate_tx_fee, SNICKERReceiver, direct_send) +TEST_PROPOSALS_FILE = "test_proposals.txt" log = get_log() @pytest.mark.parametrize( @@ -45,7 +47,7 @@ def test_snicker_e2e(setup_snicker, nw, wallet_structures, assert tx, "Failed to spend from receiver wallet" print("Parent transaction OK. It was: ") - print(btc.hrt(tx)) + print(btc.human_readable_transaction(tx)) wallet_r.process_new_tx(tx) # we must identify the receiver's output we're going to use; # it can be destination or change, that's up to the proposer @@ -97,10 +99,10 @@ def test_snicker_e2e(setup_snicker, nw, wallet_structures, prop_utxo['script'], change_spk, version_byte=1) + b"," + bintohex(p).encode('utf-8')) - with open("test_proposals.txt", "wb") as f: + with open(TEST_PROPOSALS_FILE, "wb") as f: f.write(b"\n".join(encrypted_proposals)) sR = SNICKERReceiver(wallet_r) - sR.proposals_source = "test_proposals.txt" # avoid clashing with mainnet + sR.proposals_source = TEST_PROPOSALS_FILE # avoid clashing with mainnet sR.poll_for_proposals() assert len(sR.successful_txs) == 1 wallet_r.process_new_tx(sR.successful_txs[0]) @@ -110,5 +112,9 @@ def test_snicker_e2e(setup_snicker, nw, wallet_structures, assert receiver_end_bal == receiver_start_bal + net_transfer @pytest.fixture(scope="module") -def setup_snicker(): +def setup_snicker(request): load_test_config() + def teardown(): + if os.path.exists(TEST_PROPOSALS_FILE): + os.remove(TEST_PROPOSALS_FILE) + request.addfinalizer(teardown) diff --git a/test/payjoinserver.py b/test/payjoinserver.py index cda5e5c..b18c99a 100644 --- a/test/payjoinserver.py +++ b/test/payjoinserver.py @@ -98,7 +98,7 @@ class PayjoinServer(Resource): version=payment_psbt.unsigned_tx.nVersion, locktime=payment_psbt.unsigned_tx.nLockTime) print("we created this unsigned tx: ") - print(btc.hrt(unsigned_payjoin_tx)) + print(btc.human_readable_transaction(unsigned_payjoin_tx)) # to create the PSBT we need the spent_outs for each input, # in the right order: spent_outs = [] @@ -126,7 +126,7 @@ class PayjoinServer(Resource): r_payjoin_psbt = self.wallet_service.create_psbt_from_tx(unsigned_payjoin_tx, spent_outs=spent_outs) print("Receiver created payjoin PSBT:\n{}".format( - self.wallet_service.hr_psbt(r_payjoin_psbt))) + self.wallet_service.human_readable_psbt(r_payjoin_psbt))) signresultandpsbt, err = self.wallet_service.sign_psbt(r_payjoin_psbt.serialize(), with_sign_result=True) @@ -136,7 +136,7 @@ class PayjoinServer(Resource): assert not signresult.is_final print("Receiver signing successful. Payjoin PSBT is now:\n{}".format( - self.wallet_service.hr_psbt(receiver_signed_psbt))) + self.wallet_service.human_readable_psbt(receiver_signed_psbt))) content = receiver_signed_psbt.to_base64() request.setHeader(b"content-length", ("%d" % len(content)).encode("ascii")) return content.encode("ascii")