Browse Source

Implement no-history synchronization

No-history is a method for synchronizing a wallet by scanning the UTXO
set. It can be useful for checking whether seed phrase backups have
money on them before committing the time and effort required to
rescanning the blockchain. No-history sync is compatible with pruning.
The sync method cannot tell which empty addresses have been used, so
cannot guarentee avoidance of address reuse. For this reason no-history
sync disables wallet address generation and can only be used with
wallet-tool and for sending transactions without change addresses.
master
chris-belcher 6 years ago
parent
commit
cbf69c6a60
No known key found for this signature in database
GPG Key ID: EF734EA677F31129
  1. 12
      docs/release-notes/release-notes-nohistory-sync.md
  2. 119
      jmclient/jmclient/blockchaininterface.py
  3. 14
      jmclient/jmclient/configure.py
  4. 9
      jmclient/jmclient/wallet.py
  5. 84
      jmclient/jmclient/wallet_service.py
  6. 36
      jmclient/jmclient/wallet_utils.py

12
docs/release-notes/release-notes-nohistory-sync.md

@ -0,0 +1,12 @@
Notable changes
===============
### No-history wallet synchronization
The no-history synchronization method is enabled by setting `blockchain_source = bitcoin-rpc-no-history` in the `joinmarket.cfg` file.
The method can be used to import a seed phrase to see whether it has any money on it within just 5-10 minutes. No-history sync doesn't require a long blockchain rescan, although it needs a full node which can be pruned.
No-history sync works by scanning the full node's UTXO set. The downside is that it cannot find the history but only the current unspent balance, so it cannot avoid address reuse. Therefore when using no-history synchronization the wallet cannot generate new addresses. Any found money can only be spent by fully-sweeping the funds but not partially spending them which requires a change address. When using the method make sure to increase the gap limit to a large amount to cover all the possible bitcoin addresses where coins might be.
The mode does not work with the Joinmarket-Qt GUI application but might do in future.

119
jmclient/jmclient/blockchaininterface.py

@ -5,6 +5,7 @@ from builtins import * # noqa: F401
import abc
import random
import sys
import time
from decimal import Decimal
from twisted.internet import reactor, task
@ -12,7 +13,7 @@ import jmbitcoin as btc
from jmclient.jsonrpc import JsonRpcConnectionError, JsonRpcError
from jmclient.configure import jm_single
from jmbase.support import get_log, jmprint, EXIT_SUCCESS, EXIT_FAILURE
from jmbase.support import get_log, jmprint, EXIT_FAILURE
# an inaccessible blockheight; consider rewriting in 1900 years
@ -57,6 +58,11 @@ class BlockchainInterface(object):
required for inclusion in the next N blocks.
'''
@abc.abstractmethod
def import_addresses_if_needed(self, addresses, wallet_name):
"""import addresses to the underlying blockchain interface if needed
returns True if the sync call needs to do a system exit"""
def fee_per_kb_has_been_manually_set(self, N):
'''if the 'block' target is higher than 1000, interpret it
as manually set fee/Kb.
@ -66,7 +72,6 @@ class BlockchainInterface(object):
else:
return False
class ElectrumWalletInterface(BlockchainInterface): #pragma: no cover
"""A pseudo-blockchain interface using the existing
Electrum server connection in an Electrum wallet.
@ -167,7 +172,9 @@ class BitcoinCoreInterface(BlockchainInterface):
actualNet = blockchainInfo['chain']
netmap = {'main': 'mainnet', 'test': 'testnet', 'regtest': 'regtest'}
if netmap[actualNet] != network:
if netmap[actualNet] != network and \
(not (actualNet == "regtest" and network == "testnet")):
#special case of regtest and testnet having the same addr format
raise Exception('wrong network configured')
def get_block(self, blockheight):
@ -182,7 +189,8 @@ class BitcoinCoreInterface(BlockchainInterface):
def rpc(self, method, args):
if method not in ['importaddress', 'walletpassphrase', 'getaccount',
'gettransaction', 'getrawtransaction', 'gettxout',
'importmulti', 'listtransactions', 'getblockcount']:
'importmulti', 'listtransactions', 'getblockcount',
'scantxoutset']:
log.debug('rpc: ' + method + " " + str(args))
res = self.jsonRpc.call(method, args)
return res
@ -230,25 +238,20 @@ class BitcoinCoreInterface(BlockchainInterface):
jmprint(fatal_msg, "important")
sys.exit(EXIT_FAILURE)
def add_watchonly_addresses(self, addr_list, wallet_name, restart_cb=None):
"""For backwards compatibility, this fn name is preserved
as the case where we quit the program if a rescan is required;
but in some cases a rescan is not required (if the address is known
to be new/unused). For that case use import_addresses instead.
"""
self.import_addresses(addr_list, wallet_name)
if jm_single().config.get("BLOCKCHAIN",
"blockchain_source") != 'regtest': #pragma: no cover
#Exit conditions cannot be included in tests
restart_msg = ("restart Bitcoin Core with -rescan or use "
"`bitcoin-cli rescanblockchain` if you're "
"recovering an existing wallet from backup seed\n"
"Otherwise just restart this joinmarket application.")
if restart_cb:
restart_cb(restart_msg)
def import_addresses_if_needed(self, addresses, wallet_name):
try:
imported_addresses = set(self.rpc('getaddressesbyaccount',
[wallet_name]))
except JsonRpcError:
if wallet_name in self.rpc('listlabels', []):
imported_addresses = set(self.rpc('getaddressesbylabel',
[wallet_name]).keys())
else:
jmprint(restart_msg, "important")
sys.exit(EXIT_SUCCESS)
imported_addresses = set()
import_needed = not addresses.issubset(imported_addresses)
if import_needed:
self.import_addresses(addresses - imported_addresses, wallet_name)
return import_needed
def _yield_transactions(self, wallet_name):
batch_size = 1000
@ -387,6 +390,78 @@ class BitcoinCoreInterface(BlockchainInterface):
return 10000
return int(Decimal(1e8) * Decimal(estimate))
class BitcoinCoreNoHistoryInterface(BitcoinCoreInterface):
def __init__(self, jsonRpc, network):
super(BitcoinCoreNoHistoryInterface, self).__init__(jsonRpc, network)
self.import_addresses_call_count = 0
self.wallet_name = None
self.scan_result = None
def import_addresses_if_needed(self, addresses, wallet_name):
self.import_addresses_call_count += 1
if self.import_addresses_call_count == 1:
self.wallet_name = wallet_name
addr_list = ["addr(" + a + ")" for a in addresses]
log.debug("Starting scan of UTXO set")
st = time.time()
try:
self.rpc("scantxoutset", ["abort", []])
self.scan_result = self.rpc("scantxoutset", ["start",
addr_list])
except JsonRpcError as e:
raise RuntimeError("Bitcoin Core 0.17.0 or higher required "
+ "for no-history sync (" + repr(e) + ")")
et = time.time()
log.debug("UTXO set scan took " + str(et - st) + "sec")
elif self.import_addresses_call_count > 4:
#called twice for the first call of sync_addresses(), then two
# more times for the second call. the second call happens because
# sync_addresses() re-runs in order to have gap_limit new addresses
assert False
return False
def _get_addr_from_desc(self, desc_str):
#example
#'desc': 'addr(2MvAfRVvRAeBS18NT7mKVc1gFim169GkFC5)#h5yn9eq4',
assert desc_str.startswith("addr(")
return desc_str[5:desc_str.find(")")]
def _yield_transactions(self, wallet_name):
for u in self.scan_result["unspents"]:
tx = {"category": "receive", "address":
self._get_addr_from_desc(u["desc"])}
yield tx
def list_transactions(self, num):
return []
def rpc(self, method, args):
if method == "listaddressgroupings":
raise RuntimeError("default sync not supported by bitcoin-rpc-nohistory, use --recoversync")
elif method == "listunspent":
minconf = 0 if len(args) < 1 else args[0]
maxconf = 9999999 if len(args) < 2 else args[1]
return [{
"address": self._get_addr_from_desc(u["desc"]),
"label": self.wallet_name,
"height": u["height"],
"txid": u["txid"],
"vout": u["vout"],
"scriptPubKey": u["scriptPubKey"],
"amount": u["amount"]
} for u in self.scan_result["unspents"]]
else:
return super(BitcoinCoreNoHistoryInterface, self).rpc(method, args)
def set_wallet_no_history(self, wallet):
#make wallet-tool not display any new addresses
#because no-history cant tell if an address is used and empty
#so this is necessary to avoid address reuse
wallet.gap_limit = 0
#disable generating change addresses, also because cant guarantee
# avoidance of address reuse
wallet.disable_new_scripts = True
# class for regtest chain access
# running on local daemon. Only

14
jmclient/jmclient/configure.py

@ -107,7 +107,8 @@ daemon_host = localhost
use_ssl = false
[BLOCKCHAIN]
#options: bitcoin-rpc, regtest
#options: bitcoin-rpc, regtest, bitcoin-rpc-no-history
# when using bitcoin-rpc-no-history remember to increase the gap limit to scan for more addresses, try -g 5000
blockchain_source = bitcoin-rpc
network = mainnet
rpc_host = localhost
@ -516,12 +517,13 @@ def get_blockchain_interface_instance(_config):
# todo: refactor joinmarket module to get rid of loops
# importing here is necessary to avoid import loops
from jmclient.blockchaininterface import BitcoinCoreInterface, \
RegtestBitcoinCoreInterface, ElectrumWalletInterface
RegtestBitcoinCoreInterface, ElectrumWalletInterface, \
BitcoinCoreNoHistoryInterface
from jmclient.electruminterface import ElectrumInterface
source = _config.get("BLOCKCHAIN", "blockchain_source")
network = get_network()
testnet = network == 'testnet'
if source in ('bitcoin-rpc', 'regtest'):
if source in ('bitcoin-rpc', 'regtest', 'bitcoin-rpc-no-history'):
rpc_host = _config.get("BLOCKCHAIN", "rpc_host")
rpc_port = _config.get("BLOCKCHAIN", "rpc_port")
rpc_user, rpc_password = get_bitcoin_rpc_credentials(_config)
@ -530,8 +532,12 @@ def get_blockchain_interface_instance(_config):
rpc_wallet_file)
if source == 'bitcoin-rpc': #pragma: no cover
bc_interface = BitcoinCoreInterface(rpc, network)
else:
elif source == 'regtest':
bc_interface = RegtestBitcoinCoreInterface(rpc)
elif source == "bitcoin-rpc-no-history":
bc_interface = BitcoinCoreNoHistoryInterface(rpc, network)
else:
assert 0
elif source == 'electrum':
bc_interface = ElectrumWalletInterface(testnet)
elif source == 'electrum-server':

9
jmclient/jmclient/wallet.py

@ -1283,6 +1283,7 @@ class BIP32Wallet(BaseWallet):
self.get_bip32_priv_export(0, 0).encode('ascii')).digest())\
.digest()[:3]
self._populate_script_map()
self.disable_new_scripts = False
@classmethod
def initialize(cls, storage, network, max_mixdepth=2, timestamp=None,
@ -1372,7 +1373,7 @@ class BIP32Wallet(BaseWallet):
current_index = self._index_cache[md][int_type]
if index == current_index:
return self.get_new_script(md, int_type)
return self.get_new_script_override_disable(md, int_type)
priv, engine = self._get_priv_from_path(path)
script = engine.privkey_to_script(priv)
@ -1454,6 +1455,12 @@ class BIP32Wallet(BaseWallet):
return path[0] == self._key_ident
def get_new_script(self, mixdepth, internal):
if self.disable_new_scripts:
raise RuntimeError("Obtaining new wallet addresses "
+ "disabled, due to nohistory mode")
return self.get_new_script_override_disable(mixdepth, internal)
def get_new_script_override_disable(self, mixdepth, internal):
# This is called by get_script_path and calls back there. We need to
# ensure all conditions match to avoid endless recursion.
int_type = self._get_internal_type(internal)

84
jmclient/jmclient/wallet_service.py

@ -6,6 +6,7 @@ import collections
import time
import ast
import binascii
import sys
from decimal import Decimal
from copy import deepcopy
from twisted.internet import reactor
@ -14,8 +15,9 @@ from twisted.application.service import Service
from numbers import Integral
from jmclient.configure import jm_single, get_log
from jmclient.output import fmt_tx_data
from jmclient.jsonrpc import JsonRpcError
from jmclient.blockchaininterface import INF_HEIGHT
from jmclient.blockchaininterface import (INF_HEIGHT, BitcoinCoreInterface,
BitcoinCoreNoHistoryInterface)
from jmbase.support import jmprint, EXIT_SUCCESS
"""Wallet service
The purpose of this independent service is to allow
@ -333,6 +335,8 @@ class WalletService(Service):
# before startup
self.old_txs = [x['txid'] for x in self.bci.list_transactions(100)
if "txid" in x]
if isinstance(self.bci, BitcoinCoreNoHistoryInterface):
self.bci.set_wallet_no_history(self.wallet)
return self.synced
def resync_wallet(self, fast=True):
@ -447,6 +451,24 @@ class WalletService(Service):
self.restart_callback)
self.synced = True
def display_rescan_message_and_system_exit(self, restart_cb):
#TODO using system exit here should be avoided as it makes the code
# harder to understand and reason about
#theres also a sys.exit() in BitcoinCoreInterface.import_addresses()
#perhaps have sys.exit() placed inside the restart_cb that only
# CLI scripts will use
if self.bci.__class__ == BitcoinCoreInterface:
#Exit conditions cannot be included in tests
restart_msg = ("restart Bitcoin Core with -rescan or use "
"`bitcoin-cli rescanblockchain` if you're "
"recovering an existing wallet from backup seed\n"
"Otherwise just restart this joinmarket application.")
if restart_cb:
restart_cb(restart_msg)
else:
jmprint(restart_msg, "important")
sys.exit(EXIT_SUCCESS)
def sync_addresses(self):
""" Triggered by use of --recoversync option in scripts,
attempts a full scan of the blockchain without assuming
@ -456,19 +478,11 @@ class WalletService(Service):
jlog.debug("requesting detailed wallet history")
wallet_name = self.get_wallet_name()
addresses, saved_indices = self.collect_addresses_init()
try:
imported_addresses = set(self.bci.rpc('getaddressesbyaccount',
[wallet_name]))
except JsonRpcError:
if wallet_name in self.bci.rpc('listlabels', []):
imported_addresses = set(self.bci.rpc('getaddressesbylabel',
[wallet_name]).keys())
else:
imported_addresses = set()
if not addresses.issubset(imported_addresses):
self.bci.add_watchonly_addresses(addresses - imported_addresses,
wallet_name, self.restart_callback)
import_needed = self.bci.import_addresses_if_needed(addresses,
wallet_name)
if import_needed:
self.display_rescan_message_and_system_exit(self.restart_callback)
return
used_addresses_gen = (tx['address']
@ -480,13 +494,12 @@ class WalletService(Service):
self.rewind_wallet_indices(used_indices, saved_indices)
new_addresses = self.collect_addresses_gap()
if not new_addresses.issubset(imported_addresses):
jlog.debug("Syncing iteration finished, additional step required")
self.bci.add_watchonly_addresses(new_addresses - imported_addresses,
wallet_name, self.restart_callback)
if self.bci.import_addresses_if_needed(new_addresses, wallet_name):
jlog.debug("Syncing iteration finished, additional step required (more address import required)")
self.synced = False
self.display_rescan_message_and_system_exit(self.restart_callback)
elif gap_limit_used:
jlog.debug("Syncing iteration finished, additional step required")
jlog.debug("Syncing iteration finished, additional step required (gap limit used)")
self.synced = False
else:
jlog.debug("Wallet successfully synced")
@ -513,14 +526,30 @@ class WalletService(Service):
our_unspent_list = [x for x in unspent_list if (
self.bci.is_address_labeled(x, wallet_name) or
self.bci.is_address_labeled(x, self.EXTERNAL_WALLET_LABEL))]
for u in our_unspent_list:
if not self.is_known_addr(u['address']):
for utxo in our_unspent_list:
if not self.is_known_addr(utxo['address']):
continue
self._add_unspent_utxo(u, current_blockheight)
# The result of bitcoin core's listunspent RPC call does not have
# a "height" field, only "confirmations".
# But the result of scantxoutset used in no-history sync does
# have "height".
if "height" in utxo:
height = utxo["height"]
else:
height = None
# wallet's utxo database needs to store an absolute rather
# than relative height measure:
confs = int(utxo['confirmations'])
if confs < 0:
jlog.warning("Utxo not added, has a conflict: " + str(utxo))
continue
if confs >= 1:
height = current_blockheight - confs + 1
self._add_unspent_txo(utxo, height)
et = time.time()
jlog.debug('bitcoind sync_unspent took ' + str((et - st)) + 'sec')
def _add_unspent_utxo(self, utxo, current_blockheight):
def _add_unspent_txo(self, utxo, height):
"""
Add a UTXO as returned by rpc's listunspent call to the wallet.
@ -532,15 +561,6 @@ class WalletService(Service):
txid = binascii.unhexlify(utxo['txid'])
script = binascii.unhexlify(utxo['scriptPubKey'])
value = int(Decimal(str(utxo['amount'])) * Decimal('1e8'))
confs = int(utxo['confirmations'])
# wallet's utxo database needs to store an absolute rather
# than relative height measure:
height = None
if confs < 0:
jlog.warning("Utxo not added, has a conflict: " + str(utxo))
return
if confs >=1 :
height = current_blockheight - confs + 1
self.add_utxo(txid, int(utxo['vout']), script, value, height)

36
jmclient/jmclient/wallet_utils.py

@ -386,7 +386,7 @@ def wallet_showutxos(wallet, showprivkey):
return json.dumps(unsp, indent=4)
def wallet_display(wallet_service, gaplimit, showprivkey, displayall=False,
def wallet_display(wallet_service, showprivkey, displayall=False,
serialized=True, summarized=False):
"""build the walletview object,
then return its serialization directly if serialized,
@ -399,16 +399,22 @@ def wallet_display(wallet_service, gaplimit, showprivkey, displayall=False,
if addr_path != utxodata['path']:
continue
addr_balance += utxodata['value']
is_coinjoin, cj_amount, cj_n = \
get_tx_info(binascii.hexlify(utxo[0]).decode('ascii'))[:3]
if is_coinjoin and utxodata['value'] == cj_amount:
status.append('cj-out')
elif is_coinjoin:
status.append('change-out')
elif is_internal:
status.append('non-cj-change')
else:
status.append('deposit')
#TODO it is a failure of abstraction here that
# the bitcoin core interface is used directly
#the function should either be removed or added to bci
#or possibly add some kind of `gettransaction` function
# to bci
if jm_single().bc_interface.__class__ == BitcoinCoreInterface:
is_coinjoin, cj_amount, cj_n = \
get_tx_info(binascii.hexlify(utxo[0]).decode('ascii'))[:3]
if is_coinjoin and utxodata['value'] == cj_amount:
status.append('cj-out')
elif is_coinjoin:
status.append('change-out')
elif is_internal:
status.append('non-cj-change')
else:
status.append('deposit')
out_status = 'new' if is_new else 'used'
if len(status) > 1:
@ -433,7 +439,7 @@ def wallet_display(wallet_service, gaplimit, showprivkey, displayall=False,
xpub_key = ""
unused_index = wallet_service.get_next_unused_index(m, forchange)
for k in range(unused_index + gaplimit):
for k in range(unused_index + wallet_service.gap_limit):
path = wallet_service.get_path(m, forchange, k)
addr = wallet_service.get_addr_path(path)
balance, used = get_addr_status(
@ -1254,12 +1260,12 @@ def wallet_tool_main(wallet_root_path):
#Now the wallet/data is prepared, execute the script according to the method
if method == "display":
return wallet_display(wallet_service, options.gaplimit, options.showprivkey)
return wallet_display(wallet_service, options.showprivkey)
elif method == "displayall":
return wallet_display(wallet_service, options.gaplimit, options.showprivkey,
return wallet_display(wallet_service, options.showprivkey,
displayall=True)
elif method == "summary":
return wallet_display(wallet_service, options.gaplimit, options.showprivkey, summarized=True)
return wallet_display(wallet_service, options.showprivkey, summarized=True)
elif method == "history":
if not isinstance(jm_single().bc_interface, BitcoinCoreInterface):
jmprint('showing history only available when using the Bitcoin Core ' +

Loading…
Cancel
Save