joinmarket-clientserver/src/jmclient/wallet_service.py

#! /usr/bin/env python

import collections
import itertools
import time
import sys
from typing import Dict, List, Optional, Set, Tuple
from decimal import Decimal
from copy import deepcopy
from twisted.internet import reactor
from twisted.internet import task
from twisted.application.service import Service
from numbers import Integral
import jmbitcoin as btc
from jmclient.configure import jm_single, get_log
from jmclient.output import fmt_tx_data
from jmclient.blockchaininterface import (INF_HEIGHT, BitcoinCoreInterface,
    BitcoinCoreNoHistoryInterface)
from jmclient.wallet import FidelityBondMixin, BaseWallet
from jmbase import (stop_reactor, hextobin, utxo_to_utxostr,
                    jmprint, EXIT_SUCCESS, EXIT_FAILURE)

"""Wallet service

The purpose of this independent service is to allow
running applications to keep an up to date, asynchronous
view of the current state of its wallet, deferring any
polling mechanisms needed against the backend blockchain
interface here.
"""

jlog = get_log()

class WalletService(Service):
    EXTERNAL_WALLET_LABEL = "joinmarket-notify"

    def __init__(self, wallet):
        # The two principal member variables
        # are the blockchaininterface instance,
        # which is currently global in JM but
        # could be more flexible in future, and
        # the JM wallet object.
        self.bci = jm_single().bc_interface

        # main loop used to check for transactions, instantiated
        # after wallet is synced:
        self.monitor_loop = None
        self.wallet = wallet
        self.synced = False

        # used to flag RPC failure at construction of object:
        self.rpc_error = False

        # keep track of the quasi-real-time blockheight
        # (updated in main monitor loop)
        self.current_blockheight = None

        if self.bci is not None:
            if not self.update_blockheight():
                # this accounts for the unusual case
                # where the application started up with
                # a functioning blockchain interface, but
                # that bci is now failing when we are starting
                # the wallet service.
                jlog.error("Failure of RPC connection to Bitcoin Core in "
                          "wallet service startup. Application cannot "
                          "continue, shutting down.")
                self.rpc_error = ("Failure of RPC connection to Bitcoin "
                                  "Core in wallet service startup.")
                # no need to call stopService as it has not yet been started.
                stop_reactor()
        else:
            jlog.warning("No blockchain source available, " +
                "wallet tools will not show correct balances.")

        # Dicts of registered callbacks, by type
        # and then by txinfo, for events
        # on transactions.
        self.callbacks = {
            "all": [],  # note: list, not dict
            "unconfirmed": {},
            "confirmed": {},
        }

        self.restart_callback = None

        # transactions we are actively monitoring,
        # i.e. they are not new but we want to track:
        self.active_txs = {}
        # to ensure transactions are only processed once:
        self.processed_txids = set()

        self.set_autofreeze_warning_cb()

    def update_blockheight(self):
        """ Can be called manually (on startup, or for tests)
        but will be called as part of main monitoring
        loop to ensure new transactions are added at
        the right height.
        Any failure of the RPC call must result in this returning
        False, otherwise return True (means self.current_blockheight
        has been correctly updated).
        """

        def critical_error():
            jlog.error("Critical error updating blockheight.")
            # this cleanup (a) closes the wallet, removing the lock
            # and (b) signals to clients that the service is no longer
            # in a running state, both of which can be useful
            # post reactor shutdown.
            self.stopService()
            stop_reactor()
            return False

        if self.current_blockheight:
            old_blockheight = self.current_blockheight
        else:
            old_blockheight = -1
        try:
            self.current_blockheight = self.bci.get_current_block_height()
        except Exception as e:
            # This should never happen now, as we are catching every
            # possible Exception in jsonrpc or bci.rpc:
            return critical_error()
        if not self.current_blockheight:
            return critical_error()

        # We have received a new blockheight from Core, sanity check it:
        assert isinstance(self.current_blockheight, Integral)
        assert self.current_blockheight >= 0
        if self.current_blockheight < old_blockheight:
            jlog.warn("Bitcoin Core is reporting a lower blockheight, "
                      "possibly a reorg.")
        return True

    def startService(self):
        """ Encapsulates start up actions.
        Here wallet sync.
        """
        super().startService()
        self.request_sync_wallet()

    def stopService(self):
        """ Encapsulates shut down actions.
        Note that after the service is stopped, it
        should *not* be restarted, instead a new
        WalletService instance should be created.
        """
        if self.monitor_loop and self.monitor_loop.running:
            self.monitor_loop.stop()
        self.wallet.close()
        super().stopService()

    def isRunning(self):
        if self.running == 1:
            return True
        return False

    def add_restart_callback(self, callback):
        """ Sets the function that will be
        called in the event that the wallet
        sync completes with a restart warning.
        The only argument is a message string,
        which the calling function displays to
        the user before quitting gracefully.
        """
        self.restart_callback = callback

    def request_sync_wallet(self):
        """ Ensures wallet sync is complete
        before the main event loop starts.
        """
        if self.bci is not None:
            d = task.deferLater(reactor, 0.0, self.sync_wallet)
            d.addCallback(self.start_wallet_monitoring)

    def register_callbacks(self, callbacks, txinfo, cb_type="all"):
        """ Register callbacks that will be called by the
        transaction monitor loop, on transactions stored under
        our wallet label (see WalletService.get_wallet_name()).
        Callback arguments are currently (txd, txid) and return
        is boolean, except "confirmed" callbacks which have
        arguments (txd, txid, confirmations).
        Note that callbacks MUST correctly return True if they
        recognized the transaction and processed it, and False
        if not. The True return value will be used to remove
        the callback from the list.
        Arguments:
        `callbacks` - a list of functions with signature as above
        and return type boolean.
        `txinfo` - either a txid expected for the transaction, if
        known, or a tuple of the ordered output set, of the form
        ((CScript, int), ..). This is be constructed from the
        CMutableTransaction vout list.
        See WalletService.transaction_monitor().
        `cb_type` - must be one of "all", "unconfirmed", "confirmed";
        the first type will be called back once for every new
        transaction, the second only once when the number of
        confirmations is 0, and the third only once when the number
        of confirmations is > 0.
        """
        if cb_type == "all":
            # note that in this case, txid is ignored.
            self.callbacks["all"].extend(callbacks)
        elif cb_type in ["unconfirmed", "confirmed"]:
            if callbacks:
                reg = self.callbacks[cb_type].setdefault(txinfo, [])
                if isinstance(reg, str):
                    # found a txid breadcrumb for this txinfo
                    reg = self.callbacks[cb_type].setdefault(reg, [])
                reg.extend(callbacks)
        else:
            assert False, "Invalid argument: " + cb_type


    def start_wallet_monitoring(self, syncresult):
        """ Once the initialization of the service
        (currently, means: wallet sync) is complete,
        we start the main monitoring jobs of the
        wallet service (currently, means: monitoring
        all new transactions on the blockchain that
        are recognised as belonging to the Bitcoin
        Core wallet with the JM wallet's label).
        """
        if not syncresult:
            jlog.error("Failed to sync the bitcoin wallet. Shutting down.")
            self.stopService()
            if reactor.running:
                reactor.stop()
            return
        jlog.info("Starting transaction monitor in walletservice")
        self.monitor_loop = task.LoopingCall(
            self.transaction_monitor)
        self.monitor_loop.start(5.0)

    def import_non_wallet_address(self, address):
        """ Used for keeping track of transactions which
        have no in-wallet destinations. External wallet
        label is used to avoid breaking fast sync (which
        assumes label => wallet)
        """
        if not self.bci.is_address_imported(address):
            self.bci.import_addresses([address], self.EXTERNAL_WALLET_LABEL,
                                  restart_cb=self.restart_callback)

    def default_autofreeze_warning_cb(self, utxo):
        success, utxostr = utxo_to_utxostr(utxo)
        assert success, "Autofreeze warning cb called with invalid utxo."
        jlog.warning("WARNING: new utxo has been automatically "
             "frozen to prevent forced address reuse: ")
        jlog.warning(utxostr)
        jlog.warning("You can unfreeze this utxo with the method "
                 "'freeze' of wallet-tool.py or the Coins tab "
                 "of Joinmarket-Qt.")

    def set_autofreeze_warning_cb(self, cb=None):
        """ This callback takes a single argument, the
        string representation of a utxo in form txid:index,
        and informs the user that the utxo has been frozen.
        It returns nothing (the user is not deciding in this case,
        as the decision was already taken by the configuration).
        """
        if cb is None:
            self.autofreeze_warning_cb = self.default_autofreeze_warning_cb
        else:
            self.autofreeze_warning_cb = cb

    def check_for_reuse(self, added_utxos):
        """ (a) Check if addresses in new utxos are already in
        used address list, (b) record new addresses as now used
        (c) disable the new utxo if it returned as true for (a),
        and it passes the filter set in the configuration.
        """
        to_be_frozen = set()
        for au in added_utxos:
            if self.has_address_been_used(added_utxos[au]["address"]):
                to_be_frozen.add(au)
        # any utxos actually added must have their destination address
        # added to the used address list for this program run:
        for au in added_utxos.values():
            self.used_addresses.add(au["address"])

        # disable those that passed the first check, before the addition,
        # if they satisfy configured logic
        for utxo in to_be_frozen:
            freeze_threshold = jm_single().config.getint("POLICY",
                                        "max_sats_freeze_reuse")
            if freeze_threshold == -1 or added_utxos[
                utxo]["value"] <= freeze_threshold:
                # freezing of coins must be communicated to user:
                self.autofreeze_warning_cb(utxo)
                self.disable_utxo(*utxo)

    def _yield_new_transactions(self):
        """ Constrains the sequence generated by bci.yield_transactions so
        that it stops just before it would yield the newest transaction
        previously yielded by _yield_new_transactions.
        """
        since_txid = self.last_seen_txid
        last = True
        for tx in self.bci.yield_transactions():
            if 'txid' in tx:
                txid = tx['txid']
                if txid == since_txid:
                    return
                if last:
                    self.last_seen_txid = txid
                    last = False
            yield tx

    def transaction_monitor(self):
        """Keeps track of any changes in the wallet (new transactions).
        Intended to be run as a twisted task.LoopingCall so that this
        Service is constantly in near-realtime sync with the blockchain.
        """

        if not self.update_blockheight():
            return

        # for this invocation of transaction_monitor, we *don't* want
        # to call `gettransaction` more than once per txid, even if the
        # `listtransactions` result has multiple instances for different
        # wallet labels; so we use a temporary variable to cache.
        seen_txids = set()
        wallet_name = self.get_wallet_name()
        for txid in itertools.chain(list(self.active_txs), reversed(
                [tx['txid'] for tx in self._yield_new_transactions()
                    if 'txid' in tx])):
            if txid in seen_txids:
                continue
            seen_txids.add(txid)
            res = self.bci.get_transaction(hextobin(txid))
            if not res:
                continue
            confs = res["confirmations"]
            if not isinstance(confs, Integral):
                jlog.warning("Malformed gettx result: " + str(res))
                continue
            if confs < 0:
                jlog.info(
                    "Transaction: " + txid + " has a conflict, abandoning.")
                self.active_txs.pop(txid, None)
                continue
            if confs == 0:
                if txid in self.active_txs:
                    # avoid processing an unconfirmed tx we've already seen
                    continue
                height = None
            else:
                height = self.current_blockheight - confs + 1

            txd = self.active_txs.get(txid) or \
                    self.bci.get_deser_from_gettransaction(res)
            if txd is None:
                continue
            removed_utxos, added_utxos = self.wallet.process_new_tx(txd, height)
            if txid not in self.processed_txids:
                # apply checks to disable/freeze utxos to reused addrs if needed:
                self.check_for_reuse(added_utxos)
                # TODO note that this log message will be missed if confirmation
                # is absurdly fast, this is considered acceptable compared with
                # additional complexity.
                self.log_new_tx(removed_utxos, added_utxos, txid)
                self.processed_txids.add(txid)

            # first fire 'all' type callbacks, irrespective of if the
            # transaction pertains to anything known (but must
            # have correct label per above); filter on this Joinmarket wallet label,
            # or the external monitoring label:
            if (any(self.bci.is_address_labeled(addr, wallet_name) or
                    self.bci.is_address_labeled(addr,
                        self.EXTERNAL_WALLET_LABEL)
                    for addr in res["details"])):
                for f in self.callbacks["all"]:
                    # note we need no return value as we will never
                    # remove these from the list
                    f(txd, txid)

            # txid is not always available at the time of callback registration.
            # Migrate any callbacks registered under the provisional key, and
            # leave a txid breadcrumb so check_callback_called can find it.
            txos = tuple((x.scriptPubKey, x.nValue) for x in txd.vout)
            for cb_type in ["unconfirmed", "confirmed"]:
                callbacks = self.callbacks[cb_type]
                reg = callbacks.get(txos)
                if isinstance(reg, list):
                    callbacks.setdefault(txid, [])[:0] = reg
                    callbacks[txos] = txid

            # note that len(added_utxos) > 0 is not a sufficient condition for
            # the tx being new, since wallet.add_new_utxos will happily re-add
            # a utxo that already exists; but this does not cause re-firing
            # of callbacks since we in these cases delete the callback after being
            # called once.
            # Note also that it's entirely possible that there are only removals,
            # not additions, to the utxo set, specifically in sweeps to external
            # addresses. In this case, since removal can by definition only
            # happen once, we must allow txids in self.active_txs through the
            # filter.
            if len(added_utxos) > 0 or len(removed_utxos) > 0 \
                    or txid in self.active_txs:
                if confs == 0:
                    callbacks = [f for f in
                            self.callbacks["unconfirmed"].pop(txid, [])
                            if not f(txd, txid)]
                    if callbacks:
                        self.callbacks["unconfirmed"][txid] = callbacks
                    else:
                        self.callbacks["unconfirmed"].pop(txos, None)
                    # we always add into active_txs, even if the caller
                    # didn't create any confirmed callbacks, because we
                    # need to trigger process_new_tx logic to update
                    # the height of the utxo in UtxoManager
                    self.active_txs[txid] = txd
                elif confs > 0:
                    callbacks = [f for f in
                            self.callbacks["confirmed"].pop(txid, [])
                            if not f(txd, txid, confs)]
                    if callbacks:
                        self.callbacks["confirmed"][txid] = callbacks
                    else:
                        self.callbacks["confirmed"].pop(txos, None)
                        # no more callbacks registered; stop monitoring tx
                        self.active_txs.pop(txid, None)

    def check_callback_called(self, txinfo, callback, cbtype, msg):
        """ Intended to be a deferred Task to be scheduled some
        set time after the callback was registered. "all" type
        callbacks do not expire and are not included.
        If the callback was previously called, return True, otherwise False.
        """
        assert cbtype in ["unconfirmed", "confirmed"]
        callbacks = self.callbacks[cbtype]
        if isinstance(txinfo, str):
            txid = txinfo
            reg = callbacks.get(txid)
        else:
            txid = None
            reg = callbacks.get(txinfo)
            if isinstance(reg, str):
                # found a txid breadcrumb for this txinfo
                txid = reg
                reg = callbacks.get(txid)
        if reg:
            if callback in reg:
                # the callback was not called, drop it and warn
                reg.remove(callback)
                if not reg:
                    del callbacks[txinfo]
                    if txid:
                        callbacks.pop(txid, None)
                        # no more callbacks registered; stop monitoring tx
                        self.active_txs.pop(txid, None)
                jlog.info("Timed out: " + msg)
                return False
            # if callback is not in the list, it was already
            # processed and so do nothing.
        return True

    def log_new_tx(self, removed_utxos, added_utxos, txid):
        """ Changes to the wallet are logged at INFO level by
        the WalletService.
        """
        def report_changed(x, utxos):
            if len(utxos.keys()) > 0:
                jlog.info(x + ' utxos=\n{}'.format('\n'.join(
                    '{} - {}'.format(utxo_to_utxostr(u)[1],
                        fmt_tx_data(tx_data, self)) for u,
                    tx_data in utxos.items())))

        report_changed("Removed", removed_utxos)
        report_changed("Added", added_utxos)


        """ Wallet syncing code
        """

    def sync_wallet(self, fast=True):
        """ Syncs wallet; note that if slow sync
        requires multiple rounds this must be called
        until self.synced is True.
        Before starting the event loop, we cache
        the current most recent transactions as
        reported by the blockchain interface, since
        we are interested in deltas.
        """
        # If this is called when syncing already complete:
        if self.synced:
            return True
        if fast:
            self.sync_wallet_fast()
        else:
            self.sync_addresses()
            self.sync_unspent()
        # Don't attempt updates on transactions that existed
        # before startup
        self.last_seen_txid = next(
                (tx['txid'] for tx in self.bci.yield_transactions()
                    if 'txid' in tx), None)
        if isinstance(self.bci, BitcoinCoreNoHistoryInterface):
            self.bci.set_wallet_no_history(self.wallet)
        return self.synced

    def resync_wallet(self, fast=True):
        """ The self.synced state is generally
        updated to True, once, at the start of
        a run of a particular program. Here we
        can manually force re-sync.
        """
        self.synced = False
        self.sync_wallet(fast=fast)

    def sync_wallet_fast(self):
        """Exploits the fact that given an index_cache,
        all addresses necessary should be imported, so we
        can just list all used addresses to find the right
        index values.
        """
        self.sync_addresses_fast()
        self.sync_unspent()

    def has_address_been_used(self, address):
        """ Once wallet has been synced, the set of used
        addresses includes those identified at sync time,
        plus any used during operation. This is stored in
        the WalletService object as self.used_addresses.
        """
        return address in self.used_addresses

    def get_address_usages(self):
        """ sets, at time of sync, the list of addresses that
        have been used in our Core wallet with the specific label
        for our JM wallet. This operation is generally immediate.
        """
        agd = self.bci.listaddressgroupings()
        # flatten all groups into a single list; then, remove duplicates
        fagd = (tuple(item) for sublist in agd for item in sublist)
        # "deduplicated flattened address grouping data" = dfagd
        dfagd = set(fagd)
        used_addresses = set()
        for addr_info in dfagd:
            if len(addr_info) < 3 or addr_info[2] != self.get_wallet_name():
                continue
            used_addresses.add(addr_info[0])
        self.used_addresses = used_addresses

    def sync_addresses_fast(self):
        """Locates all used addresses in the account (whether spent or
        unspent outputs), and then, assuming that all such usages must be
        related to our wallet, calculates the correct wallet indices and
        does any needed imports.

        This will not result in a full sync if working with a new
        Bitcoin Core instance, in which case "recoversync" should have
        been specifically chosen by the user.
        """
        self.get_address_usages()
        # for a first run, import first chunk
        if not self.used_addresses:
            jlog.info("Detected new wallet, performing initial import")
            # delegate inital address import to sync_addresses
            # this should be fast because "getaddressesbyaccount" should return
            # an empty list in this case
            self.sync_addresses()
            self.synced = True
            return

        # Wallet has been used; scan forwards.
        jlog.debug("Fast sync in progress. Got this many used addresses: " + str(
            len(self.used_addresses)))
        # Need to have wallet.index point to the last used address
        # Algo:
        #    1. Scan batch 1 of each branch, record matched wallet addresses.
        #    2. Check if all addresses in 'used addresses' have been matched, if
        #       so, break.
        #    3. Repeat the above for batch 2, 3.. up to max 20 batches.
        #    4. If after all 20 batches not all used addresses were matched,
        #       quit with error.
        #    5. Calculate used indices.
        #    6. If all used addresses were matched, set wallet index to highest
        #       found index in each branch and mark wallet sync complete.
        # Rationale for this algo:
        #    Retrieving addresses is a non-zero computational load, so batching
        #    and then caching allows a small sync to complete *reasonably*
        #    quickly while a larger one is not really negatively affected.
        #    The downside is another free variable, batch size, but this need
        #    not be exposed to the user; it is not the same as gap limit.
        #    The assumption is that usage of addresses occurs only if already
        #    imported, either through in-app usage during coinjoins, or because
        #    deposit by user will be based on wallet_display() which is only
        #    showing imported addresses. Hence the gap-limit import at the end
        #    to ensure this is always true.
        remaining_used_addresses = self.used_addresses.copy()
        addresses, saved_indices = self.collect_addresses_init()
        for addr in addresses:
            remaining_used_addresses.discard(addr)

        BATCH_SIZE = 100
        MAX_ITERATIONS = 20
        current_indices = deepcopy(saved_indices)
        for j in range(MAX_ITERATIONS):
            if not remaining_used_addresses:
                break
            gap_addrs = self.collect_addresses_gap(gap_limit=BATCH_SIZE)
            # note: gap addresses *not* imported here; we are still trying
            # to find the highest-index used address, and assume that imports
            # are up to that index (at least) - see above main rationale.
            for addr in gap_addrs:
                remaining_used_addresses.discard(addr)

            # increase wallet indices for next iteration
            for md in current_indices:
                current_indices[md][0] += BATCH_SIZE
                current_indices[md][1] += BATCH_SIZE
            self.rewind_wallet_indices(current_indices, current_indices)
        else:
            self.rewind_wallet_indices(saved_indices, saved_indices)
            raise Exception("Failed to sync in fast mode after 20 batches; "
                            "please re-try wallet sync with --recoversync flag.")

        # creating used_indices on-the-fly would be more efficient, but the
        # overall performance gain is probably negligible
        used_indices = self.get_used_indices(self.used_addresses)
        self.rewind_wallet_indices(used_indices, saved_indices)
        # at this point we have the correct up to date index at each branch;
        # we ensure that all addresses that will be displayed (see wallet_utils.py,
        # function wallet_display()) are imported by importing gap limit beyond current
        # index:
        self.bci.import_addresses(self.collect_addresses_gap(), self.get_wallet_name(),
                                  self.restart_callback)

        if isinstance(self.wallet, FidelityBondMixin):
            mixdepth = FidelityBondMixin.FIDELITY_BOND_MIXDEPTH
            address_type = FidelityBondMixin.BIP32_BURN_ID

            burner_outputs = self.wallet.get_burner_outputs()
            max_index = 0
            for path_repr in burner_outputs:
                index = self.wallet.path_repr_to_path(path_repr.decode())[-1]
                max_index = max(index+1, max_index)
            self.wallet.set_next_index(mixdepth, address_type, max_index,
                force=True)

        self.synced = True

    def display_rescan_message_and_system_exit(self, restart_cb):
        #TODO using system exit here should be avoided as it makes the code
        # harder to understand and reason about
        #theres also a sys.exit() in BitcoinCoreInterface.import_addresses()
        #perhaps have sys.exit() placed inside the restart_cb that only
        # CLI scripts will use
        if self.bci.__class__ == BitcoinCoreInterface:
            #Exit conditions cannot be included in tests
            restart_msg = ("Use `bitcoin-cli rescanblockchain` if you're "
                           "recovering an existing wallet from backup seed\n"
                           "Otherwise just restart this joinmarket application.")
            if restart_cb:
                restart_cb(restart_msg)
            else:
                jmprint(restart_msg, "important")
                sys.exit(EXIT_SUCCESS)

    def sync_burner_outputs(self, burner_txes):
        mixdepth = FidelityBondMixin.FIDELITY_BOND_MIXDEPTH
        address_type = FidelityBondMixin.BIP32_BURN_ID
        self.wallet.set_next_index(mixdepth, address_type, self.wallet.gap_limit,
            force=True)
        highest_used_index = 0
        known_burner_outputs = self.wallet.get_burner_outputs()

        index = -1
        while index - highest_used_index < self.wallet.gap_limit:
            index += 1
            self.wallet.set_next_index(mixdepth, address_type, index, force=True)
            path = self.wallet.get_path(mixdepth, address_type, index)
            path_privkey, engine = self.wallet._get_key_from_path(path)
            path_pubkey = engine.privkey_to_pubkey(path_privkey)
            path_pubkeyhash = btc.Hash160(path_pubkey)
            for burner_tx in burner_txes:
                burner_pubkeyhash, gettx = burner_tx
                if burner_pubkeyhash != path_pubkeyhash:
                    continue
                highest_used_index = index
                path_repr = self.wallet.get_path_repr(path)
                if path_repr.encode() in known_burner_outputs:
                    continue
                txid = gettx["txid"]
                jlog.info("Found a burner transaction txid=" + txid + " path = "
                    + path_repr)
                try:
                    merkle_branch = self.bci.get_tx_merkle_branch(txid, gettx["blockhash"])
                except ValueError as e:
                    jlog.warning(repr(e))
                    jlog.warning("Merkle branch likely not available, use "
                        + "wallet-tool `addtxoutproof`")
                    merkle_branch = None
                block_height = self.bci.get_block_height(gettx["blockhash"])
                if merkle_branch:
                    assert self.bci.verify_tx_merkle_branch(txid, block_height, merkle_branch)
                self.wallet.add_burner_output(path_repr, gettx["hex"], block_height,
                    merkle_branch, gettx["blockindex"])

        self.wallet.set_next_index(mixdepth, address_type, highest_used_index + 1)

    def get_all_transactions(self):
        """ Get all transactions (spending or receiving) that
        are currently recorded by our blockchain interface as relating
        to this wallet, as a list.
        """
        res = []
        processed_txids = set()
        for r in self.bci.yield_transactions():
            txid = r["txid"]
            if txid not in processed_txids:
                tx = self.bci.get_transaction(hextobin(txid))
                res.append(self.bci.get_deser_from_gettransaction(tx))
                processed_txids.add(txid)
        return res

    def get_transaction(self, txid):
        """ If the transaction for txid is an in-wallet
        transaction, will return a CTransaction object for it;
        if not, will return None.
        """
        tx = self.bci.get_transaction(txid)
        if not tx:
            return None
        return self.bci.get_deser_from_gettransaction(tx)

    def get_block_height(self, blockhash):
        return self.bci.get_block_height(blockhash)

    def rescanblockchain(self, start_height: int, end_height: Optional[int] = None) -> None:
        self.bci.rescanblockchain(start_height, end_height)

    def get_backend_wallet_rescan_status(self) -> Tuple[bool, Optional[Decimal]]:
        return self.bci.get_wallet_rescan_status()

    def get_transaction_block_height(self, tx):
        """ Given a CTransaction object tx, return
        the block height at which it was mined, or False
        if it is not found in the blockchain (including if
        it is not a wallet tx and so can't be queried).
        """
        txid = tx.GetTxid()[::-1]
        return self.bci.get_block_height(self.bci.get_transaction(
            txid)["blockhash"])

    def sync_addresses(self):
        """ Triggered by use of --recoversync option in scripts,
        attempts a full scan of the blockchain without assuming
        anything about past usages of addresses (does not use
        wallet.index_cache as hint).
        """
        jlog.debug("requesting detailed wallet history")
        wallet_name = self.get_wallet_name()
        addresses, saved_indices = self.collect_addresses_init()

        import_needed = self.bci.import_addresses_if_needed(addresses,
            wallet_name)
        if import_needed:
            self.display_rescan_message_and_system_exit(self.restart_callback)
            return

        if isinstance(self.wallet, FidelityBondMixin):
            tx_receive = []
            burner_txes = []
            for tx in self.bci.yield_transactions():
                if tx['category'] == 'receive':
                    tx_receive.append(tx)
                elif tx["category"] == "send":
                    gettx = self.bci.get_transaction(hextobin(tx["txid"]))
                    txd = self.bci.get_deser_from_gettransaction(gettx)
                    if len(txd.vout) > 1:
                        continue
                    #must be mined into a block to sync
                    #otherwise there's no merkleproof or block index
                    if gettx["confirmations"] < 1:
                        continue
                    script = txd.vout[0].scriptPubKey
                    if script[0] != 0x6a: #OP_RETURN
                        continue
                    pubkeyhash = script[2:]
                    burner_txes.append((pubkeyhash, gettx))

            self.sync_burner_outputs(burner_txes)
            used_addresses_gen = set(tx["address"] for tx in tx_receive)
        else:
            #not fidelity bond wallet, significantly faster sync
            used_addresses_gen = set(tx['address']
                                  for tx in self.bci.yield_transactions()
                                  if tx['category'] == 'receive')
        # needed for address-reuse check:
        self.used_addresses = used_addresses_gen
        used_indices = self.get_used_indices(used_addresses_gen)
        jlog.debug("got used indices: {}".format(used_indices))
        gap_limit_used = not self.check_gap_indices(used_indices)
        self.rewind_wallet_indices(used_indices, saved_indices)

        new_addresses = self.collect_addresses_gap()
        if self.bci.import_addresses_if_needed(new_addresses, wallet_name):
            jlog.debug("Syncing iteration finished, additional step required (more address import required)")
            self.synced = False
            self.display_rescan_message_and_system_exit(self.restart_callback)
        elif gap_limit_used:
            jlog.debug("Syncing iteration finished, additional step required (gap limit used)")
            self.synced = False
        else:
            jlog.debug("Wallet successfully synced")
            self.rewind_wallet_indices(used_indices, saved_indices)
            self.synced = True

    def sync_unspent(self):
        st = time.time()
        # block height needs to be real time for addition to our utxos.
        # We are a little aggressive in trying this RPC call, because if
        # it fails we cannot continue:
        for i in range(5):
            current_blockheight = self.bci.get_current_block_height()
            if current_blockheight:
                break
            if i < 4:
                jlog.debug("Error calling blockheight RPC, trying again in 0.5s.")
            # sync unspent calls are synchronous; this short sleep should
            # not affect us since we are not yet in main monitor loop.
            time.sleep(0.5)
        if not current_blockheight:
            # this failure is critical; give up:
            jlog.error("Failed to access the current blockheight, "
                       "and therefore could not sync the wallet. "
                       "Shutting down.")
            if self.isRunning:
                self.stopService()
            stop_reactor()
            sys.exit(EXIT_FAILURE)
        wallet_name = self.get_wallet_name()
        self.reset_utxos()

        unspent_list = self.bci.listunspent()
        # filter on label, but note (a) in certain circumstances (in-
        # wallet transfer) it is possible for the utxo to be labeled
        # with the external label, and (b) the wallet will know if it
        # belongs or not anyway (is_known_addr):
        our_unspent_list = [x for x in unspent_list if (
            self.bci.is_address_labeled(x, wallet_name) or
            self.bci.is_address_labeled(x, self.EXTERNAL_WALLET_LABEL))]
        for utxo in our_unspent_list:
            if not self.is_known_addr(utxo['address']):
                continue
            # The result of bitcoin core's listunspent RPC call does not have
            # a "height" field, only "confirmations".
            # But the result of scantxoutset used in no-history sync does
            # have "height".
            if "height" in utxo:
                height = utxo["height"]
            else:
                height = None
                # wallet's utxo database needs to store an absolute rather
                # than relative height measure:
                confs = int(utxo['confirmations'])
                if confs < 0:
                    jlog.warning("Utxo not added, has a conflict: " + str(utxo))
                    continue
                if confs >= 1:
                    height = current_blockheight - confs + 1
            self._add_unspent_txo(utxo, height)
        et = time.time()
        jlog.debug('bitcoind sync_unspent took ' + str((et - st)) + 'sec')

    def _add_unspent_txo(self, utxo, height):
        """
        Add a UTXO as returned by rpc's listunspent call to the wallet.
        Note that these are returned as little endian outpoint txids, so
        are converted.
        params:
            utxo: single utxo dict as returned by listunspent
            current_blockheight: blockheight as integer, used to
            set the block in which a confirmed utxo is included.
        """
        txid = hextobin(utxo['txid'])
        script = hextobin(utxo['scriptPubKey'])
        value = int(Decimal(str(utxo['amount'])) * Decimal('1e8'))
        self.add_utxo(txid, int(utxo['vout']), script, value, height)
        # if we start up with unconfirmed outputs, they must be
        # put into the transaction monitor state, so we can recognize
        # when they transition to confirmed.
        if height is None:
            txd = self.bci.get_deser_from_gettransaction(
                self.bci.get_transaction(txid))
            self.active_txs[utxo['txid']] = txd
            self.processed_txids.add(utxo['txid'])


    """ The following functions mostly are not pure
    pass through to the underlying wallet, so declared
    here; the remainder are covered by the __getattr__
    fallback.
    """

    def save_wallet(self):
        self.wallet.save()

    def get_utxos_by_mixdepth(self, include_disabled=False,
                              verbose=False, includeconfs=False):
        """ Returns utxos by mixdepth in a dict, optionally including
        information about how many confirmations each utxo has.
        """
        def height_to_confs(x):
            # convert height entries to confirmations:
            ubym_conv = collections.defaultdict(dict)
            for m, i in x.items():
                for u, d in i.items():
                    ubym_conv[m][u] = d
                    h = ubym_conv[m][u].pop("height")
                    if h == INF_HEIGHT:
                        confs = 0
                    else:
                        confs = self.current_blockheight - h + 1
                    ubym_conv[m][u]["confs"] = confs
            return ubym_conv
        ubym = self.wallet.get_utxos_by_mixdepth(
            include_disabled=include_disabled, includeheight=includeconfs)
        if not includeconfs:
            return ubym
        else:
            return height_to_confs(ubym)

    def minconfs_to_maxheight(self, minconfs):
        if minconfs is None:
            return None
        else:
            return self.current_blockheight - minconfs + 1

    def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None,
                     minconfs=None, includeaddr=False, require_auth_address=False):
        """ Request utxos from the wallet in a particular mixdepth to satisfy
        a certain total amount, optionally set the selector function (or use
        the currently configured function set by the wallet, and optionally
        require a minimum of minconfs confirmations (default none means
        unconfirmed are allowed).
        """
        return self.wallet.select_utxos(
            mixdepth, amount, utxo_filter=utxo_filter, select_fn=select_fn,
            maxheight=self.minconfs_to_maxheight(minconfs),
            includeaddr=includeaddr, require_auth_address=require_auth_address)

    def get_balance_by_mixdepth(self, verbose=True,
                                include_disabled=False,
                                minconfs=None):
        if minconfs is None:
            maxheight = None
        else:
            maxheight = self.current_blockheight - minconfs + 1
        return self.wallet.get_balance_by_mixdepth(verbose=verbose,
                                                   include_disabled=include_disabled,
                                                   maxheight=maxheight)

    def import_addr(self, addr):
        if self.bci is not None and hasattr(self.bci, 'import_addresses'):
            self.bci.import_addresses([addr], self.wallet.get_wallet_name())

    def get_internal_addr(self, mixdepth):
        addr = self.wallet.get_internal_addr(mixdepth)
        self.import_addr(addr)
        return addr

    def collect_addresses_init(self) -> Tuple[Set[str], Dict[int, List[int]]]:
        """ Collects the "current" set of addresses,
        as defined by the indices recorded in the wallet's
        index cache (persisted in the wallet file usually).
        Note that it collects up to the current indices plus
        the gap limit.
        """
        addresses = set()
        saved_indices = dict()

        for md in range(self.max_mixdepth + 1):
            saved_indices[md] = [0, 0]
            for address_type in (BaseWallet.ADDRESS_TYPE_EXTERNAL,
                                 BaseWallet.ADDRESS_TYPE_INTERNAL):
                next_unused = self.get_next_unused_index(md, address_type)
                for index in range(next_unused):
                    addresses.add(self.get_addr(md, address_type, index))
                for index in range(self.gap_limit):
                    addresses.add(self.get_new_addr(md, address_type))
                # reset the indices to the value we had before the
                # new address calls:
                self.set_next_index(md, address_type, next_unused)
                saved_indices[md][address_type] = next_unused
            # include any imported addresses
            for path in self.yield_imported_paths(md):
                addresses.add(self.get_address_from_path(path))

        if isinstance(self.wallet, FidelityBondMixin):
            md = FidelityBondMixin.FIDELITY_BOND_MIXDEPTH
            address_type = FidelityBondMixin.BIP32_TIMELOCK_ID
            for timenumber in range(FidelityBondMixin.TIMENUMBER_COUNT):
                addresses.add(self.get_addr(md, address_type, timenumber))

        return addresses, saved_indices

    def collect_addresses_gap(self, gap_limit=None):
        gap_limit = gap_limit or self.gap_limit
        addresses = set()

        for md in range(self.max_mixdepth + 1):
            for address_type in (BaseWallet.ADDRESS_TYPE_INTERNAL,
                                 BaseWallet.ADDRESS_TYPE_EXTERNAL):
                old_next = self.get_next_unused_index(md, address_type)
                for index in range(gap_limit):
                    addresses.add(self.get_new_addr(md, address_type))
                self.set_next_index(md, address_type, old_next)

        return addresses

    def get_external_addr(self, mixdepth):
        addr = self.wallet.get_external_addr(mixdepth)
        self.import_addr(addr)
        return addr

    def __getattr__(self, attr):
        # any method not present here is passed
        # to the wallet:
        return getattr(self.wallet, attr)