diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py index 7dcbe5fd7..c8c261bdd 100644 --- a/electrum/address_synchronizer.py +++ b/electrum/address_synchronizer.py @@ -138,9 +138,8 @@ class AddressSynchronizer(Logger, EventListener): return len(self._history_local.get(addr, ())) def get_txin_address(self, txin: TxInput) -> Optional[str]: - if isinstance(txin, PartialTxInput): - if txin.address: - return txin.address + if txin.address: + return txin.address prevout_hash = txin.prevout.txid.hex() prevout_n = txin.prevout.out_idx for addr in self.db.get_txo_addresses(prevout_hash): diff --git a/electrum/commands.py b/electrum/commands.py index 33073bb39..91c42a2c8 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -762,6 +762,8 @@ class Commands: coins = wallet.get_spendable_coins(None) if domain_coins is not None: coins = [coin for coin in coins if (coin.prevout.to_str() in domain_coins)] + tx.add_info_from_wallet(wallet) + await tx.add_info_from_network(self.network) new_tx = wallet.bump_fee( tx=tx, txid=tx.txid(), diff --git a/electrum/gui/kivy/uix/dialogs/tx_dialog.py b/electrum/gui/kivy/uix/dialogs/tx_dialog.py index c10a9aac8..0c8406c22 100644 --- a/electrum/gui/kivy/uix/dialogs/tx_dialog.py +++ b/electrum/gui/kivy/uix/dialogs/tx_dialog.py @@ -16,7 +16,7 @@ from electrum.util import InvalidPassword from electrum.address_synchronizer import TX_HEIGHT_LOCAL from electrum.wallet import CannotBumpFee, CannotCPFP, CannotDoubleSpendTx from electrum.transaction import Transaction, PartialTransaction -from electrum.network import NetworkException +from electrum.network import NetworkException, Network from electrum.gui.kivy.i18n import _ from electrum.gui.kivy.util import address_colors @@ -120,19 +120,21 @@ Builder.load_string(''' class TxDialog(Factory.Popup): - def __init__(self, app, tx): + def __init__(self, app, tx: Transaction): Factory.Popup.__init__(self) self.app = app # type: ElectrumWindow self.wallet = self.app.wallet - self.tx = tx # type: Transaction + self.tx = tx self.config = self.app.electrum_config # If the wallet can populate the inputs with more info, do it now. # As a result, e.g. we might learn an imported address tx is segwit, # or that a beyond-gap-limit address is is_mine. # note: this might fetch prev txs over the network. - # note: this is a no-op for complete txs tx.add_info_from_wallet(self.wallet) + if not tx.is_complete() and tx.is_missing_info_from_network(): + Network.run_from_another_thread( + tx.add_info_from_network(self.wallet.network, timeout=10)) # FIXME is this needed?... def on_open(self): self.update() @@ -201,19 +203,6 @@ class TxDialog(Factory.Popup): ) action_dropdown.update(options=options) - def _add_info_to_tx_from_wallet_and_network(self, tx: PartialTransaction) -> bool: - """Returns whether successful.""" - # note side-effect: tx is being mutated - assert isinstance(tx, PartialTransaction) - try: - # note: this might download input utxos over network - # FIXME network code in gui thread... - tx.add_info_from_wallet(self.wallet, ignore_network_issues=False) - except NetworkException as e: - self.app.show_error(repr(e)) - return False - return True - def do_rbf(self): from .bump_fee_dialog import BumpFeeDialog tx = self.tx @@ -221,7 +210,7 @@ class TxDialog(Factory.Popup): assert txid if not isinstance(tx, PartialTransaction): tx = PartialTransaction.from_tx(tx) - if not self._add_info_to_tx_from_wallet_and_network(tx): + if not tx.add_info_from_wallet_and_network(wallet=self.wallet, show_error=self.app.show_error): return fee = tx.get_fee() assert fee is not None @@ -295,7 +284,7 @@ class TxDialog(Factory.Popup): assert txid if not isinstance(tx, PartialTransaction): tx = PartialTransaction.from_tx(tx) - if not self._add_info_to_tx_from_wallet_and_network(tx): + if not tx.add_info_from_wallet_and_network(wallet=self.wallet, show_error=self.app.show_error): return fee = tx.get_fee() assert fee is not None diff --git a/electrum/gui/qml/qetransactionlistmodel.py b/electrum/gui/qml/qetransactionlistmodel.py index 33f2f4bde..9081af66d 100644 --- a/electrum/gui/qml/qetransactionlistmodel.py +++ b/electrum/gui/qml/qetransactionlistmodel.py @@ -1,4 +1,5 @@ from datetime import datetime, timedelta +from typing import TYPE_CHECKING from PyQt5.QtCore import pyqtProperty, pyqtSignal, pyqtSlot, QObject from PyQt5.QtCore import Qt, QAbstractListModel, QModelIndex @@ -9,6 +10,10 @@ from electrum.util import Satoshis, TxMinedInfo from .qetypes import QEAmount from .util import QtEventListener, qt_event_listener +if TYPE_CHECKING: + from electrum.wallet import Abstract_Wallet + + class QETransactionListModel(QAbstractListModel, QtEventListener): _logger = get_logger(__name__) @@ -22,7 +27,7 @@ class QETransactionListModel(QAbstractListModel, QtEventListener): requestRefresh = pyqtSignal() - def __init__(self, wallet, parent=None, *, onchain_domain=None, include_lightning=True): + def __init__(self, wallet: 'Abstract_Wallet', parent=None, *, onchain_domain=None, include_lightning=True): super().__init__(parent) self.wallet = wallet self.onchain_domain = onchain_domain @@ -101,7 +106,8 @@ class QETransactionListModel(QAbstractListModel, QtEventListener): item['balance'] = QEAmount(amount_sat=item['balance'].value) if 'txid' in item: - tx = self.wallet.get_input_tx(item['txid']) + tx = self.wallet.db.get_transaction(item['txid']) + assert tx is not None item['complete'] = tx.is_complete() # newly arriving txs, or (partially/fully signed) local txs have no (block) timestamp diff --git a/electrum/gui/qml/qetxdetails.py b/electrum/gui/qml/qetxdetails.py index 8da45ca64..54e3cab05 100644 --- a/electrum/gui/qml/qetxdetails.py +++ b/electrum/gui/qml/qetxdetails.py @@ -1,9 +1,12 @@ +from typing import Optional + from PyQt5.QtCore import pyqtProperty, pyqtSignal, pyqtSlot, QObject from electrum.i18n import _ from electrum.logging import get_logger from electrum.util import format_time, AddTransactionException from electrum.transaction import tx_from_any +from electrum.network import Network from .qewallet import QEWallet from .qetypes import QEAmount @@ -23,7 +26,7 @@ class QETxDetails(QObject, QtEventListener): self.register_callbacks() self.destroyed.connect(lambda: self.on_destroy()) - self._wallet = None + self._wallet = None # type: Optional[QEWallet] self._txid = '' self._rawtx = '' self._label = '' @@ -229,13 +232,16 @@ class QETxDetails(QObject, QtEventListener): return if not self._rawtx: - # abusing get_input_tx to get tx from txid - self._tx = self._wallet.wallet.get_input_tx(self._txid) + self._tx = self._wallet.wallet.db.get_transaction(self._txid) + assert self._tx is not None #self._logger.debug(repr(self._tx.to_json())) self._logger.debug('adding info from wallet') self._tx.add_info_from_wallet(self._wallet.wallet) + if not self._tx.is_complete() and self._tx.is_missing_info_from_network(): + Network.run_from_another_thread( + self._tx.add_info_from_network(self._wallet.wallet.network, timeout=10)) # FIXME is this needed?... self._inputs = list(map(lambda x: x.to_json(), self._tx.inputs())) self._outputs = list(map(lambda x: { diff --git a/electrum/gui/qml/qetxfinalizer.py b/electrum/gui/qml/qetxfinalizer.py index bf5bda24f..2a53f2416 100644 --- a/electrum/gui/qml/qetxfinalizer.py +++ b/electrum/gui/qml/qetxfinalizer.py @@ -494,7 +494,7 @@ class QETxRbfFeeBumper(TxFeeSlider, TxMonMixin): def get_tx(self): assert self._txid - self._orig_tx = self._wallet.wallet.get_input_tx(self._txid) + self._orig_tx = self._wallet.wallet.db.get_transaction(self._txid) assert self._orig_tx if self._wallet.wallet.get_swap_by_funding_tx(self._orig_tx): @@ -504,7 +504,7 @@ class QETxRbfFeeBumper(TxFeeSlider, TxMonMixin): if not isinstance(self._orig_tx, PartialTransaction): self._orig_tx = PartialTransaction.from_tx(self._orig_tx) - if not self._add_info_to_tx_from_wallet_and_network(self._orig_tx): + if not self._orig_tx.add_info_from_wallet_and_network(wallet=self._wallet.wallet, show_error=self._logger.error): return self.update_from_tx(self._orig_tx) @@ -513,21 +513,6 @@ class QETxRbfFeeBumper(TxFeeSlider, TxMonMixin): self.oldfeeRate = self.feeRate self.update() - # TODO: duplicated from kivy gui, candidate for moving into backend wallet - def _add_info_to_tx_from_wallet_and_network(self, tx: PartialTransaction) -> bool: - """Returns whether successful.""" - # note side-effect: tx is being mutated - assert isinstance(tx, PartialTransaction) - try: - # note: this might download input utxos over network - # FIXME network code in gui thread... - tx.add_info_from_wallet(self._wallet.wallet, ignore_network_issues=False) - except NetworkException as e: - # self.app.show_error(repr(e)) - self._logger.error(repr(e)) - return False - return True - def update(self): if not self._txid: # not initialized yet @@ -616,13 +601,13 @@ class QETxCanceller(TxFeeSlider, TxMonMixin): def get_tx(self): assert self._txid - self._orig_tx = self._wallet.wallet.get_input_tx(self._txid) + self._orig_tx = self._wallet.wallet.db.get_transaction(self._txid) assert self._orig_tx if not isinstance(self._orig_tx, PartialTransaction): self._orig_tx = PartialTransaction.from_tx(self._orig_tx) - if not self._add_info_to_tx_from_wallet_and_network(self._orig_tx): + if not self._orig_tx.add_info_from_wallet_and_network(wallet=self._wallet.wallet, show_error=self._logger.error): return self.update_from_tx(self._orig_tx) @@ -631,21 +616,6 @@ class QETxCanceller(TxFeeSlider, TxMonMixin): self.oldfeeRate = self.feeRate self.update() - # TODO: duplicated from kivy gui, candidate for moving into backend wallet - def _add_info_to_tx_from_wallet_and_network(self, tx: PartialTransaction) -> bool: - """Returns whether successful.""" - # note side-effect: tx is being mutated - assert isinstance(tx, PartialTransaction) - try: - # note: this might download input utxos over network - # FIXME network code in gui thread... - tx.add_info_from_wallet(self._wallet.wallet, ignore_network_issues=False) - except NetworkException as e: - # self.app.show_error(repr(e)) - self._logger.error(repr(e)) - return False - return True - def update(self): if not self._txid: # not initialized yet @@ -757,7 +727,7 @@ class QETxCpfpFeeBumper(TxFeeSlider, TxMonMixin): def get_tx(self): assert self._txid - self._parent_tx = self._wallet.wallet.get_input_tx(self._txid) + self._parent_tx = self._wallet.wallet.db.get_transaction(self._txid) assert self._parent_tx if isinstance(self._parent_tx, PartialTransaction): diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index 4545b2353..0de92a984 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -2668,27 +2668,11 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger, QtEventListener): return self.show_transaction(new_tx) - def _add_info_to_tx_from_wallet_and_network(self, tx: PartialTransaction) -> bool: - """Returns whether successful.""" - # note side-effect: tx is being mutated - assert isinstance(tx, PartialTransaction) - try: - # note: this might download input utxos over network - BlockingWaitingDialog( - self, - _("Adding info to tx, from wallet and network..."), - lambda: tx.add_info_from_wallet(self.wallet, ignore_network_issues=False), - ) - except NetworkException as e: - self.show_error(repr(e)) - return False - return True - def bump_fee_dialog(self, tx: Transaction): txid = tx.txid() if not isinstance(tx, PartialTransaction): tx = PartialTransaction.from_tx(tx) - if not self._add_info_to_tx_from_wallet_and_network(tx): + if not tx.add_info_from_wallet_and_network(wallet=self.wallet, show_error=self.show_error): return d = BumpFeeDialog(main_window=self, tx=tx, txid=txid) d.run() @@ -2697,7 +2681,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger, QtEventListener): txid = tx.txid() if not isinstance(tx, PartialTransaction): tx = PartialTransaction.from_tx(tx) - if not self._add_info_to_tx_from_wallet_and_network(tx): + if not tx.add_info_from_wallet_and_network(wallet=self.wallet, show_error=self.show_error): return d = DSCancelDialog(main_window=self, tx=tx, txid=txid) d.run() diff --git a/electrum/gui/qt/rate_limiter.py b/electrum/gui/qt/rate_limiter.py new file mode 100644 index 000000000..c90d595ee --- /dev/null +++ b/electrum/gui/qt/rate_limiter.py @@ -0,0 +1,234 @@ +# Copyright (c) 2019 Calin Culianu +# Distributed under the MIT software license, see the accompanying +# file LICENCE or http://www.opensource.org/licenses/mit-license.php + +from functools import wraps +import threading +import time +import weakref + +from PyQt5.QtCore import QObject, QTimer + +from electrum.logging import Logger, get_logger + + +_logger = get_logger(__name__) + + +class RateLimiter(Logger): + ''' Manages the state of a @rate_limited decorated function, collating + multiple invocations. This class is not intented to be used directly. Instead, + use the @rate_limited decorator (for instance methods). + This state instance gets inserted into the instance attributes of the target + object wherever a @rate_limited decorator appears. + The inserted attribute is named "__FUNCNAME__RateLimiter". ''' + # some defaults + last_ts = 0.0 + timer = None + saved_args = (tuple(),dict()) + ctr = 0 + + def __init__(self, rate, ts_after, obj, func): + self.n = func.__name__ + self.qn = func.__qualname__ + self.rate = rate + self.ts_after = ts_after + self.obj = weakref.ref(obj) # keep a weak reference to the object to prevent cycles + self.func = func + Logger.__init__(self) + #self.logger.debug(f"*** Created: {func=},{obj=},{rate=}") + + def diagnostic_name(self): + return "{}:{}".format("rate_limited",self.qn) + + def kill_timer(self): + if self.timer: + #self.logger.debug("deleting timer") + try: + self.timer.stop() + self.timer.deleteLater() + except RuntimeError as e: + if 'c++ object' in str(e).lower(): + # This can happen if the attached object which actually owns + # QTimer is deleted by Qt before this call path executes. + # This call path may be executed from a queued connection in + # some circumstances, hence the crazyness (I think). + self.logger.debug("advisory: QTimer was already deleted by Qt, ignoring...") + else: + raise + finally: + self.timer = None + + @classmethod + def attr_name(cls, func): return "__{}__{}".format(func.__name__, cls.__name__) + + @classmethod + def invoke(cls, rate, ts_after, func, args, kwargs): + ''' Calls _invoke() on an existing RateLimiter object (or creates a new + one for the given function on first run per target object instance). ''' + assert args and isinstance(args[0], object), "@rate_limited decorator may only be used with object instance methods" + assert threading.current_thread() is threading.main_thread(), "@rate_limited decorator may only be used with functions called in the main thread" + obj = args[0] + a_name = cls.attr_name(func) + #_logger.debug(f"*** {a_name=}, {obj=}") + rl = getattr(obj, a_name, None) # we hide the RateLimiter state object in an attribute (name based on the wrapped function name) in the target object + if rl is None: + # must be the first invocation, create a new RateLimiter state instance. + rl = cls(rate, ts_after, obj, func) + setattr(obj, a_name, rl) + return rl._invoke(args, kwargs) + + def _invoke(self, args, kwargs): + self._push_args(args, kwargs) # since we're collating, save latest invocation's args unconditionally. any future invocation will use the latest saved args. + self.ctr += 1 # increment call counter + #self.logger.debug(f"args_saved={args}, kwarg_saved={kwargs}") + if not self.timer: # check if there's a pending invocation already + now = time.time() + diff = float(self.rate) - (now - self.last_ts) + if diff <= 0: + # Time since last invocation was greater than self.rate, so call the function directly now. + #self.logger.debug("calling directly") + return self._doIt() + else: + # Time since last invocation was less than self.rate, so defer to the future with a timer. + self.timer = QTimer(self.obj() if isinstance(self.obj(), QObject) else None) + self.timer.timeout.connect(self._doIt) + #self.timer.destroyed.connect(lambda x=None,qn=self.qn: print(qn,"Timer deallocated")) + self.timer.setSingleShot(True) + self.timer.start(int(diff*1e3)) + #self.logger.debug("deferring") + else: + # We had a timer active, which means as future call will occur. So return early and let that call happenin the future. + # Note that a side-effect of this aborted invocation was to update self.saved_args. + pass + #self.logger.debug("ignoring (already scheduled)") + + def _pop_args(self): + args, kwargs = self.saved_args # grab the latest collated invocation's args. this attribute is always defined. + self.saved_args = (tuple(),dict()) # clear saved args immediately + return args, kwargs + + def _push_args(self, args, kwargs): + self.saved_args = (args, kwargs) + + def _doIt(self): + #self.logger.debug("called!") + t0 = time.time() + args, kwargs = self._pop_args() + #self.logger.debug(f"args_actually_used={args}, kwarg_actually_used={kwargs}") + ctr0 = self.ctr # read back current call counter to compare later for reentrancy detection + retval = self.func(*args, **kwargs) # and.. call the function. use latest invocation's args + was_reentrant = self.ctr != ctr0 # if ctr is not the same, func() led to a call this function! + del args, kwargs # deref args right away (allow them to get gc'd) + tf = time.time() + time_taken = tf-t0 + if self.ts_after: + self.last_ts = tf + else: + if time_taken > float(self.rate): + self.logger.debug(f"method took too long: {time_taken} > {self.rate}. Fudging timestamps to compensate.") + self.last_ts = tf # Hmm. This function takes longer than its rate to complete. so mark its last run time as 'now'. This breaks the rate but at least prevents this function from starving the CPU (benforces a delay). + else: + self.last_ts = t0 # Function takes less than rate to complete, so mark its t0 as when we entered to keep the rate constant. + + if self.timer: # timer is not None if and only if we were a delayed (collated) invocation. + if was_reentrant: + # we got a reentrant call to this function as a result of calling func() above! re-schedule the timer. + self.logger.debug("*** detected a re-entrant call, re-starting timer") + time_left = float(self.rate) - (tf - self.last_ts) + self.timer.start(time_left*1e3) + else: + # We did not get a reentrant call, so kill the timer so subsequent calls can schedule the timer and/or call func() immediately. + self.kill_timer() + elif was_reentrant: + self.logger.debug("*** detected a re-entrant call") + + return retval + + +class RateLimiterClassLvl(RateLimiter): + ''' This RateLimiter object is used if classlevel=True is specified to the + @rate_limited decorator. It inserts the __RateLimiterClassLvl state object + on the class level and collates calls for all instances to not exceed rate. + Each instance is guaranteed to receive at least 1 call and to have multiple + calls updated with the latest args for the final call. So for instance: + a.foo(1) + a.foo(2) + b.foo(10) + b.foo(3) + Would collate to a single 'class-level' call using 'rate': + a.foo(2) # latest arg taken, collapsed to 1 call + b.foo(3) # latest arg taken, collapsed to 1 call + ''' + + @classmethod + def invoke(cls, rate, ts_after, func, args, kwargs): + assert args and not isinstance(args[0], type), "@rate_limited decorator may not be used with static or class methods" + obj = args[0] + objcls = obj.__class__ + args = list(args) + args.insert(0, objcls) # prepend obj class to trick super.invoke() into making this state object be class-level. + return super(RateLimiterClassLvl, cls).invoke(rate, ts_after, func, args, kwargs) + + def _push_args(self, args, kwargs): + objcls, obj = args[0:2] + args = args[2:] + self.saved_args[obj] = (args, kwargs) + + def _pop_args(self): + weak_dict = self.saved_args + self.saved_args = weakref.WeakKeyDictionary() + return (weak_dict,),dict() + + def _call_func_for_all(self, weak_dict): + for ref in weak_dict.keyrefs(): + obj = ref() + if obj: + args,kwargs = weak_dict[obj] + obj_name = obj.diagnostic_name() if hasattr(obj, "diagnostic_name") else obj + #self.logger.debug(f"calling for {obj_name}, timer={bool(self.timer)}") + self.func_target(obj, *args, **kwargs) + + def __init__(self, rate, ts_after, obj, func): + # note: obj here is really the __class__ of the obj because we prepended the class in our custom invoke() above. + super().__init__(rate, ts_after, obj, func) + self.func_target = func + self.func = self._call_func_for_all + self.saved_args = weakref.WeakKeyDictionary() # we don't use a simple arg tuple, but instead an instance -> args,kwargs dictionary to store collated calls, per instance collated + + +def rate_limited(rate, *, classlevel=False, ts_after=False): + """ A Function decorator for rate-limiting GUI event callbacks. Argument + rate in seconds is the minimum allowed time between subsequent calls of + this instance of the function. Calls that arrive more frequently than + rate seconds will be collated into a single call that is deferred onto + a QTimer. It is preferable to use this decorator on QObject subclass + instance methods. This decorator is particularly useful in limiting + frequent calls to GUI update functions. + params: + rate - calls are collated to not exceed rate (in seconds) + classlevel - if True, specify that the calls should be collated at + 1 per `rate` secs. for *all* instances of a class, otherwise + calls will be collated on a per-instance basis. + ts_after - if True, mark the timestamp of the 'last call' AFTER the + target method completes. That is, the collation of calls will + ensure at least `rate` seconds will always elapse between + subsequent calls. If False, the timestamp is taken right before + the collated calls execute (thus ensuring a fixed period for + collated calls). + TL;DR: ts_after=True : `rate` defines the time interval you want + from last call's exit to entry into next + call. + ts_adter=False: `rate` defines the time between each + call's entry. + (See on_fx_quotes & on_fx_history in main_window.py for example usages + of this decorator). """ + def wrapper0(func): + @wraps(func) + def wrapper(*args, **kwargs): + if classlevel: + return RateLimiterClassLvl.invoke(rate, ts_after, func, args, kwargs) + return RateLimiter.invoke(rate, ts_after, func, args, kwargs) + return wrapper + return wrapper0 + diff --git a/electrum/gui/qt/transaction_dialog.py b/electrum/gui/qt/transaction_dialog.py index 14c53809e..1fb6ad5ce 100644 --- a/electrum/gui/qt/transaction_dialog.py +++ b/electrum/gui/qt/transaction_dialog.py @@ -23,7 +23,9 @@ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +import asyncio import sys +import concurrent.futures import copy import datetime import traceback @@ -32,7 +34,7 @@ from typing import TYPE_CHECKING, Callable, Optional, List, Union, Tuple from functools import partial from decimal import Decimal -from PyQt5.QtCore import QSize, Qt, QUrl, QPoint +from PyQt5.QtCore import QSize, Qt, QUrl, QPoint, pyqtSignal from PyQt5.QtGui import QTextCharFormat, QBrush, QFont, QPixmap, QCursor from PyQt5.QtWidgets import (QDialog, QLabel, QPushButton, QHBoxLayout, QVBoxLayout, QWidget, QGridLayout, QTextEdit, QFrame, QAction, QToolButton, QMenu, QCheckBox, QTextBrowser, QToolTip, @@ -49,8 +51,10 @@ from electrum.i18n import _ from electrum.plugin import run_hook from electrum import simple_config from electrum.transaction import SerializationError, Transaction, PartialTransaction, PartialTxInput, TxOutpoint +from electrum.transaction import TxinDataFetchProgress from electrum.logging import get_logger -from electrum.util import ShortID +from electrum.util import ShortID, get_asyncio_loop +from electrum.network import Network from .util import (MessageBoxMixin, read_QIcon, Buttons, icon_path, MONOSPACE_FONT, ColorScheme, ButtonsLineEdit, ShowQRLineEdit, text_dialog, @@ -59,6 +63,7 @@ from .util import (MessageBoxMixin, read_QIcon, Buttons, icon_path, TRANSACTION_FILE_EXTENSION_FILTER_ONLY_PARTIAL_TX, BlockingWaitingDialog, getSaveFileName, ColorSchemeItem, get_iconname_qrcode) +from .rate_limiter import rate_limited if TYPE_CHECKING: @@ -105,6 +110,11 @@ class TxInOutWidget(QWidget): self.inputs_textedit.textInteractionFlags() | Qt.LinksAccessibleByMouse | Qt.LinksAccessibleByKeyboard) self.inputs_textedit.setContextMenuPolicy(Qt.CustomContextMenu) self.inputs_textedit.customContextMenuRequested.connect(self.on_context_menu_for_inputs) + + self.inheader_hbox = QHBoxLayout() + self.inheader_hbox.setContentsMargins(0, 0, 0, 0) + self.inheader_hbox.addWidget(self.inputs_header) + self.txo_color_recv = TxOutputColoring( legend=_("Receiving Address"), color=ColorScheme.GREEN, tooltip=_("Wallet receive address")) self.txo_color_change = TxOutputColoring( @@ -129,7 +139,7 @@ class TxInOutWidget(QWidget): outheader_hbox.addWidget(self.txo_color_2fa.legend_label) vbox = QVBoxLayout() - vbox.addWidget(self.inputs_header) + vbox.addLayout(self.inheader_hbox) vbox.addWidget(self.inputs_textedit) vbox.addLayout(outheader_hbox) vbox.addWidget(self.outputs_textedit) @@ -373,6 +383,8 @@ def show_transaction(tx: Transaction, *, parent: 'ElectrumWindow', prompt_if_uns class TxDialog(QDialog, MessageBoxMixin): + throttled_update_sig = pyqtSignal() # emit from thread to do update in main thread + def __init__(self, tx: Transaction, *, parent: 'ElectrumWindow', prompt_if_unsaved, external_keypairs=None): '''Transactions in the wallet will show their description. Pass desc to give a description for txs not yet in the wallet. @@ -407,6 +419,20 @@ class TxDialog(QDialog, MessageBoxMixin): self.io_widget = TxInOutWidget(self.main_window, self.wallet) vbox.addWidget(self.io_widget) + # add "fetch_txin_data" checkbox to io_widget + fetch_txin_data_cb = QCheckBox(_('Download input data')) + fetch_txin_data_cb.setChecked(bool(self.config.get('tx_dialog_fetch_txin_data', False))) + fetch_txin_data_cb.setToolTip(_('Download parent transactions from the network.\n' + 'Allows filling in missing fee and address details.')) + def on_fetch_txin_data_cb(x): + self.config.set_key('tx_dialog_fetch_txin_data', bool(x)) + if x: + self.initiate_fetch_txin_data() + fetch_txin_data_cb.stateChanged.connect(on_fetch_txin_data_cb) + self.io_widget.inheader_hbox.addStretch(1) + self.io_widget.inheader_hbox.addWidget(fetch_txin_data_cb) + self.io_widget.inheader_hbox.addStretch(10) + self.sign_button = b = QPushButton(_("Sign")) b.clicked.connect(self.sign) @@ -460,6 +486,10 @@ class TxDialog(QDialog, MessageBoxMixin): vbox.addLayout(hbox) dialogs.append(self) + self._fetch_txin_data_fut = None # type: Optional[concurrent.futures.Future] + self._fetch_txin_data_progress = None # type: Optional[TxinDataFetchProgress] + self.throttled_update_sig.connect(self._throttled_update, Qt.QueuedConnection) + self.set_tx(tx) self.update() self.set_title() @@ -477,11 +507,18 @@ class TxDialog(QDialog, MessageBoxMixin): # As a result, e.g. we might learn an imported address tx is segwit, # or that a beyond-gap-limit address is is_mine. # note: this might fetch prev txs over the network. - BlockingWaitingDialog( - self, - _("Adding info to tx, from wallet and network..."), - lambda: tx.add_info_from_wallet(self.wallet), - ) + tx.add_info_from_wallet(self.wallet) + # FIXME for PSBTs, we do a blocking fetch, as the missing data might be needed for e.g. signing + # - otherwise, the missing data is for display-completeness only, e.g. fee, input addresses (we do it async) + if not tx.is_complete() and tx.is_missing_info_from_network(): + BlockingWaitingDialog( + self, + _("Adding info to tx, from network..."), + lambda: Network.run_from_another_thread( + tx.add_info_from_network(self.wallet.network, timeout=10)), + ) + elif self.config.get('tx_dialog_fetch_txin_data', False): + self.initiate_fetch_txin_data() def do_broadcast(self): self.main_window.push_top_level_window(self) @@ -503,6 +540,9 @@ class TxDialog(QDialog, MessageBoxMixin): dialogs.remove(self) except ValueError: pass # was not in list already + if self._fetch_txin_data_fut: + self._fetch_txin_data_fut.cancel() + self._fetch_txin_data_fut = None def reject(self): # Override escape-key to close normally (and invoke closeEvent) @@ -535,7 +575,8 @@ class TxDialog(QDialog, MessageBoxMixin): if not isinstance(self.tx, PartialTransaction): raise Exception("Can only export partial transactions for hardware device.") tx = copy.deepcopy(self.tx) - tx.prepare_for_export_for_hardware_device(self.wallet) + Network.run_from_another_thread( + tx.prepare_for_export_for_hardware_device(self.wallet)) return tx def copy_to_clipboard(self, *, tx: Transaction = None): @@ -655,6 +696,10 @@ class TxDialog(QDialog, MessageBoxMixin): return self.update() + @rate_limited(0.5, ts_after=True) + def _throttled_update(self): + self.update() + def update(self): if self.tx is None: return @@ -737,25 +782,30 @@ class TxDialog(QDialog, MessageBoxMixin): else: amount_str = _("Amount sent:") + ' %s' % format_amount(-amount) + ' ' + base_unit if fx.is_enabled(): - if tx_item_fiat: - amount_str += ' (%s)' % tx_item_fiat['fiat_value'].to_ui_string() - else: - amount_str += ' (%s)' % format_fiat_and_units(abs(amount)) + if tx_item_fiat: # historical tx -> using historical price + amount_str += ' ({})'.format(tx_item_fiat['fiat_value'].to_ui_string()) + elif tx_details.is_related_to_wallet: # probably "tx preview" -> using current price + amount_str += ' ({})'.format(format_fiat_and_units(abs(amount))) if amount_str: self.amount_label.setText(amount_str) else: self.amount_label.hide() size_str = _("Size:") + ' %d bytes'% size if fee is None: - fee_str = _("Fee") + ': ' + _("unknown") + if prog := self._fetch_txin_data_progress: + if not prog.has_errored: + fee_str = _("Downloading input data...") + f" ({prog.num_tasks_done}/{prog.num_tasks_total})" + else: + fee_str = _("Downloading input data...") + f" error." + else: + fee_str = _("Fee") + ': ' + _("unknown") else: fee_str = _("Fee") + f': {format_amount(fee)} {base_unit}' if fx.is_enabled(): - if tx_item_fiat: - fiat_fee_str = tx_item_fiat['fiat_fee'].to_ui_string() - else: - fiat_fee_str = format_fiat_and_units(fee) - fee_str += f' ({fiat_fee_str})' + if tx_item_fiat: # historical tx -> using historical price + fee_str += ' ({})'.format(tx_item_fiat['fiat_fee'].to_ui_string()) + elif tx_details.is_related_to_wallet: # probably "tx preview" -> using current price + fee_str += ' ({})'.format(format_fiat_and_units(fee)) if fee is not None: fee_rate = Decimal(fee) / size # sat/byte fee_str += ' ( %s ) ' % self.main_window.format_fee_rate(fee_rate * 1000) @@ -882,6 +932,30 @@ class TxDialog(QDialog, MessageBoxMixin): def update_fee_fields(self): pass # overridden in subclass + def initiate_fetch_txin_data(self): + """Download missing input data from the network, asynchronously. + Note: we fetch the prev txs, which allows calculating the fee and showing "input addresses". + We could also SPV-verify the tx, to fill in missing tx_mined_status (block height, blockhash, timestamp), + but this is not done currently. + """ + tx = self.tx + if not tx: + return + if self._fetch_txin_data_fut is not None: + return + network = self.wallet.network + def progress_cb(prog: TxinDataFetchProgress): + self._fetch_txin_data_progress = prog + self.throttled_update_sig.emit() + async def wrapper(): + try: + await tx.add_info_from_network(network, progress_cb=progress_cb) + finally: + self._fetch_txin_data_fut = None + + self._fetch_txin_data_progress = None + self._fetch_txin_data_fut = asyncio.run_coroutine_threadsafe(wrapper(), get_asyncio_loop()) + class TxDetailLabel(QLabel): def __init__(self, *, word_wrap=None): diff --git a/electrum/tests/test_wallet_vertical.py b/electrum/tests/test_wallet_vertical.py index 9918fa4bb..54c547359 100644 --- a/electrum/tests/test_wallet_vertical.py +++ b/electrum/tests/test_wallet_vertical.py @@ -1047,61 +1047,61 @@ class TestWalletSending(ElectrumTestCase): for simulate_moving_txs in (False, True): with TmpConfig() as config: with self.subTest(msg="_bump_fee_p2pkh_when_there_is_a_change_address", simulate_moving_txs=simulate_moving_txs): - self._bump_fee_p2pkh_when_there_is_a_change_address( + await self._bump_fee_p2pkh_when_there_is_a_change_address( simulate_moving_txs=simulate_moving_txs, config=config) with TmpConfig() as config: with self.subTest(msg="_bump_fee_p2wpkh_when_there_is_a_change_address", simulate_moving_txs=simulate_moving_txs): - self._bump_fee_p2wpkh_when_there_is_a_change_address( + await self._bump_fee_p2wpkh_when_there_is_a_change_address( simulate_moving_txs=simulate_moving_txs, config=config) with TmpConfig() as config: with self.subTest(msg="_bump_fee_p2pkh_when_there_are_two_ismine_outs_one_change_one_recv", simulate_moving_txs=simulate_moving_txs): - self._bump_fee_p2pkh_when_there_are_two_ismine_outs_one_change_one_recv( + await self._bump_fee_p2pkh_when_there_are_two_ismine_outs_one_change_one_recv( simulate_moving_txs=simulate_moving_txs, config=config) with TmpConfig() as config: with self.subTest(msg="_bump_fee_when_user_sends_max", simulate_moving_txs=simulate_moving_txs): - self._bump_fee_when_user_sends_max( + await self._bump_fee_when_user_sends_max( simulate_moving_txs=simulate_moving_txs, config=config) with TmpConfig() as config: with self.subTest(msg="_bump_fee_when_new_inputs_need_to_be_added", simulate_moving_txs=simulate_moving_txs): - self._bump_fee_when_new_inputs_need_to_be_added( + await self._bump_fee_when_new_inputs_need_to_be_added( simulate_moving_txs=simulate_moving_txs, config=config) with TmpConfig() as config: with self.subTest(msg="_bump_fee_p2wpkh_when_there_is_only_a_single_output_and_that_is_a_change_address", simulate_moving_txs=simulate_moving_txs): - self._bump_fee_p2wpkh_when_there_is_only_a_single_output_and_that_is_a_change_address( + await self._bump_fee_p2wpkh_when_there_is_only_a_single_output_and_that_is_a_change_address( simulate_moving_txs=simulate_moving_txs, config=config) with TmpConfig() as config: with self.subTest(msg="_rbf_batching", simulate_moving_txs=simulate_moving_txs): - self._rbf_batching( + await self._rbf_batching( simulate_moving_txs=simulate_moving_txs, config=config) with TmpConfig() as config: with self.subTest(msg="_bump_fee_when_not_all_inputs_are_ismine_subcase_some_outputs_are_ismine_but_not_all", simulate_moving_txs=simulate_moving_txs): - self._bump_fee_when_not_all_inputs_are_ismine_subcase_some_outputs_are_ismine_but_not_all( + await self._bump_fee_when_not_all_inputs_are_ismine_subcase_some_outputs_are_ismine_but_not_all( simulate_moving_txs=simulate_moving_txs, config=config) with TmpConfig() as config: with self.subTest(msg="_bump_fee_when_not_all_inputs_are_ismine_subcase_all_outputs_are_ismine", simulate_moving_txs=simulate_moving_txs): - self._bump_fee_when_not_all_inputs_are_ismine_subcase_all_outputs_are_ismine( + await self._bump_fee_when_not_all_inputs_are_ismine_subcase_all_outputs_are_ismine( simulate_moving_txs=simulate_moving_txs, config=config) with TmpConfig() as config: with self.subTest(msg="_bump_fee_p2wpkh_decrease_payment", simulate_moving_txs=simulate_moving_txs): - self._bump_fee_p2wpkh_decrease_payment( + await self._bump_fee_p2wpkh_decrease_payment( simulate_moving_txs=simulate_moving_txs, config=config) with TmpConfig() as config: with self.subTest(msg="_bump_fee_p2wpkh_decrease_payment_batch", simulate_moving_txs=simulate_moving_txs): - self._bump_fee_p2wpkh_decrease_payment_batch( + await self._bump_fee_p2wpkh_decrease_payment_batch( simulate_moving_txs=simulate_moving_txs, config=config) - def _bump_fee_p2pkh_when_there_is_a_change_address(self, *, simulate_moving_txs, config): + async def _bump_fee_p2pkh_when_there_is_a_change_address(self, *, simulate_moving_txs, config): wallet = self.create_standard_wallet_from_seed('fold object utility erase deputy output stadium feed stereo usage modify bean', config=config) @@ -1165,7 +1165,7 @@ class TestWalletSending(ElectrumTestCase): wallet.adb.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED) self.assertEqual((0, 7484320, 0), wallet.get_balance()) - def _bump_fee_p2pkh_when_there_are_two_ismine_outs_one_change_one_recv(self, *, simulate_moving_txs, config): + async def _bump_fee_p2pkh_when_there_are_two_ismine_outs_one_change_one_recv(self, *, simulate_moving_txs, config): """This tests a regression where sometimes we created a replacement tx that spent from the original (which is clearly invalid). """ @@ -1207,7 +1207,7 @@ class TestWalletSending(ElectrumTestCase): wallet.adb.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED) self.assertEqual((0, 461600, 0), wallet.get_balance()) - def _bump_fee_p2wpkh_decrease_payment(self, *, simulate_moving_txs, config): + async def _bump_fee_p2wpkh_decrease_payment(self, *, simulate_moving_txs, config): wallet = self.create_standard_wallet_from_seed('leader company camera enlist crash sleep insane aware anger hole hammer label', config=config) @@ -1249,7 +1249,7 @@ class TestWalletSending(ElectrumTestCase): wallet.adb.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED) self.assertEqual((0, 45000, 0), wallet.get_balance()) - def _bump_fee_p2wpkh_decrease_payment_batch(self, *, simulate_moving_txs, config): + async def _bump_fee_p2wpkh_decrease_payment_batch(self, *, simulate_moving_txs, config): wallet = self.create_standard_wallet_from_seed('leader company camera enlist crash sleep insane aware anger hole hammer label', config=config) @@ -1324,7 +1324,7 @@ class TestWalletSending(ElectrumTestCase): wallet.adb.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED) self.assertEqual((0, funding_output_value - 50000, 0), wallet.get_balance()) - def _bump_fee_p2wpkh_when_there_is_a_change_address(self, *, simulate_moving_txs, config): + async def _bump_fee_p2wpkh_when_there_is_a_change_address(self, *, simulate_moving_txs, config): wallet = self.create_standard_wallet_from_seed('frost repair depend effort salon ring foam oak cancel receive save usage', config=config) @@ -1388,13 +1388,10 @@ class TestWalletSending(ElectrumTestCase): wallet.adb.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED) self.assertEqual((0, 7490060, 0), wallet.get_balance()) - def _bump_fee_when_not_all_inputs_are_ismine_subcase_some_outputs_are_ismine_but_not_all(self, *, simulate_moving_txs, config): + async def _bump_fee_when_not_all_inputs_are_ismine_subcase_some_outputs_are_ismine_but_not_all(self, *, simulate_moving_txs, config): class NetworkMock: relay_fee = 1000 async def get_transaction(self, txid, timeout=None): - return self._gettx(txid) - @staticmethod - def _gettx(txid): if txid == "597098f9077cd2a7bf5bb2a03c9ae5fcd9d1f07c0891cb42cbb129cf9eaf57fd": return "02000000000102a5883f3de780d260e6f26cf85144403c7744a65a44cd38f9ff45aecadf010c540000000000fdffffffbdeb0175b1c51c96843d1952f7e1c49c1703717d7d020048d4de0a8eed94dad50000000000fdffffff03b2a00700000000001600140cd6c9f8ce0aa73d77fcf7f156c74f5cbec6906bb2a00700000000001600146435504ddc95e6019a90bb7dfc7ca81a88a8633106d790000000000016001444bd3017ee214370abf683abaa7f6204c9f40210024730440220652a04a2a301d9a031a034f3ae48174e204e17acf7bfc27f0dcab14243f73e2202207b29e964c434dfb2c515232d36566a40dccd4dd93ccb7fd15260ecbda10f0d9801210231994e564a0530068d17a9b0f85bec58d1352517a2861ea99e5b3070d2c5dbda02473044022072186473874919019da0e3d92b6e0aa4f88cba448ed5434615e5a3c8e2b7c42a02203ec05cef66960d5bc45d0f3d25675190cf8035b11a05ed4b719fd9c3a894899b012102f5fdca8c4e30ba0a1babf9cf9ebe62519b08aead351c349ed1ffc8316c24f542d7f61c00" else: @@ -1413,7 +1410,6 @@ class TestWalletSending(ElectrumTestCase): wallet = self.create_standard_wallet_from_seed('mix total present junior leader live state athlete mistake crack wall valve', config=config) wallet.network = NetworkMock() - wallet._get_rawtx_from_network = NetworkMock._gettx # bootstrap wallet funding_tx = Transaction('02000000000101a5883f3de780d260e6f26cf85144403c7744a65a44cd38f9ff45aecadf010c540100000000fdffffff0220a1070000000000160014db44724ac632ae47ee5765954d64796dd5fec72708de3c000000000016001424b32aadb42a89016c4de8f11741c3b29b15f21c02473044022045cc6c1cc875cbb0c0d8fe323dc1de9716e49ed5659741b0fb3dd9a196894066022077c242640071d12ec5763c5870f482a4823d8713e4bd14353dd621ed29a7f96d012102aea8d439a0f79d8b58e8d7bda83009f587e1f3da350adaa484329bf47cd03465fef61c00') @@ -1427,7 +1423,10 @@ class TestWalletSending(ElectrumTestCase): wallet.adb.receive_tx_callback(orig_rbf_txid, orig_rbf_tx, TX_HEIGHT_UNCONFIRMED) # bump tx - tx = wallet.bump_fee(tx=tx_from_any(orig_rbf_tx.serialize()), new_fee_rate=70) + orig_rbf_tx = tx_from_any(orig_rbf_tx.serialize()) + orig_rbf_tx.add_info_from_wallet(wallet=wallet) + await orig_rbf_tx.add_info_from_network(network=wallet.network) + tx = wallet.bump_fee(tx=orig_rbf_tx, new_fee_rate=70) tx.locktime = 1898268 tx.version = 2 if simulate_moving_txs: @@ -1445,13 +1444,10 @@ class TestWalletSending(ElectrumTestCase): tx_copy.serialize_as_bytes().hex()) self.assertEqual('6a8ed07cd97a10ace851b67a65035f04ff477d67cde62bb8679007e87b214e79', tx_copy.txid()) - def _bump_fee_when_not_all_inputs_are_ismine_subcase_all_outputs_are_ismine(self, *, simulate_moving_txs, config): + async def _bump_fee_when_not_all_inputs_are_ismine_subcase_all_outputs_are_ismine(self, *, simulate_moving_txs, config): class NetworkMock: relay_fee = 1000 async def get_transaction(self, txid, timeout=None): - return self._gettx(txid) - @staticmethod - def _gettx(txid): if txid == "08557327673db61cc921e1a30826608599b86457836be3021105c13940d9a9a3": return "02000000000101a5883f3de780d260e6f26cf85144403c7744a65a44cd38f9ff45aecadf010c540100000000fdffffff0220a1070000000000160014db44724ac632ae47ee5765954d64796dd5fec72708de3c000000000016001424b32aadb42a89016c4de8f11741c3b29b15f21c02473044022045cc6c1cc875cbb0c0d8fe323dc1de9716e49ed5659741b0fb3dd9a196894066022077c242640071d12ec5763c5870f482a4823d8713e4bd14353dd621ed29a7f96d012102aea8d439a0f79d8b58e8d7bda83009f587e1f3da350adaa484329bf47cd03465fef61c00" else: @@ -1473,7 +1469,6 @@ class TestWalletSending(ElectrumTestCase): gap_limit=4, ) wallet.network = NetworkMock() - wallet._get_rawtx_from_network = NetworkMock._gettx # bootstrap wallet funding_tx = Transaction('02000000000102c247447533b530cacc3e716aae84621857f04a483252374cbdccfdf8b4ef816b0000000000fdffffffc247447533b530cacc3e716aae84621857f04a483252374cbdccfdf8b4ef816b0100000000fdffffff01d63f0f00000000001600141ef4658adb12ec745a1a1fef6ab8897f04bade060247304402201dc5be86749d8ce33571a6f1a2f8bbfceba89b9dbf2b4683e66c8c17cf7df6090220729199516cb894569ebbe3e998d47fc74030231ed30f110c9babd8a9dc361115012102728251a5f5f55375eef3c14fe59ab0755ba4d5f388619895238033ac9b51aad20247304402202e5d416489c20810e96e931b98a84b0c0c4fc32d2d34d3470b7ee16810246a4c022040f86cf8030d2117d6487bbe6e23d68d6d70408b002d8055de1f33d038d3a0550121039c009e7e7dad07e74ec5a8ac9f9e3499420dd9fe9709995525c714170152512620f71c00') @@ -1487,7 +1482,10 @@ class TestWalletSending(ElectrumTestCase): wallet.adb.receive_tx_callback(orig_rbf_txid, orig_rbf_tx, TX_HEIGHT_UNCONFIRMED) # bump tx - tx = wallet.bump_fee(tx=tx_from_any(orig_rbf_tx.serialize()), new_fee_rate=50) + orig_rbf_tx = tx_from_any(orig_rbf_tx.serialize()) + orig_rbf_tx.add_info_from_wallet(wallet=wallet) + await orig_rbf_tx.add_info_from_network(network=wallet.network) + tx = wallet.bump_fee(tx=orig_rbf_tx, new_fee_rate=50) tx.locktime = 1898273 tx.version = 2 if simulate_moving_txs: @@ -1506,7 +1504,7 @@ class TestWalletSending(ElectrumTestCase): self.assertEqual('b46cdce7e7564dfd09618ab9008ec3a921c6372f3dcdab2f6094735b024485f0', tx_copy.txid()) - def _bump_fee_p2wpkh_when_there_is_only_a_single_output_and_that_is_a_change_address(self, *, simulate_moving_txs, config): + async def _bump_fee_p2wpkh_when_there_is_only_a_single_output_and_that_is_a_change_address(self, *, simulate_moving_txs, config): wallet = self.create_standard_wallet_from_seed('frost repair depend effort salon ring foam oak cancel receive save usage', config=config) @@ -1568,7 +1566,7 @@ class TestWalletSending(ElectrumTestCase): wallet.adb.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED) self.assertEqual((0, 9991750, 0), wallet.get_balance()) - def _bump_fee_when_user_sends_max(self, *, simulate_moving_txs, config): + async def _bump_fee_when_user_sends_max(self, *, simulate_moving_txs, config): wallet = self.create_standard_wallet_from_seed('frost repair depend effort salon ring foam oak cancel receive save usage', config=config) @@ -1631,7 +1629,7 @@ class TestWalletSending(ElectrumTestCase): wallet.adb.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED) self.assertEqual((0, 0, 0), wallet.get_balance()) - def _bump_fee_when_new_inputs_need_to_be_added(self, *, simulate_moving_txs, config): + async def _bump_fee_when_new_inputs_need_to_be_added(self, *, simulate_moving_txs, config): wallet = self.create_standard_wallet_from_seed('frost repair depend effort salon ring foam oak cancel receive save usage', config=config) @@ -1703,7 +1701,7 @@ class TestWalletSending(ElectrumTestCase): wallet.adb.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED) self.assertEqual((0, 4_990_300, 0), wallet.get_balance()) - def _rbf_batching(self, *, simulate_moving_txs, config): + async def _rbf_batching(self, *, simulate_moving_txs, config): wallet = self.create_standard_wallet_from_seed('frost repair depend effort salon ring foam oak cancel receive save usage', config=config) wallet.config.set_key('batch_rbf', True) @@ -2177,23 +2175,23 @@ class TestWalletSending(ElectrumTestCase): for simulate_moving_txs in (False, True): with self.subTest(msg="_dscancel_when_all_outputs_are_ismine", simulate_moving_txs=simulate_moving_txs): - self._dscancel_when_all_outputs_are_ismine( + await self._dscancel_when_all_outputs_are_ismine( simulate_moving_txs=simulate_moving_txs, config=config) with self.subTest(msg="_dscancel_p2wpkh_when_there_is_a_change_address", simulate_moving_txs=simulate_moving_txs): - self._dscancel_p2wpkh_when_there_is_a_change_address( + await self._dscancel_p2wpkh_when_there_is_a_change_address( simulate_moving_txs=simulate_moving_txs, config=config) with self.subTest(msg="_dscancel_when_user_sends_max", simulate_moving_txs=simulate_moving_txs): - self._dscancel_when_user_sends_max( + await self._dscancel_when_user_sends_max( simulate_moving_txs=simulate_moving_txs, config=config) with self.subTest(msg="_dscancel_when_not_all_inputs_are_ismine", simulate_moving_txs=simulate_moving_txs): - self._dscancel_when_not_all_inputs_are_ismine( + await self._dscancel_when_not_all_inputs_are_ismine( simulate_moving_txs=simulate_moving_txs, config=config) - def _dscancel_when_all_outputs_are_ismine(self, *, simulate_moving_txs, config): + async def _dscancel_when_all_outputs_are_ismine(self, *, simulate_moving_txs, config): wallet = self.create_standard_wallet_from_seed('fold object utility erase deputy output stadium feed stereo usage modify bean', config=config) @@ -2238,7 +2236,7 @@ class TestWalletSending(ElectrumTestCase): tx_details = wallet.get_tx_info(tx_from_any(tx.serialize())) self.assertFalse(tx_details.can_dscancel) - def _dscancel_p2wpkh_when_there_is_a_change_address(self, *, simulate_moving_txs, config): + async def _dscancel_p2wpkh_when_there_is_a_change_address(self, *, simulate_moving_txs, config): wallet = self.create_standard_wallet_from_seed('frost repair depend effort salon ring foam oak cancel receive save usage', config=config) @@ -2304,7 +2302,7 @@ class TestWalletSending(ElectrumTestCase): wallet.adb.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED) self.assertEqual((0, 9992300, 0), wallet.get_balance()) - def _dscancel_when_user_sends_max(self, *, simulate_moving_txs, config): + async def _dscancel_when_user_sends_max(self, *, simulate_moving_txs, config): wallet = self.create_standard_wallet_from_seed('frost repair depend effort salon ring foam oak cancel receive save usage', config=config) @@ -2369,13 +2367,10 @@ class TestWalletSending(ElectrumTestCase): wallet.adb.receive_tx_callback(tx.txid(), tx, TX_HEIGHT_UNCONFIRMED) self.assertEqual((0, 9992300, 0), wallet.get_balance()) - def _dscancel_when_not_all_inputs_are_ismine(self, *, simulate_moving_txs, config): + async def _dscancel_when_not_all_inputs_are_ismine(self, *, simulate_moving_txs, config): class NetworkMock: relay_fee = 1000 async def get_transaction(self, txid, timeout=None): - return self._gettx(txid) - @staticmethod - def _gettx(txid): if txid == "597098f9077cd2a7bf5bb2a03c9ae5fcd9d1f07c0891cb42cbb129cf9eaf57fd": return "02000000000102a5883f3de780d260e6f26cf85144403c7744a65a44cd38f9ff45aecadf010c540000000000fdffffffbdeb0175b1c51c96843d1952f7e1c49c1703717d7d020048d4de0a8eed94dad50000000000fdffffff03b2a00700000000001600140cd6c9f8ce0aa73d77fcf7f156c74f5cbec6906bb2a00700000000001600146435504ddc95e6019a90bb7dfc7ca81a88a8633106d790000000000016001444bd3017ee214370abf683abaa7f6204c9f40210024730440220652a04a2a301d9a031a034f3ae48174e204e17acf7bfc27f0dcab14243f73e2202207b29e964c434dfb2c515232d36566a40dccd4dd93ccb7fd15260ecbda10f0d9801210231994e564a0530068d17a9b0f85bec58d1352517a2861ea99e5b3070d2c5dbda02473044022072186473874919019da0e3d92b6e0aa4f88cba448ed5434615e5a3c8e2b7c42a02203ec05cef66960d5bc45d0f3d25675190cf8035b11a05ed4b719fd9c3a894899b012102f5fdca8c4e30ba0a1babf9cf9ebe62519b08aead351c349ed1ffc8316c24f542d7f61c00" else: @@ -2394,7 +2389,6 @@ class TestWalletSending(ElectrumTestCase): wallet = self.create_standard_wallet_from_seed('mix total present junior leader live state athlete mistake crack wall valve', config=config) wallet.network = NetworkMock() - wallet._get_rawtx_from_network = NetworkMock._gettx # bootstrap wallet funding_tx = Transaction('02000000000101a5883f3de780d260e6f26cf85144403c7744a65a44cd38f9ff45aecadf010c540100000000fdffffff0220a1070000000000160014db44724ac632ae47ee5765954d64796dd5fec72708de3c000000000016001424b32aadb42a89016c4de8f11741c3b29b15f21c02473044022045cc6c1cc875cbb0c0d8fe323dc1de9716e49ed5659741b0fb3dd9a196894066022077c242640071d12ec5763c5870f482a4823d8713e4bd14353dd621ed29a7f96d012102aea8d439a0f79d8b58e8d7bda83009f587e1f3da350adaa484329bf47cd03465fef61c00') @@ -2408,7 +2402,10 @@ class TestWalletSending(ElectrumTestCase): wallet.adb.receive_tx_callback(orig_rbf_txid, orig_rbf_tx, TX_HEIGHT_UNCONFIRMED) # bump tx - tx = wallet.dscancel(tx=tx_from_any(orig_rbf_tx.serialize()), new_fee_rate=70) + orig_rbf_tx = tx_from_any(orig_rbf_tx.serialize()) + orig_rbf_tx.add_info_from_wallet(wallet=wallet) + await orig_rbf_tx.add_info_from_network(network=wallet.network) + tx = wallet.dscancel(tx=orig_rbf_tx, new_fee_rate=70) tx.locktime = 1898278 tx.version = 2 if simulate_moving_txs: @@ -2686,7 +2683,7 @@ class TestWalletSending(ElectrumTestCase): tx.inputs()[0].to_json()['bip32_paths']) self.assertEqual("70736274ff01007d020000000122c3730eb6314cf59e11988c41bfdd73f70cb55b294ec6f2eda828b5c939c0980100000000fdffffff0240e20100000000001600147e45d43294b0ff2b08a5f45232649815e516cff058ab05000000000022002014d2823afee4d75f0f83b91a9d625972df41be222c1373d28e068c3eaae9e00a7b4a24000001012b20a10700000000002200207f50b9d6eb4d899c710d8c48903de33d966ff52445d5a57b5210d02a5dd7e3bf0100fd7e0102000000000102deab5844de4aadc177d992696fda2aa6e4692403633d31a4b4073710594d2fca0000000000fdffffffdeab5844de4aadc177d992696fda2aa6e4692403633d31a4b4073710594d2fca0100000000fdffffff02f49f070000000000160014473b34b7da0aa9f7add803019f649e0729fd39d220a10700000000002200207f50b9d6eb4d899c710d8c48903de33d966ff52445d5a57b5210d02a5dd7e3bf0247304402202a4ec3df7bf2b82505bcd4833eeb32875784b4e93d09ac3cf4a8981dc89a049b02205239bad290877fb810a12538a275d5467f3f6afc88d1e0be3d8f6dc4876e6793012103e48cae7f140e15440f4ad6b3d96cb0deb471bbb45daf527e6eb4d5f6c5e26ec802473044022031028192a8307e52829ad1428941000629de73726306ca71d18c5bcfcb98a4a602205ad0240f7dd6c83686ea257f3146ba595b787d7f68b514569962fd5d3692b07c0121033c8af340bd9abf4a56c7cf7554f52e84a1128e5206ffe5da166ca18a57a260077b4a24000105475221022c4338968f87a09b0fefd0aaac36f1b983bab237565d521944c60fdc482750492103cf9a6ac058d36a6dc325b19715a2223c6416e1cef13bc047a99bded8c99463ca52ae2206022c4338968f87a09b0fefd0aaac36f1b983bab237565d521944c60fdc48275049109559fbd10f2700800000000000000000220603cf9a6ac058d36a6dc325b19715a2223c6416e1cef13bc047a99bded8c99463ca0c015148ee000000000000000000000101475221027f7f2eaf9a44316c2cd98b67584d1e71ccaced29a347673f3364efe16f5919e221028d9b8ff374e0f60fbc698c5a494c12d9a31a3ce364b1f81ae4a46f48ae45acdd52ae2202027f7f2eaf9a44316c2cd98b67584d1e71ccaced29a347673f3364efe16f5919e2109559fbd10f27008001000000000000002202028d9b8ff374e0f60fbc698c5a494c12d9a31a3ce364b1f81ae4a46f48ae45acdd0c015148ee010000000000000000", tx.serialize_as_bytes().hex()) - tx.prepare_for_export_for_hardware_device(wallet) + await tx.prepare_for_export_for_hardware_device(wallet) # As the keystores were created from just xpubs, they are missing key origin information # (derivation prefix and root fingerprint). # Note that info for ks1 contains the expected bip32 path (m/9999') and fingerprint, but not ks0. @@ -2716,7 +2713,7 @@ class TestWalletSending(ElectrumTestCase): tx.inputs()[0].to_json()['bip32_paths']) self.assertEqual("70736274ff01007d020000000122c3730eb6314cf59e11988c41bfdd73f70cb55b294ec6f2eda828b5c939c0980100000000fdffffff0240e20100000000001600147e45d43294b0ff2b08a5f45232649815e516cff058ab05000000000022002014d2823afee4d75f0f83b91a9d625972df41be222c1373d28e068c3eaae9e00a7b4a24000001012b20a10700000000002200207f50b9d6eb4d899c710d8c48903de33d966ff52445d5a57b5210d02a5dd7e3bf0100fd7e0102000000000102deab5844de4aadc177d992696fda2aa6e4692403633d31a4b4073710594d2fca0000000000fdffffffdeab5844de4aadc177d992696fda2aa6e4692403633d31a4b4073710594d2fca0100000000fdffffff02f49f070000000000160014473b34b7da0aa9f7add803019f649e0729fd39d220a10700000000002200207f50b9d6eb4d899c710d8c48903de33d966ff52445d5a57b5210d02a5dd7e3bf0247304402202a4ec3df7bf2b82505bcd4833eeb32875784b4e93d09ac3cf4a8981dc89a049b02205239bad290877fb810a12538a275d5467f3f6afc88d1e0be3d8f6dc4876e6793012103e48cae7f140e15440f4ad6b3d96cb0deb471bbb45daf527e6eb4d5f6c5e26ec802473044022031028192a8307e52829ad1428941000629de73726306ca71d18c5bcfcb98a4a602205ad0240f7dd6c83686ea257f3146ba595b787d7f68b514569962fd5d3692b07c0121033c8af340bd9abf4a56c7cf7554f52e84a1128e5206ffe5da166ca18a57a260077b4a24000105475221022c4338968f87a09b0fefd0aaac36f1b983bab237565d521944c60fdc482750492103cf9a6ac058d36a6dc325b19715a2223c6416e1cef13bc047a99bded8c99463ca52ae2206022c4338968f87a09b0fefd0aaac36f1b983bab237565d521944c60fdc48275049109559fbd10f2700800000000000000000220603cf9a6ac058d36a6dc325b19715a2223c6416e1cef13bc047a99bded8c99463ca1c30cf1be530000080010000800000008002000080000000000000000000000101475221027f7f2eaf9a44316c2cd98b67584d1e71ccaced29a347673f3364efe16f5919e221028d9b8ff374e0f60fbc698c5a494c12d9a31a3ce364b1f81ae4a46f48ae45acdd52ae2202027f7f2eaf9a44316c2cd98b67584d1e71ccaced29a347673f3364efe16f5919e2109559fbd10f27008001000000000000002202028d9b8ff374e0f60fbc698c5a494c12d9a31a3ce364b1f81ae4a46f48ae45acdd1c30cf1be530000080010000800000008002000080010000000000000000", tx.serialize_as_bytes().hex()) - tx.prepare_for_export_for_hardware_device(wallet) + await tx.prepare_for_export_for_hardware_device(wallet) self.assertEqual( {'tpubDFF7YPCSGHZy55HkQj6HJkXCR8DWbKKXpTYBH38fSHf6VuoEzNmZQZdAoKEVy36S8zXkbGeV4XQU6vaRXGsQfgptFYPR4HSpAenqkY7J7Lg': ('30cf1be5', "m/48h/1h/0h/2h"), 'tpubD9MoDeHnEQnU5EMgt9mc4yKU6SURbfq2ooMToY5GH95B8Li1CEsuo9dBKXM2sdjuDGq4KCXLuigss3y22fZULzVrfVuZDxEN55Sp6CcU9DK': ('9559fbd1', "m/9999h")}, @@ -2754,7 +2751,7 @@ class TestWalletSending(ElectrumTestCase): self.assertEqual("70736274ff0100710200000001916fa04d7080ae0cb19bd08671d37dbe3dc925be383737bb34b3097d82830dc70000000000fdffffff0240e20100000000001600147e45d43294b0ff2b08a5f45232649815e516cff0ceaa05000000000016001456ec9cad206160ab578fa1dfbe13311b3be4a3107f4a24000001011f96a007000000000016001413ce91db66299806c4f35b2b4f8426b0bd4f2cd70100fd2e010200000000010122c3730eb6314cf59e11988c41bfdd73f70cb55b294ec6f2eda828b5c939c0980100000000fdffffff0196a007000000000016001413ce91db66299806c4f35b2b4f8426b0bd4f2cd704004730440220112840ce5486c6b2d15bc3b12e45c2a4518828e1b34f9bb0b3a78220c0cec52f02205b146a1f683289909ecbd3f53932d5acc321444101d8002e435b38a54adbf47201473044022058dfb4c75de119595119f35dcd7b1b2c28c40d7e2e746baeae83f09396c6bb9e02201c3c40fb684253638f12392af3934a90a6c6a512441aac861022f927473c952001475221022c4338968f87a09b0fefd0aaac36f1b983bab237565d521944c60fdc482750492103cf9a6ac058d36a6dc325b19715a2223c6416e1cef13bc047a99bded8c99463ca52ae4a4a24002206029e65093d22877cbfcc27cb754c58d144ec96635af1fcc63e5a7b90b23bb6acb81830cf1be5540000800100008000000080000000000000000000002202031503b2e74b21d4583b7f0d9e65b2c0ef19fd6e8aae7d0524fc770a1d2b2127501830cf1be5540000800100008000000080010000000000000000", tx.serialize_as_bytes().hex()) # if there are no multisig inputs, we never include xpubs in the psbt: - tx.prepare_for_export_for_hardware_device(wallet) + await tx.prepare_for_export_for_hardware_device(wallet) self.assertEqual({}, tx.to_json()['xpubs']) self.assertEqual("70736274ff0100710200000001916fa04d7080ae0cb19bd08671d37dbe3dc925be383737bb34b3097d82830dc70000000000fdffffff0240e20100000000001600147e45d43294b0ff2b08a5f45232649815e516cff0ceaa05000000000016001456ec9cad206160ab578fa1dfbe13311b3be4a3107f4a24000001011f96a007000000000016001413ce91db66299806c4f35b2b4f8426b0bd4f2cd70100fd2e010200000000010122c3730eb6314cf59e11988c41bfdd73f70cb55b294ec6f2eda828b5c939c0980100000000fdffffff0196a007000000000016001413ce91db66299806c4f35b2b4f8426b0bd4f2cd704004730440220112840ce5486c6b2d15bc3b12e45c2a4518828e1b34f9bb0b3a78220c0cec52f02205b146a1f683289909ecbd3f53932d5acc321444101d8002e435b38a54adbf47201473044022058dfb4c75de119595119f35dcd7b1b2c28c40d7e2e746baeae83f09396c6bb9e02201c3c40fb684253638f12392af3934a90a6c6a512441aac861022f927473c952001475221022c4338968f87a09b0fefd0aaac36f1b983bab237565d521944c60fdc482750492103cf9a6ac058d36a6dc325b19715a2223c6416e1cef13bc047a99bded8c99463ca52ae4a4a24002206029e65093d22877cbfcc27cb754c58d144ec96635af1fcc63e5a7b90b23bb6acb81830cf1be5540000800100008000000080000000000000000000002202031503b2e74b21d4583b7f0d9e65b2c0ef19fd6e8aae7d0524fc770a1d2b2127501830cf1be5540000800100008000000080010000000000000000", tx.serialize_as_bytes().hex()) diff --git a/electrum/transaction.py b/electrum/transaction.py index d9ad81fa0..5d6090e6b 100644 --- a/electrum/transaction.py +++ b/electrum/transaction.py @@ -51,11 +51,12 @@ from .bitcoin import (TYPE_ADDRESS, TYPE_SCRIPT, hash_160, base_encode, construct_witness, construct_script) from .crypto import sha256d from .logging import get_logger -from .util import ShortID +from .util import ShortID, OldTaskGroup from .descriptor import Descriptor, MissingSolutionPiece, create_dummy_descriptor_from_address if TYPE_CHECKING: from .wallet import Abstract_Wallet + from .network import Network _logger = get_logger(__name__) @@ -90,6 +91,13 @@ class MissingTxInputAmount(Exception): pass +class TxinDataFetchProgress(NamedTuple): + num_tasks_done: int + num_tasks_total: int + has_errored: bool + has_finished: bool + + class Sighash(IntEnum): # note: this is not an IntFlag, as ALL|NONE != SINGLE @@ -256,6 +264,7 @@ class TxInput: self.block_txpos = None self.spent_height = None # type: Optional[int] # height at which the TXO got spent self.spent_txid = None # type: Optional[str] # txid of the spender + self._utxo = None # type: Optional[Transaction] @property def short_id(self): @@ -264,6 +273,30 @@ class TxInput: else: return self.prevout.short_name() + @property + def utxo(self): + return self._utxo + + @utxo.setter + def utxo(self, tx: Optional['Transaction']): + if tx is None: + return + # note that tx might be a PartialTransaction + # serialize and de-serialize tx now. this might e.g. convert a complete PartialTx to a Tx + tx = tx_from_any(str(tx)) + # 'utxo' field should not be a PSBT: + if not tx.is_complete(): + return + self.validate_data(utxo=tx) + self._utxo = tx + + def validate_data(self, *, utxo: Optional['Transaction'] = None, **kwargs) -> None: + utxo = utxo or self.utxo + if utxo: + if self.prevout.txid.hex() != utxo.txid(): + raise PSBTInputConsistencyFailure(f"PSBT input validation: " + f"If a non-witness UTXO is provided, its hash must match the hash specified in the prevout") + def is_coinbase_input(self) -> bool: """Whether this is the input of a coinbase tx.""" return self.prevout.is_coinbase() @@ -275,6 +308,22 @@ class TxInput: return self._is_coinbase_output def value_sats(self) -> Optional[int]: + if self.utxo: + out_idx = self.prevout.out_idx + return self.utxo.outputs()[out_idx].value + return None + + @property + def address(self) -> Optional[str]: + if self.scriptpubkey: + return get_address_from_output_script(self.scriptpubkey) + return None + + @property + def scriptpubkey(self) -> Optional[bytes]: + if self.utxo: + out_idx = self.prevout.out_idx + return self.utxo.outputs()[out_idx].scriptpubkey return None def to_json(self): @@ -314,6 +363,35 @@ class TxInput: return True return False + async def add_info_from_network( + self, + network: Optional['Network'], + *, + ignore_network_issues: bool = True, + timeout=None, + ) -> bool: + """Returns True iff successful.""" + from .network import NetworkException + async def fetch_from_network(txid) -> Optional[Transaction]: + tx = None + if network and network.has_internet_connection(): + try: + raw_tx = await network.get_transaction(txid, timeout=timeout) + except NetworkException as e: + _logger.info(f'got network error getting input txn. err: {repr(e)}. txid: {txid}. ' + f'if you are intentionally offline, consider using the --offline flag') + if not ignore_network_issues: + raise e + else: + tx = Transaction(raw_tx) + if not tx and not ignore_network_issues: + raise NetworkException('failed to get prev tx from network') + return tx + + if self.utxo is None: + self.utxo = await fetch_from_network(txid=self.prevout.txid.hex()) + return self.utxo is not None + class BCDataStream(object): """Workalike python implementation of Bitcoin's CDataStream class.""" @@ -895,7 +973,75 @@ class Transaction: return sha256d(bfh(ser))[::-1].hex() def add_info_from_wallet(self, wallet: 'Abstract_Wallet', **kwargs) -> None: - return # no-op + # populate prev_txs + for txin in self.inputs(): + wallet.add_input_info(txin) + + async def add_info_from_network( + self, + network: Optional['Network'], + *, + ignore_network_issues: bool = True, + progress_cb: Callable[[TxinDataFetchProgress], None] = None, + timeout=None, + ) -> None: + """note: it is recommended to call add_info_from_wallet first, as this can save some network requests""" + if not self.is_missing_info_from_network(): + return + if progress_cb is None: + progress_cb = lambda *args, **kwargs: None + num_tasks_done = 0 + num_tasks_total = 0 + has_errored = False + has_finished = False + async def add_info_to_txin(txin: TxInput): + nonlocal num_tasks_done, has_errored + progress_cb(TxinDataFetchProgress(num_tasks_done, num_tasks_total, has_errored, has_finished)) + success = await txin.add_info_from_network( + network=network, + ignore_network_issues=ignore_network_issues, + timeout=timeout, + ) + if success: + num_tasks_done += 1 + else: + has_errored = True + progress_cb(TxinDataFetchProgress(num_tasks_done, num_tasks_total, has_errored, has_finished)) + # schedule a network task for each txin + try: + async with OldTaskGroup() as group: + for txin in self.inputs(): + if txin.utxo is None: + num_tasks_total += 1 + await group.spawn(add_info_to_txin(txin=txin)) + except Exception as e: + has_errored = True + _logger.error(f"tx.add_info_from_network() got exc: {e!r}") + finally: + has_finished = True + progress_cb(TxinDataFetchProgress(num_tasks_done, num_tasks_total, has_errored, has_finished)) + + def is_missing_info_from_network(self) -> bool: + return any(txin.utxo is None for txin in self.inputs()) + + def add_info_from_wallet_and_network( + self, *, wallet: 'Abstract_Wallet', show_error: Callable[[str], None], + ) -> bool: + """Returns whether successful. + note: This is sort of a legacy hack... doing network requests in non-async code. + Relatedly, this should *not* be called from the network thread. + """ + # note side-effect: tx is being mutated + from .network import NetworkException + self.add_info_from_wallet(wallet) + try: + if self.is_missing_info_from_network(): + Network.run_from_another_thread( + self.add_info_from_network(wallet.network, ignore_network_issues=False)) + except NetworkException as e: + show_error(repr(e)) + return False + return True def is_final(self) -> bool: """Whether RBF is disabled.""" @@ -1004,6 +1150,21 @@ class Transaction: else: raise Exception('output not found', addr) + def input_value(self) -> int: + input_values = [txin.value_sats() for txin in self.inputs()] + if any([val is None for val in input_values]): + raise MissingTxInputAmount() + return sum(input_values) + + def output_value(self) -> int: + return sum(o.value for o in self.outputs()) + + def get_fee(self) -> Optional[int]: + try: + return self.input_value() - self.output_value() + except MissingTxInputAmount: + return None + def get_input_idx_that_spent_prevout(self, prevout: TxOutpoint) -> Optional[int]: # build cache if there isn't one yet # note: can become stale and return incorrect data @@ -1177,7 +1338,6 @@ class PSBTSection: class PartialTxInput(TxInput, PSBTSection): def __init__(self, *args, **kwargs): TxInput.__init__(self, *args, **kwargs) - self._utxo = None # type: Optional[Transaction] self._witness_utxo = None # type: Optional[TxOutput] self.part_sigs = {} # type: Dict[bytes, bytes] # pubkey -> sig self.sighash = None # type: Optional[int] @@ -1193,23 +1353,6 @@ class PartialTxInput(TxInput, PSBTSection): self._is_native_segwit = None # type: Optional[bool] # None means unknown self.witness_sizehint = None # type: Optional[int] # byte size of serialized complete witness, for tx size est - @property - def utxo(self): - return self._utxo - - @utxo.setter - def utxo(self, tx: Optional[Transaction]): - if tx is None: - return - # note that tx might be a PartialTransaction - # serialize and de-serialize tx now. this might e.g. convert a complete PartialTx to a Tx - tx = tx_from_any(str(tx)) - # 'utxo' field in PSBT cannot be another PSBT: - if not tx.is_complete(): - return - self.validate_data(utxo=tx) - self._utxo = tx - @property def witness_utxo(self): return self._witness_utxo @@ -1268,6 +1411,7 @@ class PartialTxInput(TxInput, PSBTSection): nsequence=txin.nsequence, witness=None if strip_witness else txin.witness, is_coinbase_output=txin.is_coinbase_output()) + res.utxo = txin.utxo return res def validate_data( @@ -1397,31 +1541,28 @@ class PartialTxInput(TxInput, PSBTSection): wr(key_type, val, key=key) def value_sats(self) -> Optional[int]: + if (val := super().value_sats()) is not None: + return val if self._trusted_value_sats is not None: return self._trusted_value_sats - if self.utxo: - out_idx = self.prevout.out_idx - return self.utxo.outputs()[out_idx].value if self.witness_utxo: return self.witness_utxo.value return None @property def address(self) -> Optional[str]: + if (addr := super().address) is not None: + return addr if self._trusted_address is not None: return self._trusted_address - scriptpubkey = self.scriptpubkey - if scriptpubkey: - return get_address_from_output_script(scriptpubkey) return None @property def scriptpubkey(self) -> Optional[bytes]: + if (spk := super().scriptpubkey) is not None: + return spk if self._trusted_address is not None: return bfh(bitcoin.address_to_script(self._trusted_address)) - if self.utxo: - out_idx = self.prevout.out_idx - return self.utxo.outputs()[out_idx].scriptpubkey if self.witness_utxo: return self.witness_utxo.scriptpubkey return None @@ -1886,21 +2027,6 @@ class PartialTransaction(Transaction): self._outputs.sort(key = lambda o: (o.value, o.scriptpubkey)) self.invalidate_ser_cache() - def input_value(self) -> int: - input_values = [txin.value_sats() for txin in self.inputs()] - if any([val is None for val in input_values]): - raise MissingTxInputAmount() - return sum(input_values) - - def output_value(self) -> int: - return sum(o.value for o in self.outputs()) - - def get_fee(self) -> Optional[int]: - try: - return self.input_value() - self.output_value() - except MissingTxInputAmount: - return None - def serialize_preimage(self, txin_index: int, *, bip143_shared_txdigest_fields: BIP143SharedTxDigestFields = None) -> str: nVersion = int_to_hex(self.version, 4) @@ -2052,7 +2178,6 @@ class PartialTransaction(Transaction): wallet: 'Abstract_Wallet', *, include_xpubs: bool = False, - ignore_network_issues: bool = True, ) -> None: if self.is_complete(): return @@ -2074,7 +2199,6 @@ class PartialTransaction(Transaction): wallet.add_input_info( txin, only_der_suffix=False, - ignore_network_issues=ignore_network_issues, ) for txout in self.outputs(): wallet.add_output_info( @@ -2104,8 +2228,9 @@ class PartialTransaction(Transaction): txout.bip32_paths.clear() txout._unknown.clear() - def prepare_for_export_for_hardware_device(self, wallet: 'Abstract_Wallet') -> None: + async def prepare_for_export_for_hardware_device(self, wallet: 'Abstract_Wallet') -> None: self.add_info_from_wallet(wallet, include_xpubs=True) + await self.add_info_from_network(wallet.network) # log warning if PSBT_*_BIP32_DERIVATION fields cannot be filled with full path due to missing info from .keystore import Xpub def is_ks_missing_info(ks): diff --git a/electrum/wallet.py b/electrum/wallet.py index 17f083f21..af42eba51 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -277,6 +277,7 @@ class TxWalletDetails(NamedTuple): mempool_depth_bytes: Optional[int] can_remove: bool # whether user should be allowed to delete tx is_lightning_funding_tx: bool + is_related_to_wallet: bool class Abstract_Wallet(ABC, Logger, EventListener): @@ -862,6 +863,7 @@ class Abstract_Wallet(ABC, Logger, EventListener): mempool_depth_bytes=exp_n, can_remove=can_remove, is_lightning_funding_tx=is_lightning_funding_tx, + is_related_to_wallet=is_relevant, ) def get_tx_parents(self, txid) -> Dict: @@ -1861,6 +1863,9 @@ class Abstract_Wallet(ABC, Logger, EventListener): """Increase the miner fee of 'tx'. 'new_fee_rate' is the target min rate in sat/vbyte 'coins' is a list of UTXOs we can choose from as potential new inputs to be added + + note: it is the caller's responsibility to have already called tx.add_info_from_network(). + Without that, all txins must be ismine. """ txid = txid or tx.txid() assert txid @@ -1872,11 +1877,9 @@ class Abstract_Wallet(ABC, Logger, EventListener): if tx.is_final(): raise CannotBumpFee(_('Transaction is final')) new_fee_rate = quantize_feerate(new_fee_rate) # strip excess precision - try: - # note: this might download input utxos over network - tx.add_info_from_wallet(self, ignore_network_issues=False) - except NetworkException as e: - raise CannotBumpFee(repr(e)) + tx.add_info_from_wallet(self) + if tx.is_missing_info_from_network(): + raise Exception("tx missing info from network") old_tx_size = tx.estimated_size() old_fee = tx.get_fee() assert old_fee is not None @@ -2123,6 +2126,9 @@ class Abstract_Wallet(ABC, Logger, EventListener): """Double-Spend-Cancel: cancel an unconfirmed tx by double-spending its inputs, paying ourselves. 'new_fee_rate' is the target min rate in sat/vbyte + + note: it is the caller's responsibility to have already called tx.add_info_from_network(). + Without that, all txins must be ismine. """ if not isinstance(tx, PartialTransaction): tx = PartialTransaction.from_tx(tx) @@ -2132,11 +2138,9 @@ class Abstract_Wallet(ABC, Logger, EventListener): if tx.is_final(): raise CannotDoubleSpendTx(_('Transaction is final')) new_fee_rate = quantize_feerate(new_fee_rate) # strip excess precision - try: - # note: this might download input utxos over network - tx.add_info_from_wallet(self, ignore_network_issues=False) - except NetworkException as e: - raise CannotDoubleSpendTx(repr(e)) + tx.add_info_from_wallet(self) + if tx.is_missing_info_from_network(): + raise Exception("tx missing info from network") old_tx_size = tx.estimated_size() old_fee = tx.get_fee() assert old_fee is not None @@ -2178,7 +2182,6 @@ class Abstract_Wallet(ABC, Logger, EventListener): txin: PartialTxInput, *, address: str = None, - ignore_network_issues: bool = True, ) -> None: # - We prefer to include UTXO (full tx), even for segwit inputs (see #6198). # - For witness v0 inputs, we include *both* UTXO and WITNESS_UTXO. UTXO is a strict superset, @@ -2194,7 +2197,7 @@ class Abstract_Wallet(ABC, Logger, EventListener): txin_value = item[2] txin.witness_utxo = TxOutput.from_address_and_value(address, txin_value) if txin.utxo is None: - txin.utxo = self.get_input_tx(txin.prevout.txid.hex(), ignore_network_issues=ignore_network_issues) + txin.utxo = self.db.get_transaction(txin.prevout.txid.hex()) def _learn_derivation_path_for_address_from_txinout(self, txinout: Union[PartialTxInput, PartialTxOutput], address: str) -> bool: @@ -2206,14 +2209,21 @@ class Abstract_Wallet(ABC, Logger, EventListener): def add_input_info( self, - txin: PartialTxInput, + txin: TxInput, *, only_der_suffix: bool = False, - ignore_network_issues: bool = True, ) -> None: - address = self.adb.get_txin_address(txin) + """Populates the txin, using info the wallet already has. + That is, network requests are *not* done to fetch missing prev txs! + For that, use txin.add_info_from_network. + """ # note: we add input utxos regardless of is_mine - self._add_input_utxo_info(txin, ignore_network_issues=ignore_network_issues, address=address) + if txin.utxo is None: + txin.utxo = self.db.get_transaction(txin.prevout.txid.hex()) + if not isinstance(txin, PartialTxInput): + return + address = self.adb.get_txin_address(txin) + self._add_input_utxo_info(txin, address=address) is_mine = self.is_mine(address) if not is_mine: is_mine = self._learn_derivation_path_for_address_from_txinout(txin, address) @@ -2279,31 +2289,6 @@ class Abstract_Wallet(ABC, Logger, EventListener): return True return False - def _get_rawtx_from_network(self, txid: str) -> str: - """legacy hack. do not use in new code.""" - assert self.network - return self.network.run_from_another_thread( - self.network.get_transaction(txid, timeout=10)) - - def get_input_tx(self, tx_hash: str, *, ignore_network_issues=False) -> Optional[Transaction]: - # First look up an input transaction in the wallet where it - # will likely be. If co-signing a transaction it may not have - # all the input txs, in which case we ask the network. - tx = self.db.get_transaction(tx_hash) - if not tx and self.network and self.network.has_internet_connection(): - try: - raw_tx = self._get_rawtx_from_network(tx_hash) - except NetworkException as e: - _logger.info(f'got network error getting input txn. err: {repr(e)}. txid: {tx_hash}. ' - f'if you are intentionally offline, consider using the --offline flag') - if not ignore_network_issues: - raise e - else: - tx = Transaction(raw_tx) - if not tx and not ignore_network_issues: - raise NetworkException('failed to get prev tx from network') - return tx - def add_output_info(self, txout: PartialTxOutput, *, only_der_suffix: bool = False) -> None: address = txout.address if not self.is_mine(address):