diff --git a/gui/qt/transaction_dialog.py b/gui/qt/transaction_dialog.py index c8127f5a8..4774731db 100644 --- a/gui/qt/transaction_dialog.py +++ b/gui/qt/transaction_dialog.py @@ -83,7 +83,7 @@ class TxDialog(QDialog, MessageBoxMixin): self.saved = False self.desc = desc - self.setMinimumWidth(750) + self.setMinimumWidth(950) self.setWindowTitle(_("Transaction")) vbox = QVBoxLayout() @@ -293,15 +293,10 @@ class TxDialog(QDialog, MessageBoxMixin): else: prevout_hash = x.get('prevout_hash') prevout_n = x.get('prevout_n') - cursor.insertText(prevout_hash[0:8] + '...', ext) - cursor.insertText(prevout_hash[-8:] + ":%-4d " % prevout_n, ext) - addr = x.get('address') - if addr == "(pubkey)": - _addr = self.wallet.get_txin_address(x) - if _addr: - addr = _addr + cursor.insertText(prevout_hash + ":%-4d " % prevout_n, ext) + addr = self.wallet.get_txin_address(x) if addr is None: - addr = _('unknown') + addr = '' cursor.insertText(addr, text_format(addr)) if x.get('value'): cursor.insertText(format_amount(x['value']), ext) diff --git a/lib/storage.py b/lib/storage.py index 06ef33175..6a1c22a2e 100644 --- a/lib/storage.py +++ b/lib/storage.py @@ -32,7 +32,9 @@ import stat import pbkdf2, hmac, hashlib import base64 import zlib +from collections import defaultdict +from . import util from .util import PrintError, profiler, InvalidPassword, WalletFileException, bfh from .plugins import run_hook, plugin_loaders from .keystore import bip44_derivation @@ -44,7 +46,7 @@ from . import ecc OLD_SEED_VERSION = 4 # electrum versions < 2.0 NEW_SEED_VERSION = 11 # electrum versions >= 2.0 -FINAL_SEED_VERSION = 16 # electrum >= 2.7 will set this to prevent +FINAL_SEED_VERSION = 17 # electrum >= 2.7 will set this to prevent # old versions from overwriting new format @@ -225,8 +227,8 @@ class WalletStorage(PrintError): def put(self, key, value): try: - json.dumps(key) - json.dumps(value) + json.dumps(key, cls=util.MyEncoder) + json.dumps(value, cls=util.MyEncoder) except: self.print_error("json error: cannot save", key) return @@ -250,7 +252,7 @@ class WalletStorage(PrintError): return if not self.modified: return - s = json.dumps(self.data, indent=4, sort_keys=True) + s = json.dumps(self.data, indent=4, sort_keys=True, cls=util.MyEncoder) if self.pubkey: s = bytes(s, 'utf8') c = zlib.compress(s) @@ -329,6 +331,7 @@ class WalletStorage(PrintError): def requires_upgrade(self): return self.file_exists() and self.get_seed_version() < FINAL_SEED_VERSION + @profiler def upgrade(self): self.print_error('upgrading wallet format') @@ -339,6 +342,7 @@ class WalletStorage(PrintError): self.convert_version_14() self.convert_version_15() self.convert_version_16() + self.convert_version_17() self.put('seed_version', FINAL_SEED_VERSION) # just to be sure self.write() @@ -531,6 +535,28 @@ class WalletStorage(PrintError): self.put('seed_version', 16) + def convert_version_17(self): + # delete pruned_txo; construct spent_outpoints + if not self._is_upgrade_method_needed(16, 16): + return + + self.put('pruned_txo', None) + + from .transaction import Transaction + transactions = self.get('transactions', {}) # txid -> raw_tx + spent_outpoints = defaultdict(dict) + for txid, raw_tx in transactions.items(): + tx = Transaction(raw_tx) + for txin in tx.inputs(): + if txin['type'] == 'coinbase': + continue + prevout_hash = txin['prevout_hash'] + prevout_n = txin['prevout_n'] + spent_outpoints[prevout_hash][prevout_n] = txid + self.put('spent_outpoints', spent_outpoints) + + self.put('seed_version', 17) + def convert_imported(self): if not self._is_upgrade_method_needed(0, 13): return diff --git a/lib/util.py b/lib/util.py index 81e3a6dae..dbe5bef77 100644 --- a/lib/util.py +++ b/lib/util.py @@ -158,6 +158,8 @@ class MyEncoder(json.JSONEncoder): return str(obj) if isinstance(obj, datetime): return obj.isoformat(' ')[:-3] + if isinstance(obj, set): + return list(obj) return super(MyEncoder, self).default(obj) class PrintError(object): diff --git a/lib/wallet.py b/lib/wallet.py index b75a027b9..8d5182dd9 100644 --- a/lib/wallet.py +++ b/lib/wallet.py @@ -106,7 +106,7 @@ def append_utxos_to_inputs(inputs, network, pubkey, txin_type, imax): item['address'] = address item['type'] = txin_type item['prevout_hash'] = item['tx_hash'] - item['prevout_n'] = item['tx_pos'] + item['prevout_n'] = int(item['tx_pos']) item['pubkeys'] = [pubkey] item['x_pubkeys'] = [pubkey] item['signatures'] = [None] @@ -170,11 +170,6 @@ class UnrelatedTransactionException(AddTransactionException): return _("Transaction is unrelated to this wallet.") -class NotIsMineTransactionException(AddTransactionException): - def __str__(self): - return _("Only transactions with inputs owned by the wallet can be added.") - - class Abstract_Wallet(PrintError): """ Wallet classes are created to handle various address generation methods. @@ -216,7 +211,6 @@ class Abstract_Wallet(PrintError): self.test_addresses_sanity() self.load_transactions() self.load_local_history() - self.build_spent_outpoints() self.check_history() self.load_unverified_transactions() self.remove_local_transactions_we_dont_have() @@ -249,19 +243,29 @@ class Abstract_Wallet(PrintError): @profiler def load_transactions(self): + # load txi, txo, tx_fees self.txi = self.storage.get('txi', {}) + for txid, d in list(self.txi.items()): + for addr, lst in d.items(): + self.txi[txid][addr] = set([tuple(x) for x in lst]) self.txo = self.storage.get('txo', {}) self.tx_fees = self.storage.get('tx_fees', {}) - self.pruned_txo = self.storage.get('pruned_txo', {}) tx_list = self.storage.get('transactions', {}) + # load transactions self.transactions = {} for tx_hash, raw in tx_list.items(): tx = Transaction(raw) self.transactions[tx_hash] = tx - if self.txi.get(tx_hash) is None and self.txo.get(tx_hash) is None \ - and (tx_hash not in self.pruned_txo.values()): + if self.txi.get(tx_hash) is None and self.txo.get(tx_hash) is None: self.print_error("removing unreferenced tx", tx_hash) self.transactions.pop(tx_hash) + # load spent_outpoints + _spent_outpoints = self.storage.get('spent_outpoints', {}) + self.spent_outpoints = defaultdict(dict) + for prevout_hash, d in _spent_outpoints.items(): + for prevout_n_str, spending_txid in d.items(): + prevout_n = int(prevout_n_str) + self.spent_outpoints[prevout_hash][prevout_n] = spending_txid @profiler def load_local_history(self): @@ -286,8 +290,8 @@ class Abstract_Wallet(PrintError): self.storage.put('txi', self.txi) self.storage.put('txo', self.txo) self.storage.put('tx_fees', self.tx_fees) - self.storage.put('pruned_txo', self.pruned_txo) self.storage.put('addr_history', self.history) + self.storage.put('spent_outpoints', self.spent_outpoints) if write: self.storage.write() @@ -303,21 +307,12 @@ class Abstract_Wallet(PrintError): self.txi = {} self.txo = {} self.tx_fees = {} - self.pruned_txo = {} - self.spent_outpoints = {} + self.spent_outpoints = defaultdict(dict) self.history = {} self.verified_tx = {} self.transactions = {} self.save_transactions() - @profiler - def build_spent_outpoints(self): - self.spent_outpoints = {} - for txid, items in self.txi.items(): - for addr, l in items.items(): - for ser, v in l: - self.spent_outpoints[ser] = txid - @profiler def check_history(self): save = False @@ -333,7 +328,7 @@ class Abstract_Wallet(PrintError): hist = self.history[addr] for tx_hash, tx_height in hist: - if tx_hash in self.pruned_txo.values() or self.txi.get(tx_hash) or self.txo.get(tx_hash): + if self.txi.get(tx_hash) or self.txo.get(tx_hash): continue tx = self.transactions.get(tx_hash) if tx is not None: @@ -528,9 +523,6 @@ class Abstract_Wallet(PrintError): def get_tx_delta(self, tx_hash, address): "effect of tx on address" - # pruned - if tx_hash in self.pruned_txo.values(): - return None delta = 0 # substract the value of coins sent from address d = self.txi.get(tx_hash, {}).get(address, []) @@ -561,7 +553,7 @@ class Abstract_Wallet(PrintError): is_partial = False v_in = v_out = v_out_mine = 0 for txin in tx.inputs(): - addr = txin.get('address') + addr = self.get_txin_address(txin) if self.is_mine(addr): is_mine = True is_relevant = True @@ -786,7 +778,7 @@ class Abstract_Wallet(PrintError): def get_txin_address(self, txi): addr = txi.get('address') - if addr != "(pubkey)": + if addr and addr != "(pubkey)": return addr prevout_hash = txi.get('prevout_hash') prevout_n = txi.get('prevout_n') @@ -794,8 +786,8 @@ class Abstract_Wallet(PrintError): for addr, l in dd.items(): for n, v, is_cb in l: if n == prevout_n: - self.print_error("found pay-to-pubkey address:", addr) return addr + return None def get_txout_address(self, txo): _type, x, v = txo @@ -815,14 +807,15 @@ class Abstract_Wallet(PrintError): """ conflicting_txns = set() with self.transaction_lock: - for txi in tx.inputs(): - ser = Transaction.get_outpoint_from_txin(txi) - if ser is None: + for txin in tx.inputs(): + if txin['type'] == 'coinbase': continue - spending_tx_hash = self.spent_outpoints.get(ser, None) + prevout_hash = txin['prevout_hash'] + prevout_n = txin['prevout_n'] + spending_tx_hash = self.spent_outpoints[prevout_hash].get(prevout_n) if spending_tx_hash is None: continue - # this outpoint (ser) has already been spent, by spending_tx + # this outpoint has already been spent, by spending_tx assert spending_tx_hash in self.transactions conflicting_txns |= {spending_tx_hash} txid = tx.txid() @@ -847,11 +840,6 @@ class Abstract_Wallet(PrintError): is_coinbase = tx.inputs()[0]['type'] == 'coinbase' tx_height = self.get_tx_height(tx_hash)[0] is_mine = any([self.is_mine(txin['address']) for txin in tx.inputs()]) - # do not save if tx is local and not mine - if tx_height == TX_HEIGHT_LOCAL and not is_mine: - # FIXME the test here should be for "not all is_mine"; cannot detect conflict in some cases - raise NotIsMineTransactionException() - # raise exception if unrelated to wallet is_for_me = any([self.is_mine(self.get_txout_address(txo)) for txo in tx.outputs()]) if not is_mine and not is_for_me: raise UnrelatedTransactionException() @@ -884,26 +872,27 @@ class Abstract_Wallet(PrintError): for tx_hash2 in to_remove: self.remove_transaction(tx_hash2) # add inputs + def add_value_from_prev_output(): + dd = self.txo.get(prevout_hash, {}) + # note: this nested loop takes linear time in num is_mine outputs of prev_tx + for addr, outputs in dd.items(): + # note: instead of [(n, v, is_cb), ...]; we could store: {n -> (v, is_cb)} + for n, v, is_cb in outputs: + if n == prevout_n: + if addr and self.is_mine(addr): + if d.get(addr) is None: + d[addr] = set() + d[addr].add((ser, v)) + return self.txi[tx_hash] = d = {} for txi in tx.inputs(): - addr = self.get_txin_address(txi) - if txi['type'] != 'coinbase': - prevout_hash = txi['prevout_hash'] - prevout_n = txi['prevout_n'] - ser = prevout_hash + ':%d'%prevout_n - if addr and self.is_mine(addr): - # we only track is_mine spends - self.spent_outpoints[ser] = tx_hash - # find value from prev output - dd = self.txo.get(prevout_hash, {}) - for n, v, is_cb in dd.get(addr, []): - if n == prevout_n: - if d.get(addr) is None: - d[addr] = [] - d[addr].append((ser, v)) - break - else: - self.pruned_txo[ser] = tx_hash + if txi['type'] == 'coinbase': + continue + prevout_hash = txi['prevout_hash'] + prevout_n = txi['prevout_n'] + ser = prevout_hash + ':%d' % prevout_n + self.spent_outpoints[prevout_hash][prevout_n] = tx_hash + add_value_from_prev_output() # add outputs self.txo[tx_hash] = d = {} for n, txo in enumerate(tx.outputs()): @@ -914,15 +903,15 @@ class Abstract_Wallet(PrintError): if d.get(addr) is None: d[addr] = [] d[addr].append((n, v, is_coinbase)) - # give v to txi that spends me - next_tx = self.pruned_txo.get(ser) - if next_tx is not None: - self.pruned_txo.pop(ser) - dd = self.txi.get(next_tx, {}) - if dd.get(addr) is None: - dd[addr] = [] - dd[addr].append((ser, v)) - self._add_tx_to_local_history(next_tx) + # give v to txi that spends me + next_tx = self.spent_outpoints[tx_hash].get(n) + if next_tx is not None: + dd = self.txi.get(next_tx, {}) + if dd.get(addr) is None: + dd[addr] = set() + if (ser, v) not in dd[addr]: + dd[addr].add((ser, v)) + self._add_tx_to_local_history(next_tx) # add to local history self._add_tx_to_local_history(tx_hash) # save @@ -930,37 +919,35 @@ class Abstract_Wallet(PrintError): return True def remove_transaction(self, tx_hash): + def remove_from_spent_outpoints(): + # undo spends in spent_outpoints + if tx is not None: # if we have the tx, this branch is faster + for txin in tx.inputs(): + if txin['type'] == 'coinbase': + continue + prevout_hash = txin['prevout_hash'] + prevout_n = txin['prevout_n'] + self.spent_outpoints[prevout_hash].pop(prevout_n, None) + if not self.spent_outpoints[prevout_hash]: + self.spent_outpoints.pop(prevout_hash) + else: # expensive but always works + for prevout_hash, d in list(self.spent_outpoints.items()): + for prevout_n, spending_txid in d.items(): + if spending_txid == tx_hash: + self.spent_outpoints[prevout_hash].pop(prevout_n, None) + if not self.spent_outpoints[prevout_hash]: + self.spent_outpoints.pop(prevout_hash) + # Remove this tx itself; if nothing spends from it. + # It is not so clear what to do if other txns spend from it, but it will be + # removed when those other txns are removed. + if not self.spent_outpoints[tx_hash]: + self.spent_outpoints.pop(tx_hash) with self.transaction_lock: self.print_error("removing tx from history", tx_hash) - self.transactions.pop(tx_hash, None) - # undo spent_outpoints that are in txi - for addr, l in self.txi[tx_hash].items(): - for ser, v in l: - self.spent_outpoints.pop(ser, None) - # undo spent_outpoints that are in pruned_txo - for ser, hh in list(self.pruned_txo.items()): - if hh == tx_hash: - self.spent_outpoints.pop(ser, None) - self.pruned_txo.pop(ser) - + tx = self.transactions.pop(tx_hash, None) + remove_from_spent_outpoints() self._remove_tx_from_local_history(tx_hash) - - # add tx to pruned_txo, and undo the txi addition - for next_tx, dd in self.txi.items(): - for addr, l in list(dd.items()): - ll = l[:] - for item in ll: - ser, v = item - prev_hash, prev_n = ser.split(':') - if prev_hash == tx_hash: - l.remove(item) - self.pruned_txo[ser] = next_tx - if l == []: - dd.pop(addr) - else: - dd[addr] = l - self.txi.pop(tx_hash, None) self.txo.pop(tx_hash, None) @@ -978,10 +965,6 @@ class Abstract_Wallet(PrintError): self.verified_tx.pop(tx_hash, None) if self.verifier: self.verifier.remove_spv_proof_for_tx(tx_hash) - # but remove completely if not is_mine - if self.txi[tx_hash] == {}: - # FIXME the test here should be for "not all is_mine"; cannot detect conflict in some cases - self.remove_transaction(tx_hash) self.history[addr] = hist for tx_hash, tx_height in hist: @@ -989,8 +972,9 @@ class Abstract_Wallet(PrintError): self.add_unverified_tx(tx_hash, tx_height) # if addr is new, we have to recompute txi and txo tx = self.transactions.get(tx_hash) - if tx is not None and self.txi.get(tx_hash, {}).get(addr) is None and self.txo.get(tx_hash, {}).get(addr) is None: - self.add_transaction(tx_hash, tx) + if tx is None: + continue + self.add_transaction(tx_hash, tx) # Store fees self.tx_fees.update(tx_fees) @@ -1975,7 +1959,6 @@ class Imported_Wallet(Simple_Wallet): self.verified_tx.pop(tx_hash, None) self.unverified_tx.pop(tx_hash, None) self.transactions.pop(tx_hash, None) - # FIXME: what about pruned_txo? self.storage.put('verified_tx3', self.verified_tx) self.save_transactions()