Sync burner outputs and display in wallet-tool

jmclient/jmclient/wallet.py

@@ -1655,6 +1655,10 @@ class FidelityBondMixin(object):
     #only one mixdepth will have fidelity bonds in it
     FIDELITY_BOND_MIXDEPTH = 0
 
+    MERKLE_BRANCH_UNAVAILABLE = b"mbu"
+
+    _BURNER_OUTPUT_STORAGE_KEY = b"burner-out"
+
     @classmethod
     def _time_number_to_timestamp(cls, timenumber):
         """
@@ -1776,6 +1780,28 @@ class FidelityBondMixin(object):
         script = self.get_script(mixdepth, address_type, index, timenumber)
         return self.script_to_addr(script)
 
+    def add_burner_output(self, path, txhex, block_height, merkle_branch,
+            block_index, write=True):
+        """
+        merkle_branch = None means it was unavailable because of pruning
+        """
+        if self._BURNER_OUTPUT_STORAGE_KEY not in self._storage.data:
+            self._storage.data[self._BURNER_OUTPUT_STORAGE_KEY] = {}
+        path = path.encode()
+        txhex = unhexlify(txhex)
+        if not merkle_branch:
+            merkle_branch = self.MERKLE_BRANCH_UNAVAILABLE
+        self._storage.data[self._BURNER_OUTPUT_STORAGE_KEY][path] = [txhex,
+            block_height, merkle_branch, block_index]
+        if write:
+            self._storage.save()
+
+    def get_burner_outputs(self):
+        """
+        Result is a dict {path: [txhex, blockheight, merkleproof, blockindex]}
+        """
+        return self._storage.data.get(self._BURNER_OUTPUT_STORAGE_KEY, {})
+
 #class FidelityBondWatchonlyWallet(ImportWalletMixin, BIP39WalletMixin, FidelityBondMixin):
 
 

jmclient/jmclient/wallet_service.py

@@ -5,6 +5,7 @@ import time
 import ast
 import binascii
 import sys
+import itertools
 from decimal import Decimal
 from copy import deepcopy
 from twisted.internet import reactor
@@ -17,6 +18,7 @@ from jmclient.blockchaininterface import (INF_HEIGHT, BitcoinCoreInterface,
     BitcoinCoreNoHistoryInterface)
 from jmclient.wallet import FidelityBondMixin
 from jmbase.support import jmprint, EXIT_SUCCESS
+import jmbitcoin as btc
 """Wallet service
 
 The purpose of this independent service is to allow
@@ -517,6 +519,19 @@ class WalletService(Service):
         # index:
         self.bci.import_addresses(self.collect_addresses_gap(), self.get_wallet_name(),
                                   self.restart_callback)
+
+        if isinstance(self.wallet, FidelityBondMixin):
+            mixdepth = FidelityBondMixin.FIDELITY_BOND_MIXDEPTH
+            address_type = FidelityBondMixin.BIP32_BURN_ID
+
+            burner_outputs = self.wallet.get_burner_outputs()
+            max_index = 0
+            for path_repr in burner_outputs:
+                index = self.wallet.path_repr_to_path(path_repr.decode())[-1]
+                max_index = max(index+1, max_index)
+            self.wallet.set_next_index(mixdepth, address_type, max_index,
+                force=True)
+
         self.synced = True
 
     def display_rescan_message_and_system_exit(self, restart_cb):
@@ -543,14 +558,16 @@ class WalletService(Service):
         self.wallet.set_next_index(mixdepth, address_type, self.wallet.gap_limit,
             force=True)
         highest_used_index = 0
-
         known_burner_outputs = self.wallet.get_burner_outputs()
-        for index in range(self.wallet.gap_limit):
+
+        index = -1
+        while index - highest_used_index < self.wallet.gap_limit:
+            index += 1
+            self.wallet.set_next_index(mixdepth, address_type, index, force=True)
             path = self.wallet.get_path(mixdepth, address_type, index)
             path_privkey, engine = self.wallet._get_priv_from_path(path)
             path_pubkey = engine.privkey_to_pubkey(path_privkey)
             path_pubkeyhash = btc.bin_hash160(path_pubkey)
-
             for burner_tx in burner_txes:
                 burner_pubkeyhash, gettx = burner_tx
                 if burner_pubkeyhash != path_pubkeyhash:
@@ -593,9 +610,35 @@ class WalletService(Service):
             self.display_rescan_message_and_system_exit(self.restart_callback)
             return
 
+        if isinstance(self.wallet, FidelityBondMixin):
+            tx_receive = []
+            burner_txes = []
+            for tx in self.bci._yield_transactions(wallet_name):
+                if tx['category'] == 'receive':
+                    tx_receive.append(tx)
+                elif tx["category"] == "send":
+                    gettx = self.bci.get_transaction(tx["txid"])
+                    txd = self.bci.get_deser_from_gettransaction(gettx)
+                    if len(txd["outs"]) > 1:
+                        continue
+                    #must be mined into a block to sync
+                    #otherwise there's no merkleproof or block index
+                    if gettx["confirmations"] < 1:
+                        continue
+                    script = binascii.unhexlify(txd["outs"][0]["script"])
+                    if script[0] != 0x6a: #OP_RETURN
+                        continue
+                    pubkeyhash = script[2:]
+                    burner_txes.append((pubkeyhash, gettx))
+
+            self.sync_burner_outputs(burner_txes)
+            used_addresses_gen = (tx["address"] for tx in tx_receive)
+        else:
+            #not fidelity bond wallet, significantly faster sync
             used_addresses_gen = (tx['address']
                                   for tx in self.bci._yield_transactions(wallet_name)
                                   if tx['category'] == 'receive')
+
         used_indices = self.get_used_indices(used_addresses_gen)
         jlog.debug("got used indices: {}".format(used_indices))
         gap_limit_used = not self.check_gap_indices(used_indices)

jmclient/jmclient/wallet_utils.py

@@ -217,6 +217,12 @@ class WalletViewEntry(WalletViewBase):
             ed += self.separator + self.serclass(self.private_key)
         return self.serclass(ed)
 
+class WalletViewEntryBurnOutput(WalletViewEntry):
+    # balance in burn outputs shouldnt be counted
+    # towards the total balance
+    def get_balance(self, include_unconf=True):
+        return 0
+
 class WalletViewBranch(WalletViewBase):
     def __init__(self, wallet_path_repr, account, address_type, branchentries=None,
                  xpub=None, serclass=str, custom_separator=None):
@@ -261,7 +267,7 @@ class WalletViewAccount(WalletViewBase):
         self.account_name = account_name
         self.xpub = xpub
         if branches:
-            assert len(branches) in [2, 3] #3 if imported keys
+            assert len(branches) in [2, 3, 4] #3 if imported keys, 4 if fidelity bonds
             assert all([isinstance(x, WalletViewBranch) for x in branches])
         self.branches = branches
 
@@ -477,13 +483,54 @@ def wallet_display(wallet_service, showprivkey, displayall=False,
                         entrylist.append(WalletViewEntry(
                             wallet_service.get_path_repr(path), m, address_type, k,
                             addr, [balance, balance], priv=privkey, used=status))
-
             #TODO fidelity bond master pub key is this, although it should include burner too
             xpub_key = wallet_service.get_bip32_pub_export(m, address_type)
             path = wallet_service.get_path_repr(wallet_service.get_path(m, address_type))
             branchlist.append(WalletViewBranch(path, m, address_type, entrylist,
                 xpub=xpub_key))
 
+            entrylist = []
+            address_type = FidelityBondMixin.BIP32_BURN_ID
+            unused_index = wallet_service.get_next_unused_index(m, address_type)
+            burner_outputs = wallet_service.wallet.get_burner_outputs()
+            wallet_service.set_next_index(m, address_type, unused_index +
+                wallet_service.wallet.gap_limit, force=True)
+            for k in range(unused_index + wallet_service.wallet.gap_limit):
+                path = wallet_service.get_path(m, address_type, k)
+                path_repr = wallet_service.get_path_repr(path)
+                path_repr_b = path_repr.encode()
+
+                privkey, engine = wallet_service._get_priv_from_path(path)
+                pubkey = engine.privkey_to_pubkey(privkey)
+                pubkeyhash = btc.bin_hash160(pubkey)
+                output = "BURN-" + binascii.hexlify(pubkeyhash).decode()
+
+                balance = 0
+                status = "no transaction"
+                if path_repr_b in burner_outputs:
+                    txhex, blockheight, merkle_branch, blockindex = burner_outputs[path_repr_b]
+                    txhex = binascii.hexlify(txhex).decode()
+                    txd = btc.deserialize(txhex)
+                    assert len(txd["outs"]) == 1
+                    balance = txd["outs"][0]["value"]
+                    script = binascii.unhexlify(txd["outs"][0]["script"])
+                    assert script[0] == 0x6a #OP_RETURN
+                    tx_pubkeyhash = script[2:]
+                    assert tx_pubkeyhash == pubkeyhash
+                    status = btc.txhash(txhex) + (" [NO MERKLE PROOF]" if
+                        merkle_branch == FidelityBondMixin.MERKLE_BRANCH_UNAVAILABLE else "")
+                privkey = (wallet_service.get_wif_path(path) if showprivkey else "")
+                if displayall or balance > 0:
+                    entrylist.append(WalletViewEntryBurnOutput(path_repr, m,
+                        address_type, k, output, [balance, balance],
+                        priv=privkey, used=status))
+            wallet_service.set_next_index(m, address_type, unused_index)
+
+            xpub_key = wallet_service.get_bip32_pub_export(m, address_type)
+            path = wallet_service.get_path_repr(wallet_service.get_path(m, address_type))
+            branchlist.append(WalletViewBranch(path, m, address_type, entrylist,
+                xpub=xpub_key))
+
         ipb = get_imported_privkey_branch(wallet_service, m, showprivkey)
         if ipb:
             branchlist.append(ipb)
@@ -1294,6 +1341,11 @@ def wallet_tool_main(wallet_root_path):
         method = ('display' if len(args) == 1 else args[1].lower())
         read_only = method in readonly_methods
 
+        #special case needed for fidelity bond burner outputs
+        #maybe theres a better way to do this
+        if options.recoversync:
+            read_only = False
+
         wallet = open_test_wallet_maybe(
             wallet_path, seed, options.mixdepth, read_only=read_only,
             wallet_password_stdin=options.wallet_password_stdin, gap_limit=options.gaplimit)