From 8d454277d8d192c5cdb51b49fa6c43969b283e14 Mon Sep 17 00:00:00 2001
From: Matt Whitlock <joinmarket@mattwhitlock.name>
Date: Thu, 10 Feb 2022 04:25:25 -0500
Subject: [PATCH] BitcoinCoreInterface: improve _yield_transactions

Alter the BitcoinCoreInterface._yield_transactions generator to be
nicer in cases where only a small number of transactions need to be
examined. Geometrically ramp the requested number of transactions in
each successive request, starting from 1. Since new transactions can
appear in the Bitcoin Core wallet between RPC calls, overlap successive
request ranges and resume yielding transactions after finding the
previously yielded transaction to avoid yielding duplicates.

The listtransactions RPC of Bitcoin Core can return the same (txid,vout)
as both a "category":"receive" and a "category":"send". Therefore, use
(txid,vout,category) as the sync key to ensure that all distinct
elements are yielded.

Delete the unused wallet_name parameter from _yield_transactions, per
suggestion by Adam Gibson.
---
 jmclient/jmclient/blockchaininterface.py | 40 +++++++++++++++++-------
 jmclient/jmclient/wallet_service.py      |  7 ++---
 2 files changed, 32 insertions(+), 15 deletions(-)

diff --git a/jmclient/jmclient/blockchaininterface.py b/jmclient/jmclient/blockchaininterface.py
index 2402ff7..65a4cfc 100644
--- a/jmclient/jmclient/blockchaininterface.py
+++ b/jmclient/jmclient/blockchaininterface.py
@@ -292,18 +292,36 @@ class BitcoinCoreInterface(BlockchainInterface):
             self.import_addresses(addresses - imported_addresses, wallet_name)
         return import_needed
 
-    def _yield_transactions(self, wallet_name):
-        batch_size = 1000
-        iteration = 0
+    def _yield_transactions(self):
+        """ Generates a lazily fetched sequence of transactions seen in the
+        wallet (under any label/account), yielded in newest-first order. Care
+        is taken to avoid yielding duplicates even when new transactions are
+        actively being added to the wallet while the iteration is ongoing.
+        """
+        num, skip = 1, 0
+        txs = self.list_transactions(num, skip)
+        if not txs:
+            return
+        yielded_tx = txs[0]
+        yield yielded_tx
         while True:
-            new = self._rpc(
-                'listtransactions',
-                ["*", batch_size, iteration * batch_size, True])
-            for tx in new:
-                yield tx
-            if len(new) < batch_size:
+            num *= 2
+            txs = self.list_transactions(num, skip)
+            if not txs:
+                return
+            try:
+                idx = [(tx['txid'], tx['vout'], tx['category']) for tx in txs
+                        ].index((yielded_tx['txid'], yielded_tx['vout'],
+                        yielded_tx['category']))
+            except ValueError:
+                skip += num
+                continue
+            for tx in reversed(txs[:idx]):
+                yielded_tx = tx  # inefficient but more obvious
+                yield yielded_tx
+            if len(txs) < num:
                 return
-            iteration += 1
+            skip += num - 1
 
     def get_deser_from_gettransaction(self, rpcretval):
         """Get full transaction deserialization from a call
@@ -641,7 +659,7 @@ class BitcoinCoreNoHistoryInterface(BitcoinCoreInterface, RegtestBitcoinCoreMixi
         assert desc_str.startswith("addr(")
         return desc_str[5:desc_str.find(")")]
 
-    def _yield_transactions(self, wallet_name):
+    def _yield_transactions(self):
         for u in self.scan_result["unspents"]:
             tx = {"category": "receive", "address":
                 self._get_addr_from_desc(u["desc"])}
diff --git a/jmclient/jmclient/wallet_service.py b/jmclient/jmclient/wallet_service.py
index 332523e..d0b7952 100644
--- a/jmclient/jmclient/wallet_service.py
+++ b/jmclient/jmclient/wallet_service.py
@@ -669,8 +669,7 @@ class WalletService(Service):
         """
         res = []
         processed_txids = []
-        for r in self.bci._yield_transactions(
-            self.get_wallet_name()):
+        for r in self.bci._yield_transactions():
             txid = r["txid"]
             if txid not in processed_txids:
                 tx = self.bci.get_transaction(hextobin(txid))
@@ -720,7 +719,7 @@ class WalletService(Service):
         if isinstance(self.wallet, FidelityBondMixin):
             tx_receive = []
             burner_txes = []
-            for tx in self.bci._yield_transactions(wallet_name):
+            for tx in self.bci._yield_transactions():
                 if tx['category'] == 'receive':
                     tx_receive.append(tx)
                 elif tx["category"] == "send":
@@ -743,7 +742,7 @@ class WalletService(Service):
         else:
             #not fidelity bond wallet, significantly faster sync
             used_addresses_gen = set(tx['address']
-                                  for tx in self.bci._yield_transactions(wallet_name)
+                                  for tx in self.bci._yield_transactions()
                                   if tx['category'] == 'receive')
         # needed for address-reuse check:
         self.used_addresses = used_addresses_gen