Allow sendpayment to restart with honest makers

If N makers are chosen and M fail to respond with sig,
this alteration to taker code allows restart of
entire transaction with that specific subset of makers (N-M)
that originally responded honestly.

jmclient/jmclient/taker.py

@@ -78,6 +78,10 @@ class Taker(object):
         self.schedule = schedule
         self.order_chooser = order_chooser
         self.ignored_makers = [] if not ignored_makers else ignored_makers
+        #Used in attempts to complete with subset after second round failure:
+        self.honest_makers = []
+        #Toggle: if set, only honest makers will be used from orderbook
+        self.honest_only = False
         self.waiting_for_conf = False
         self.txid = None
         self.schedule_index = -1
@@ -95,8 +99,33 @@ class Taker(object):
         jlog.info(infotype + ":" + msg)
 
     def add_ignored_makers(self, makers):
+        """Makers should be added to this list when they have refused to
+        complete the protocol honestly, and should remain in this set
+        for the duration of the Taker run (so, the whole schedule).
+        """
         self.ignored_makers.extend(makers)
 
+    def add_honest_makers(self, makers):
+        """A maker who has shown willigness to complete the protocol
+        by returning a valid signature for a coinjoin can be added to
+        this list, the taker can optionally choose to only source
+        offers from thus-defined "honest" makers.
+        """
+        self.honest_makers.extend(makers)
+
+    def set_honest_only(self, truefalse):
+        """Toggle; if set, offers will only be accepted
+        from makers in the self.honest_makers list.
+        This should not be called unless we already have
+        a list of such honest makers (see add_honest_makers()).
+        """
+        if truefalse:
+            if not len(self.honest_makers):
+                jlog.debug("Attempt to set honest-only without "
+                           "any honest makers; ignored.")
+                return
+        self.honest_only = truefalse
+
     def initialize(self, orderbook):
         """Once the daemon is active and has returned the current orderbook,
         select offers, re-initialize variables and prepare a commitment,
@@ -172,6 +201,15 @@ class Taker(object):
         return (True, self.cjamount, commitment, revelation, self.orderbook)
 
     def filter_orderbook(self, orderbook, sweep=False):
+        #If honesty filter is set, we immediately filter to only the prescribed
+        #honest makers before continuing. In this case, the number of
+        #counterparties should already match, and this has to be set by the
+        #script instantiating the Taker.
+        #Note: If one or more of the honest makers has dropped out in the meantime,
+        #we will just have insufficient offers and it will fail in the usual way
+        #for insufficient liquidity.
+        if self.honest_only:
+            orderbook = [o for o in orderbook if o['counterparty'] in self.honest_makers]
         if sweep:
             self.orderbook = orderbook #offers choosing deferred to next step
         else:

jmclient/jmclient/taker_utils.py

@@ -159,6 +159,16 @@ def unconf_update(taker, schedulefile, tumble_log, addtolog=False):
     #is sufficient
     taker.wallet.update_cache_index()
 
+    #If honest-only was set, and we are going to continue (e.g. Tumbler),
+    #we switch off the honest-only filter. We also wipe the honest maker
+    #list, because the intention is to isolate the source of liquidity
+    #to exactly those that participated, in 1 transaction (i.e. it's a 1
+    #transaction feature). This code is here because it *must* be called
+    #before any continuation, even if confirm_callback happens before
+    #unconfirm_callback
+    taker.set_honest_only(False)
+    taker.honest_makers = []
+
     #We persist the fact that the transaction is complete to the
     #schedule file. Note that if a tweak to the schedule occurred,
     #it only affects future (non-complete) transactions, so the final
@@ -227,20 +237,53 @@ def tumbler_taker_finished_update(taker, schedulefile, tumble_log, options,
             taker.wallet.remove_old_utxos(txd)
             taker.wallet.add_new_utxos(txd, txid)
         else:
-            #a transaction failed; tumbler is aggressive in trying to
-            #complete; we tweak the schedule from this point in the mixdepth,
-            #then try again:
-            tumble_log.info("Transaction attempt failed, tweaking schedule"
-                            " and trying again.")
-            tumble_log.info("The paramaters of the failed attempt: ")
-            tumble_log.info(str(taker.schedule[taker.schedule_index]))
+            #a transaction failed, either because insufficient makers
+            #(acording to minimum_makers) responded in Stage 1, or not all
+            #makers responded in Stage 2. We'll first try to repeat without the
+            #troublemakers.
+            #Note that Taker.nonrespondants is always set to the full maker
+            #list at the start of Taker.receive_utxos, so it is always updated
+            #to a new list in each tx run.
             log.info("Schedule entry: " + str(
                 taker.schedule[taker.schedule_index]) + \
                      " failed after timeout, trying again")
-            taker.schedule_index -= 1
-            taker.schedule = tweak_tumble_schedule(options, taker.schedule,
-                                                   taker.schedule_index,
-                                                   taker.tdestaddrs)
+            taker.add_ignored_makers(taker.nonrespondants)
+            #Now we have to set the specific group we want to use, and hopefully
+            #they will respond again as they showed honesty last time.
+            taker.add_honest_makers(list(set(
+                taker.maker_utxo_data.keys()).symmetric_difference(
+                    set(taker.nonrespondants))))
+            #If no makers were honest, we can only tweak the schedule.
+            #If some were, we prefer the restart with them only:
+            if len(taker.honest_makers) != 0:
+                tumble_log.info("Transaction attempt failed, attempting to "
+                                "restart with subset.")
+                tumble_log.info("The paramaters of the failed attempt: ")
+                tumble_log.info(str(taker.schedule[taker.schedule_index]))
+                #we must reset the number of counterparties, as well as fix who they
+                #are; this is because the number is used to e.g. calculate fees.
+                #cleanest way is to reset the number in the schedule before restart.
+                taker.schedule[taker.schedule_index][2] = len(taker.honest_makers)
+                retry_str = "Retrying with: " + str(taker.schedule[
+                    taker.schedule_index][2]) + " counterparties."
+                tumble_log.info(retry_str)
+                log.info(retry_str)
+                taker.set_honest_only(True)
+                taker.schedule_index -= 1
+
+            #a tumbler is aggressive in trying to complete; we tweak the schedule
+            #from this point in the mixdepth, then try again. However we only
+            #try this strategy if the previous (select-honest-only) strategy
+            #failed.
+            else:
+                tumble_log.info("Transaction attempt failed, tweaking schedule"
+                                " and trying again.")
+                tumble_log.info("The paramaters of the failed attempt: ")
+                tumble_log.info(str(taker.schedule[taker.schedule_index]))
+                taker.schedule_index -= 1
+                taker.schedule = tweak_tumble_schedule(options, taker.schedule,
+                                                       taker.schedule_index,
+                                                       taker.tdestaddrs)
             tumble_log.info("We tweaked the schedule, the new schedule is:")
             tumble_log.info(pprint.pformat(taker.schedule))
     else:

scripts/sendpayment.py

@@ -188,8 +188,35 @@ def main():
                 reactor.callLater(waittime*60,
                                   clientfactory.getClient().clientStart)
             else:
-                #a transaction failed; just stop
-                reactor.stop()
+                #a transaction failed; we'll try to repeat without the
+                #troublemakers
+                #Note that Taker.nonrespondants is always set to the full maker
+                #list at the start of Taker.receive_utxos, so it is always updated
+                #to a new list in each run.
+                print("We failed to complete the transaction. The following "
+                      "makers didn't respond: ", taker.nonrespondants,
+                      ", so we will retry without them.")
+                taker.add_ignored_makers(taker.nonrespondants)
+                #Now we have to set the specific group we want to use, and hopefully
+                #they will respond again as they showed honesty last time.
+                taker.add_honest_makers(list(set(
+                    taker.maker_utxo_data.keys()).symmetric_difference(
+                        set(taker.nonrespondants))))
+                if len(taker.honest_makers) == 0:
+                    log.info("None of the makers responded honestly; "
+                             "giving up this attempt.")
+                    reactor.stop()
+                    return
+                #we must reset the number of counterparties, as well as fix who they
+                #are; this is because the number is used to e.g. calculate fees.
+                #cleanest way is to reset the number in the schedule before restart.
+                taker.schedule[taker.schedule_index][2] = len(taker.honest_makers)
+                log.info("Retrying with: " + str(taker.schedule[
+                    taker.schedule_index][2]) + " counterparties.")
+                #rewind to try again (index is incremented in Taker.initialize())
+                taker.schedule_index -= 1
+                taker.set_honest_only(True)
+                reactor.callLater(5.0, clientfactory.getClient().clientStart)
         else:
             if not res:
                 log.info("Did not complete successfully, shutting down")

test/ygrunner.py

@@ -27,32 +27,77 @@ class MaliciousYieldGenerator(YieldGeneratorBasic):
     to prevent taker continuing successfully (unless
     they can complete-with-subset).
     """
-    def set_maliciousness(self, frac):
+    def set_maliciousness(self, frac, mtype=None):
+        self.authmal = False
+        self.txmal = False
+        if mtype == "tx":
+            self.txmal = True
+        elif mtype == "auth":
+            self.authmal = True
+        else:
+            self.txmal = True
+            self.authmal = True
         self.mfrac = frac
+
     def on_auth_received(self, nick, offer, commitment, cr, amount, kphex):
-        if random.randint(1, 100) < self.mfrac:
+        if self.authmal:
+            if random.randint(1, 100) < self.mfrac:
+                print("Counterparty commitment rejected maliciously")
+                return (False,)
+        return super(MaliciousYieldGenerator, self).on_auth_received(nick,
+                                    offer, commitment, cr, amount, kphex)
+    def on_tx_received(self, nick, txhex, offerinfo):
+        if self.txmal:
+            if random.randint(1, 100) < self.mfrac:
+                print("Counterparty tx rejected maliciously")
+                return (False, "malicious tx rejection")
+        return super(MaliciousYieldGenerator, self).on_tx_received(nick, txhex,
+                                                                   offerinfo)
+
+class DeterministicMaliciousYieldGenerator(YieldGeneratorBasic):
+    """Overrides, randomly chosen persistently, some maker functions
+    to prevent taker continuing successfully (unless
+    they can complete-with-subset).
+    """
+    def set_maliciousness(self, frac, mtype=None):
+        self.authmal = False
+        self.txmal = False
+        if mtype == "tx":
+            if random.randint(1, 100) < frac:
+                self.txmal = True
+        elif mtype == "auth":
+            if random.randint(1, 100) < frac:
+                self.authmal = True
+        else:
+            if random.randint(1, 100) < frac:
+                self.txmal = True
+                self.authmal = True
+
+    def on_auth_received(self, nick, offer, commitment, cr, amount, kphex):
+        if self.authmal:
             print("Counterparty commitment rejected maliciously")
             return (False,)
-        return super(MaliciousYieldGenerator, self).on_auth_received(nick,
+        return super(DeterministicMaliciousYieldGenerator, self).on_auth_received(nick,
                                     offer, commitment, cr, amount, kphex)
     def on_tx_received(self, nick, txhex, offerinfo):
-        if random.randint(1, 100) < self.mfrac:
+        if self.txmal:
             print("Counterparty tx rejected maliciously")
             return (False, "malicious tx rejection")
-        return super(MaliciousYieldGenerator, self).on_tx_received(nick, txhex,
+        return super(DeterministicMaliciousYieldGenerator, self).on_tx_received(nick, txhex,
                                                                    offerinfo)
 
-
 @pytest.mark.parametrize(
-    "num_ygs, wallet_structures, mean_amt, malicious",
+    "num_ygs, wallet_structures, mean_amt, malicious, deterministic",
     [
         # 1sp 3yg, honest makers
-        (3, [[1, 3, 0, 0, 0]] * 4, 2, 0),
+        (3, [[1, 3, 0, 0, 0]] * 4, 2, 0, False),
         # 1sp 3yg, malicious makers reject on auth and on tx 30% of time
-        #(4, [[1, 3, 0, 0, 0]] * 5, 2, 30),
+        #(3, [[1, 3, 0, 0, 0]] * 4, 2, 30, False),
+        # 1 sp 9 ygs, deterministically malicious 50% of time
+        #(9, [[1, 3, 0, 0, 0]] * 10, 2, 50, True),
     ])
 def test_start_ygs(setup_ygrunner, num_ygs, wallet_structures, mean_amt,
-                   malicious):
+                   malicious, deterministic):
     """Set up some wallets, for the ygs and 1 sp.
     Then start the ygs in background and publish
     the seed of the sp wallet for easy import into -qt
@@ -71,14 +116,19 @@ def test_start_ygs(setup_ygrunner, num_ygs, wallet_structures, mean_amt,
     cjfee_r = '0.001'
     ordertype = 'swreloffer'
     minsize = 100000
-    ygclass = MaliciousYieldGenerator if malicious else YieldGeneratorBasic
+    ygclass = YieldGeneratorBasic
+    if malicious:
+        if deterministic:
+            ygclass = DeterministicMaliciousYieldGenerator
+        else:
+            ygclass = MaliciousYieldGenerator
     for i in range(num_ygs):
         
         cfg = [txfee, cjfee_a, cjfee_r, ordertype, minsize]
         sync_wallet(wallets[i]["wallet"], fast=True)
         yg = ygclass(wallets[i]["wallet"], cfg)
         if malicious:
-            yg.set_maliciousness(malicious)
+            yg.set_maliciousness(malicious, mtype="tx")
         clientfactory = JMClientProtocolFactory(yg, proto_type="MAKER")
         nodaemon = jm_single().config.getint("DAEMON", "no_daemon")
         daemon = True if nodaemon == 1 else False