fix existing tests

pyproject.toml

@@ -56,14 +56,15 @@ services = [
 ]
 test = [
     "joinmarket[services]",
-    "coverage==5.2.1",
+    "coverage==7.8.2",
     "flake8",
     "freezegun",
     "mock",
     "pexpect",
-    "pytest-cov>=2.4.0,<2.6",
-    "pytest==6.2.5",
+    "pytest-cov==6.1.1",
+    "pytest==7.4.4",
     "python-coveralls",
+    "unittest-parametrize==1.6.0",
 ]
 gui = [
     "joinmarket[services]",

src/jmbase/__init__.py

@@ -10,7 +10,7 @@ from .support import (get_log, chunks, debug_silence, jmprint,
                       IndentedHelpFormatterWithNL, wrapped_urlparse,
                       bdict_sdict_convert, random_insert, dict_factory,
                       cli_prompt_user_value, cli_prompt_user_yesno,
-                      async_hexbin, twisted_sys_exit)
+                      async_hexbin, twisted_sys_exit, is_running_from_pytest)
 from .proof_of_work import get_pow, verify_pow
 from .twisted_utils import (stop_reactor, is_hs_uri, get_tor_agent,
                             get_nontor_agent, JMHiddenService,

src/jmbase/support.py

@@ -220,6 +220,9 @@ def get_password(msg): #pragma: no cover
         password = password.encode('utf-8')
     return password
 
+def is_running_from_pytest():
+    return bool(environ.get("PYTEST_CURRENT_TEST"))
+
 def lookup_appdata_folder(appname):
     """ Given an appname as a string,
     return the correct directory for storing

src/jmclient/wallet.py

@@ -1659,8 +1659,10 @@ class BaseWallet(object):
 
     async def _populate_maps(self, paths):
         for path in paths:
-            self._script_map[await self.get_script_from_path(path)] = path
-            self._addr_map[await self.get_address_from_path(path)] = path
+            script = await self.get_script_from_path(path)
+            self._script_map[script] = path
+            addr = await self.get_address_from_path(path)
+            self._addr_map[addr] = path
 
     def addr_to_path(self, addr):
         assert isinstance(addr, str)

src/jmclient/wallet_service.py

@@ -20,7 +20,8 @@ from jmclient.blockchaininterface import (INF_HEIGHT, BitcoinCoreInterface,
 from jmclient.wallet import (FidelityBondMixin, BaseWallet, TaprootWallet,
                              FrostWallet)
 from jmbase import (stop_reactor, hextobin, utxo_to_utxostr,
-                    twisted_sys_exit, jmprint, EXIT_SUCCESS, EXIT_FAILURE)
+                    twisted_sys_exit, jmprint, EXIT_SUCCESS, EXIT_FAILURE,
+                    is_running_from_pytest)
 from .descriptor import descsum_create
 
 """Wallet service
@@ -706,7 +707,8 @@ class WalletService(Service):
         #theres also a sys.exit() in BitcoinCoreInterface.import_addresses()
         #perhaps have sys.exit() placed inside the restart_cb that only
         # CLI scripts will use
-        if isinstance(self.bci, BitcoinCoreInterface):
+        if (isinstance(self.bci, BitcoinCoreInterface)
+                and not is_running_from_pytest()):
             #Exit conditions cannot be included in tests
             restart_msg = ("Use `bitcoin-cli rescanblockchain` if you're "
                            "recovering an existing wallet from backup seed\n"

src/jmclient/wallet_utils.py

@@ -1587,7 +1587,9 @@ async def open_test_wallet_maybe(
                 del kwargs['password']
             if 'read_only' in kwargs:
                 del kwargs['read_only']
-            return test_wallet_cls(storage, **kwargs)
+            wallet = test_wallet_cls(storage, **kwargs)
+            await wallet.async_init(storage, **kwargs)
+            return wallet
 
     if wallet_password_stdin is True:
         password = read_password_stdin()

test/jmclient/commontest.py

@@ -5,6 +5,9 @@ import os
 import random
 from decimal import Decimal
 from typing import Callable, List, Optional, Set, Tuple, Union
+from unittest import IsolatedAsyncioTestCase
+
+from twisted.trial.unittest import TestCase as TrialTestCase
 
 import jmbitcoin as btc
 from jmbase import (get_log, hextobin, bintohex, dictchanger)
@@ -33,6 +36,17 @@ def dummy_accept_callback(tx, destaddr, actual_amount, fee_est,
 def dummy_info_callback(msg):
     pass
 
+
+class TrialAsyncioTestCase(TrialTestCase, IsolatedAsyncioTestCase):
+
+    def __init__(self, methodName='runTest'):
+        IsolatedAsyncioTestCase.__init__(self, methodName)
+        TrialTestCase.__init__(self, methodName)
+
+    def __call__(self, *args, **kwds):
+        return IsolatedAsyncioTestCase.run(self, *args, **kwds)
+
+
 class DummyBlockchainInterface(BlockchainInterface):
 
     def __init__(self) -> None:
@@ -174,16 +188,18 @@ class DummyBlockchainInterface(BlockchainInterface):
         return 30000
 
 
-def create_wallet_for_sync(wallet_structure, a, **kwargs):
+async def create_wallet_for_sync(wallet_structure, a, **kwargs):
     #We need a distinct seed for each run so as not to step over each other;
     #make it through a deterministic hash of all parameters including optionals.
     preimage = "".join([str(x) for x in a] + [str(y) for y in kwargs.values()]).encode("utf-8")
     print("using preimage: ", preimage)
     seedh = bintohex(btc.Hash(preimage))[:32]
-    return make_wallets(
-        1, [wallet_structure], fixed_seeds=[seedh], **kwargs)[0]['wallet']
+    wallets = await make_wallets(
+        1, [wallet_structure], fixed_seeds=[seedh], **kwargs)
+    return wallets [0]['wallet']
 
-def make_sign_and_push(ins_full,
+
+async def make_sign_and_push(ins_full,
                        wallet_service,
                        amount,
                        output_addr=None,
@@ -202,8 +218,10 @@ def make_sign_and_push(ins_full,
     total = sum(x['value'] for x in ins_full.values())
     ins = list(ins_full.keys())
     #random output address and change addr
-    output_addr = wallet_service.get_new_addr(1, BaseWallet.ADDRESS_TYPE_INTERNAL) if not output_addr else output_addr
-    change_addr = wallet_service.get_new_addr(0, BaseWallet.ADDRESS_TYPE_INTERNAL) if not change_addr else change_addr
+    output_addr = await wallet_service.get_new_addr(
+        1, BaseWallet.ADDRESS_TYPE_INTERNAL) if not output_addr else output_addr
+    change_addr = await wallet_service.get_new_addr(
+        0, BaseWallet.ADDRESS_TYPE_INTERNAL) if not change_addr else change_addr
     fee_est = estimate_tx_fee(len(ins), 2) if estimate_fee else 10000
     outs = [{'value': amount,
              'address': output_addr}, {'value': total - amount - fee_est,
@@ -214,7 +232,7 @@ def make_sign_and_push(ins_full,
     for i, j in enumerate(ins):
         scripts[i] = (ins_full[j]["script"], ins_full[j]["value"])
 
-    success, msg = wallet_service.sign_tx(tx, scripts, hashcode=hashcode)
+    success, msg = await wallet_service.sign_tx(tx, scripts, hashcode=hashcode)
     if not success:
         return False
     #pushtx returns False on any error
@@ -222,20 +240,21 @@ def make_sign_and_push(ins_full,
     if push_succeed:
         # in normal operation this happens automatically
         # but in some tests there is no monitoring loop:
-        wallet_service.process_new_tx(tx)
+        await wallet_service.process_new_tx(tx)
         return tx.GetTxid()[::-1]
     else:
         return False
 
-def make_wallets(n,
-                 wallet_structures=None,
-                 mean_amt=1,
-                 sdev_amt=0,
-                 start_index=0,
-                 fixed_seeds=None,
-                 wallet_cls=SegwitWallet,
-                 mixdepths=5,
-                 populate_internal=BaseWallet.ADDRESS_TYPE_EXTERNAL):
+
+async def make_wallets(n,
+                       wallet_structures=None,
+                       mean_amt=1,
+                       sdev_amt=0,
+                       start_index=0,
+                       fixed_seeds=None,
+                       wallet_cls=SegwitWallet,
+                       mixdepths=5,
+                       populate_internal=BaseWallet.ADDRESS_TYPE_EXTERNAL):
     '''n: number of wallets to be created
        wallet_structure: array of n arrays , each subarray
        specifying the number of addresses to be populated with coins
@@ -258,8 +277,8 @@ def make_wallets(n,
     for i in range(n):
         assert len(seeds[i]) == BIP32Wallet.ENTROPY_BYTES * 2
 
-        w = open_test_wallet_maybe(seeds[i], seeds[i], mixdepths - 1,
-                                   test_wallet_cls=wallet_cls)
+        w = await open_test_wallet_maybe(seeds[i], seeds[i], mixdepths - 1,
+                                         test_wallet_cls=wallet_cls)
         wallet_service = WalletService(w)
         wallets[i + start_index] = {'seed': seeds[i],
                                     'wallet': wallet_service}
@@ -270,8 +289,9 @@ def make_wallets(n,
                     amt = mean_amt - sdev_amt / 2.0 + deviation
                     if amt < 0: amt = 0.001
                     amt = float(Decimal(amt).quantize(Decimal(10)**-8))
-                    jm_single().bc_interface.grab_coins(wallet_service.get_new_addr(
-                        j, populate_internal), amt)
+                    addr = await wallet_service.get_new_addr(
+                        j, populate_internal)
+                    jm_single().bc_interface.grab_coins(addr, amt)
     return wallets
 
 

test/jmclient/test_blockchaininterface.py

@@ -2,172 +2,212 @@
 """Blockchaininterface functionality tests."""
 
 import binascii
-from commontest import create_wallet_for_sync
-
 import pytest
+from unittest import IsolatedAsyncioTestCase
+
+from unittest_parametrize import parametrize, ParametrizedTestCase
+
 from jmbase import get_log
 from jmclient import load_test_config, jm_single, BaseWallet
 
+from commontest import create_wallet_for_sync
+
 log = get_log()
 
 pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
 
 
-def sync_test_wallet(fast, wallet_service):
+async def sync_test_wallet(fast, wallet_service):
     sync_count = 0
     wallet_service.synced = False
     while not wallet_service.synced:
-        wallet_service.sync_wallet(fast=fast)
+        await wallet_service.sync_wallet(fast=fast)
         sync_count += 1
         # avoid infinite loop
         assert sync_count < 10
         log.debug("Tried " + str(sync_count) + " times")
 
 
-@pytest.mark.parametrize('fast', (False, True))
-def test_empty_wallet_sync(setup_wallets, fast):
-    wallet_service = create_wallet_for_sync([0, 0, 0, 0, 0], ['test_empty_wallet_sync'])
-
-    sync_test_wallet(fast, wallet_service)
-
-    broken = True
-    for md in range(wallet_service.max_mixdepth + 1):
-        for internal in (BaseWallet.ADDRESS_TYPE_INTERNAL,
-                         BaseWallet.ADDRESS_TYPE_EXTERNAL):
+@pytest.mark.usefixtures("setup_wallets")
+class AsyncioTestCase(IsolatedAsyncioTestCase, ParametrizedTestCase):
+
+    @parametrize(
+        'fast',
+        [
+            (False,),
+            (True,),
+        ])
+    async def test_empty_wallet_sync(self, fast):
+        wallet_service = await create_wallet_for_sync(
+            [0, 0, 0, 0, 0], ['test_empty_wallet_sync'])
+
+        await sync_test_wallet(fast, wallet_service)
+
+        broken = True
+        for md in range(wallet_service.max_mixdepth + 1):
+            for internal in (BaseWallet.ADDRESS_TYPE_INTERNAL,
+                             BaseWallet.ADDRESS_TYPE_EXTERNAL):
+                broken = False
+                assert 0 == wallet_service.get_next_unused_index(md, internal)
+        assert not broken
+
+    @parametrize(
+        'fast,internal',
+        [
+            (False, BaseWallet.ADDRESS_TYPE_EXTERNAL),
+            (False, BaseWallet.ADDRESS_TYPE_INTERNAL),
+            (True, BaseWallet.ADDRESS_TYPE_EXTERNAL),
+            (True, BaseWallet.ADDRESS_TYPE_INTERNAL)
+        ])
+    async def test_sequentially_used_wallet_sync(self, fast, internal):
+        used_count = [1, 3, 6, 2, 23]
+        wallet_service = await create_wallet_for_sync(
+            used_count, ['test_sequentially_used_wallet_sync'],
+            populate_internal=internal)
+
+        await sync_test_wallet(fast, wallet_service)
+
+        broken = True
+        for md in range(len(used_count)):
+            broken = False
+            assert used_count[md] == wallet_service.get_next_unused_index(md, internal)
+        assert not broken
+
+    @parametrize(
+        'fast',
+        [
+            (False,),
+        ])
+    async def test_gap_used_wallet_sync(self, fast):
+        """ After careful examination this test now only includes the Recovery sync.
+        Note: pre-Aug 2019, because of a bug, this code was not in fact testing both
+        Fast and Recovery sync, but only Recovery (twice). Also, the scenario set
+        out in this test (where coins are funded to a wallet which has no index-cache,
+        and initially no imports) is only appropriate for recovery-mode sync, not for
+        fast-mode (the now default).
+        """
+        used_count = [1, 3, 6, 2, 23]
+        wallet_service = await create_wallet_for_sync(
+            used_count, ['test_gap_used_wallet_sync'])
+        wallet_service.gap_limit = 20
+
+        for md in range(len(used_count)):
+            x = -1
+            for x in range(md):
+                assert x <= wallet_service.gap_limit, "test broken"
+                # create some unused addresses
+                await wallet_service.get_new_script(
+                    md, BaseWallet.ADDRESS_TYPE_INTERNAL)
+                await wallet_service.get_new_script(
+                    md, BaseWallet.ADDRESS_TYPE_EXTERNAL)
+            used_count[md] += x + 2
+            jm_single().bc_interface.grab_coins(
+                await wallet_service.get_new_addr(
+                    md, BaseWallet.ADDRESS_TYPE_INTERNAL), 1)
+            jm_single().bc_interface.grab_coins(
+                await wallet_service.get_new_addr(
+                    md, BaseWallet.ADDRESS_TYPE_EXTERNAL), 1)
+
+        # reset indices to simulate completely unsynced wallet
+        for md in range(wallet_service.max_mixdepth + 1):
+            wallet_service.set_next_index(md, BaseWallet.ADDRESS_TYPE_INTERNAL, 0)
+            wallet_service.set_next_index(md, BaseWallet.ADDRESS_TYPE_EXTERNAL, 0)
+        await sync_test_wallet(fast, wallet_service)
+
+        broken = True
+        for md in range(len(used_count)):
             broken = False
-            assert 0 == wallet_service.get_next_unused_index(md, internal)
-    assert not broken
-
-
-@pytest.mark.parametrize('fast,internal', (
-        (False, BaseWallet.ADDRESS_TYPE_EXTERNAL),
-        (False, BaseWallet.ADDRESS_TYPE_INTERNAL),
-        (True, BaseWallet.ADDRESS_TYPE_EXTERNAL),
-        (True, BaseWallet.ADDRESS_TYPE_INTERNAL)))
-def test_sequentially_used_wallet_sync(setup_wallets, fast, internal):
-    used_count = [1, 3, 6, 2, 23]
-    wallet_service = create_wallet_for_sync(
-        used_count, ['test_sequentially_used_wallet_sync'],
-        populate_internal=internal)
-
-    sync_test_wallet(fast, wallet_service)
-
-    broken = True
-    for md in range(len(used_count)):
-        broken = False
-        assert used_count[md] == wallet_service.get_next_unused_index(md, internal)
-    assert not broken
-
-
-@pytest.mark.parametrize('fast', (False,))
-def test_gap_used_wallet_sync(setup_wallets, fast):
-    """ After careful examination this test now only includes the Recovery sync.
-    Note: pre-Aug 2019, because of a bug, this code was not in fact testing both
-    Fast and Recovery sync, but only Recovery (twice). Also, the scenario set
-    out in this test (where coins are funded to a wallet which has no index-cache,
-    and initially no imports) is only appropriate for recovery-mode sync, not for
-    fast-mode (the now default).
-    """
-    used_count = [1, 3, 6, 2, 23]
-    wallet_service = create_wallet_for_sync(used_count, ['test_gap_used_wallet_sync'])
-    wallet_service.gap_limit = 20
-
-    for md in range(len(used_count)):
-        x = -1
-        for x in range(md):
-            assert x <= wallet_service.gap_limit, "test broken"
-            # create some unused addresses
-            wallet_service.get_new_script(md, BaseWallet.ADDRESS_TYPE_INTERNAL)
-            wallet_service.get_new_script(md, BaseWallet.ADDRESS_TYPE_EXTERNAL)
-        used_count[md] += x + 2
-        jm_single().bc_interface.grab_coins(wallet_service.get_new_addr(md,
-                                        BaseWallet.ADDRESS_TYPE_INTERNAL), 1)
-        jm_single().bc_interface.grab_coins(wallet_service.get_new_addr(md,
-                                        BaseWallet.ADDRESS_TYPE_EXTERNAL), 1)
-
-    # reset indices to simulate completely unsynced wallet
-    for md in range(wallet_service.max_mixdepth + 1):
-        wallet_service.set_next_index(md, BaseWallet.ADDRESS_TYPE_INTERNAL, 0)
-        wallet_service.set_next_index(md, BaseWallet.ADDRESS_TYPE_EXTERNAL, 0)
-    sync_test_wallet(fast, wallet_service)
-
-    broken = True
-    for md in range(len(used_count)):
-        broken = False
-        assert md + 1 == wallet_service.get_next_unused_index(md,
-                                            BaseWallet.ADDRESS_TYPE_INTERNAL)
-        assert used_count[md] == wallet_service.get_next_unused_index(md,
-                                            BaseWallet.ADDRESS_TYPE_EXTERNAL)
-    assert not broken
-
-
-@pytest.mark.parametrize('fast', (False,))
-def test_multigap_used_wallet_sync(setup_wallets, fast):
-    """ See docstring for test_gap_used_wallet_sync; exactly the
-    same applies here.
-    """
-    start_index = 5
-    used_count = [start_index, 0, 0, 0, 0]
-    wallet_service = create_wallet_for_sync(used_count, ['test_multigap_used_wallet_sync'])
-    wallet_service.gap_limit = 5
-
-    mixdepth = 0
-    for w in range(5):
-        for x in range(int(wallet_service.gap_limit * 0.6)):
-            assert x <= wallet_service.gap_limit, "test broken"
-            # create some unused addresses
-            wallet_service.get_new_script(mixdepth,
-                                        BaseWallet.ADDRESS_TYPE_INTERNAL)
-            wallet_service.get_new_script(mixdepth,
-                                        BaseWallet.ADDRESS_TYPE_EXTERNAL)
-        used_count[mixdepth] += x + 2
-        jm_single().bc_interface.grab_coins(wallet_service.get_new_addr(
-            mixdepth, BaseWallet.ADDRESS_TYPE_INTERNAL), 1)
-        jm_single().bc_interface.grab_coins(wallet_service.get_new_addr(
-            mixdepth, BaseWallet.ADDRESS_TYPE_EXTERNAL), 1)
-
-    # reset indices to simulate completely unsynced wallet
-    for md in range(wallet_service.max_mixdepth + 1):
-        wallet_service.set_next_index(md, BaseWallet.ADDRESS_TYPE_INTERNAL, 0)
-        wallet_service.set_next_index(md, BaseWallet.ADDRESS_TYPE_EXTERNAL, 0)
-
-    sync_test_wallet(fast, wallet_service)
-
-    assert used_count[mixdepth] - start_index == \
-           wallet_service.get_next_unused_index(mixdepth,
-                        BaseWallet.ADDRESS_TYPE_INTERNAL)
-    assert used_count[mixdepth] == wallet_service.get_next_unused_index(
-        mixdepth, BaseWallet.ADDRESS_TYPE_EXTERNAL)
-
-
-@pytest.mark.parametrize('fast', (False, True))
-def test_retain_unused_indices_wallet_sync(setup_wallets, fast):
-    used_count = [0, 0, 0, 0, 0]
-    wallet_service = create_wallet_for_sync(used_count,
-                    ['test_retain_unused_indices_wallet_sync'])
-
-    for x in range(9):
-        wallet_service.get_new_script(0, BaseWallet.ADDRESS_TYPE_INTERNAL)
-
-    sync_test_wallet(fast, wallet_service)
-
-    assert wallet_service.get_next_unused_index(0,
-                    BaseWallet.ADDRESS_TYPE_INTERNAL) == 9
-
-
-@pytest.mark.parametrize('fast', (False, True))
-def test_imported_wallet_sync(setup_wallets, fast):
-    used_count = [0, 0, 0, 0, 0]
-    wallet_service = create_wallet_for_sync(used_count, ['test_imported_wallet_sync'])
-    source_wallet_service = create_wallet_for_sync(used_count, ['test_imported_wallet_sync_origin'])
-
-    address = source_wallet_service.get_internal_addr(0)
-    wallet_service.import_private_key(0, source_wallet_service.get_wif(0, 1, 0))
-    txid = binascii.unhexlify(jm_single().bc_interface.grab_coins(address, 1))
-
-    sync_test_wallet(fast, wallet_service)
-
-    assert wallet_service._utxos.have_utxo(txid, 0) == 0
+            assert md + 1 == wallet_service.get_next_unused_index(md,
+                                                BaseWallet.ADDRESS_TYPE_INTERNAL)
+            assert used_count[md] == wallet_service.get_next_unused_index(md,
+                                                BaseWallet.ADDRESS_TYPE_EXTERNAL)
+        assert not broken
+
+    @parametrize(
+        'fast',
+        [
+            (False,),
+        ])
+    async def test_multigap_used_wallet_sync(self, fast):
+        """ See docstring for test_gap_used_wallet_sync; exactly the
+        same applies here.
+        """
+        start_index = 5
+        used_count = [start_index, 0, 0, 0, 0]
+        wallet_service = await create_wallet_for_sync(
+            used_count, ['test_multigap_used_wallet_sync'])
+        wallet_service.gap_limit = 5
+
+        mixdepth = 0
+        for w in range(5):
+            for x in range(int(wallet_service.gap_limit * 0.6)):
+                assert x <= wallet_service.gap_limit, "test broken"
+                # create some unused addresses
+                await wallet_service.get_new_script(
+                    mixdepth, BaseWallet.ADDRESS_TYPE_INTERNAL)
+                await wallet_service.get_new_script(
+                    mixdepth, BaseWallet.ADDRESS_TYPE_EXTERNAL)
+            used_count[mixdepth] += x + 2
+            jm_single().bc_interface.grab_coins(
+                await wallet_service.get_new_addr(
+                    mixdepth, BaseWallet.ADDRESS_TYPE_INTERNAL), 1)
+            jm_single().bc_interface.grab_coins(
+                await wallet_service.get_new_addr(
+                    mixdepth, BaseWallet.ADDRESS_TYPE_EXTERNAL), 1)
+
+        # reset indices to simulate completely unsynced wallet
+        for md in range(wallet_service.max_mixdepth + 1):
+            wallet_service.set_next_index(md, BaseWallet.ADDRESS_TYPE_INTERNAL, 0)
+            wallet_service.set_next_index(md, BaseWallet.ADDRESS_TYPE_EXTERNAL, 0)
+
+        await sync_test_wallet(fast, wallet_service)
+
+        assert used_count[mixdepth] - start_index == \
+               wallet_service.get_next_unused_index(
+                    mixdepth, BaseWallet.ADDRESS_TYPE_INTERNAL)
+        assert used_count[mixdepth] == wallet_service.get_next_unused_index(
+            mixdepth, BaseWallet.ADDRESS_TYPE_EXTERNAL)
+
+    @parametrize(
+        'fast',
+        [
+            (False,),
+            (True,),
+        ])
+    async def test_retain_unused_indices_wallet_sync(self, fast):
+        used_count = [0, 0, 0, 0, 0]
+        wallet_service = await create_wallet_for_sync(
+            used_count, ['test_retain_unused_indices_wallet_sync'])
+
+        for x in range(9):
+            await wallet_service.get_new_script(
+                0, BaseWallet.ADDRESS_TYPE_INTERNAL)
+
+        await sync_test_wallet(fast, wallet_service)
+
+        assert wallet_service.get_next_unused_index(0,
+                        BaseWallet.ADDRESS_TYPE_INTERNAL) == 9
+
+    @parametrize(
+        'fast',
+        [
+            (False,),
+            (True,),
+        ])
+    async def test_imported_wallet_sync(self, fast):
+        used_count = [0, 0, 0, 0, 0]
+        wallet_service = await create_wallet_for_sync(
+            used_count, ['test_imported_wallet_sync'])
+        source_wallet_service = await create_wallet_for_sync(
+            used_count, ['test_imported_wallet_sync_origin'])
+
+        address = await source_wallet_service.get_internal_addr(0)
+        await wallet_service.import_private_key(0, source_wallet_service.get_wif(0, 1, 0))
+        txid = binascii.unhexlify(jm_single().bc_interface.grab_coins(address, 1))
+
+        await sync_test_wallet(fast, wallet_service)
+
+        assert wallet_service._utxos.have_utxo(txid, 0) == 0
 
 
 @pytest.fixture(scope='module')

test/jmclient/test_client_protocol.py

@@ -1,6 +1,16 @@
 #! /usr/bin/env python
 '''test client-protocol interfacae.'''
 
+import json
+import jmbitcoin as bitcoin
+import twisted
+import base64
+
+import pytest
+
+import jmclient  # install asyncioreactor
+from twisted.internet import reactor
+
 from jmbase import get_log, bintohex
 from jmbase.commands import *
 from jmclient import load_test_config, Taker,\
@@ -13,16 +23,9 @@ from twisted.internet.error import (ConnectionLost, ConnectionAborted,
                                     ConnectionClosed, ConnectionDone)
 from twisted.protocols.amp import UnknownRemoteError
 from twisted.protocols import amp
-from twisted.trial import unittest
 from twisted.test import proto_helpers
 from taker_test_data import t_raw_signed_tx
-from commontest import default_max_cj_fee
-import json
-import jmbitcoin as bitcoin
-import twisted
-import base64
-
-import pytest
+from commontest import default_max_cj_fee, TrialAsyncioTestCase
 
 pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
 
@@ -274,7 +277,7 @@ class DummyClientProtocolFactory(JMClientProtocolFactory):
         return JMTakerClientProtocol(self, self.client, nick_priv=b"\xaa"*32 + b"\x01")
 
 
-class TrialTestJMClientProto(unittest.TestCase):
+class TrialTestJMClientProto(TrialAsyncioTestCase):
 
     def setUp(self):
         global clientfactory
@@ -319,7 +322,7 @@ class TrialTestJMClientProto(unittest.TestCase):
         pass
 
 
-class TestMakerClientProtocol(unittest.TestCase):
+class TestMakerClientProtocol(TrialAsyncioTestCase):
     """
     very basic test case for JMMakerClientProtocol
 

test/jmclient/test_coinjoin.py

@@ -5,10 +5,20 @@ Test doing full coinjoins, bypassing IRC
 
 import os
 import sys
-import pytest
 import copy
+import shutil
+import tempfile
+
+from unittest import IsolatedAsyncioTestCase
+
+from unittest_parametrize import parametrize, ParametrizedTestCase
+
+import jmclient  # install asyncioreactor
 from twisted.internet import reactor
 
+import pytest
+from _pytest.monkeypatch import MonkeyPatch
+
 from jmbase import get_log
 from jmclient import load_test_config, jm_single,\
     YieldGeneratorBasic, Taker, LegacyWallet, SegwitLegacyWallet, SegwitWallet,\
@@ -28,6 +38,7 @@ absoffer_type_map = {LegacyWallet: "absoffer",
                   SegwitLegacyWallet: "swabsoffer",
                   SegwitWallet: "sw0absoffer"}
 
+
 def make_wallets_to_list(make_wallets_data):
     wallets = [None for x in range(len(make_wallets_data))]
     for i in make_wallets_data:
@@ -35,14 +46,15 @@ def make_wallets_to_list(make_wallets_data):
     assert all(wallets)
     return wallets
 
-def sync_wallets(wallet_services, fast=True):
+
+async def sync_wallets(wallet_services, fast=True):
     for wallet_service in wallet_services:
         wallet_service.synced = False
         wallet_service.gap_limit = 0
         for x in range(20):
             if wallet_service.synced:
                 break
-            wallet_service.sync_wallet(fast=fast)
+            await wallet_service.sync_wallet(fast=fast)
         else:
             assert False, "Failed to sync wallet"
     # because we don't run the monitoring loops for the
@@ -51,6 +63,7 @@ def sync_wallets(wallet_services, fast=True):
     for wallet_service in wallet_services:
         wallet_service.update_blockheight()
 
+
 def create_orderbook(makers):
     orderbook = []
     for i in range(len(makers)):
@@ -75,21 +88,23 @@ def create_taker(wallet, schedule, monkeypatch):
     monkeypatch.setattr(taker, 'auth_counterparty', lambda *args: True)
     return taker
 
-def create_orders(makers):
+
+async def create_orders(makers):
     # fire the order creation immediately (delayed 2s in prod,
     # but this is too slow for test):
     for maker in makers:
-        maker.try_to_create_my_orders()
+        await maker.try_to_create_my_orders()
 
-def init_coinjoin(taker, makers, orderbook, cj_amount):
-    init_data = taker.initialize(orderbook, [])
+
+async def init_coinjoin(taker, makers, orderbook, cj_amount):
+    init_data = await taker.initialize(orderbook, [])
     assert init_data[0], "taker.initialize error"
     active_orders = init_data[4]
     maker_data = {}
     for mid in init_data[4]:
         m = makers[int(mid)]
         # note: '00' is kphex, usually set up by jmdaemon
-        response = m.on_auth_received(
+        response = await m.on_auth_received(
             'TAKER', init_data[4][mid], init_data[2][1:],
             init_data[3], init_data[1], '00')
         assert response[0], "maker.on_auth_received error"
@@ -109,180 +124,209 @@ def init_coinjoin(taker, makers, orderbook, cj_amount):
     return active_orders, maker_data
 
 
-def do_tx_signing(taker, makers, active_orders, txdata):
+async def do_tx_signing(taker, makers, active_orders, txdata):
     taker_final_result = 'not called'
     maker_signatures = {}  # left here for easier debugging
     for mid in txdata[1]:
         m = makers[int(mid)]
-        result = m.on_tx_received('TAKER', txdata[2], active_orders[mid])
+        result = await m.on_tx_received('TAKER', txdata[2], active_orders[mid])
         assert result[0], "maker.on_tx_received error"
         maker_signatures[mid] = result[1]
         for sig in result[1]:
-            taker_final_result = taker.on_sig(mid, sig)
+            taker_final_result = await taker.on_sig(mid, sig)
 
     assert taker_final_result != 'not called'
     return taker_final_result
 
 
-@pytest.mark.parametrize('wallet_cls', (LegacyWallet, SegwitLegacyWallet, SegwitWallet))
-def test_simple_coinjoin(monkeypatch, tmpdir, setup_cj, wallet_cls):
-    def raise_exit(i):
-        raise Exception("sys.exit called")
-    monkeypatch.setattr(sys, 'exit', raise_exit)
-    set_commitment_file(str(tmpdir.join('commitments.json')))
-
-    MAKER_NUM = 3
-    wallet_services = make_wallets_to_list(make_wallets(
-        MAKER_NUM + 1, wallet_structures=[[4, 0, 0, 0, 0]] * (MAKER_NUM + 1),
-        mean_amt=1, wallet_cls=wallet_cls))
-
-    jm_single().bc_interface.tickchain()
-    jm_single().bc_interface.tickchain()
-
-    sync_wallets(wallet_services)
-
-    makers = [YieldGeneratorBasic(
-        wallet_services[i],
-        [0, 2000, 0, absoffer_type_map[wallet_cls], 10**7, None, None, None]) for i in range(MAKER_NUM)]
-    create_orders(makers)
-
-    orderbook = create_orderbook(makers)
-    assert len(orderbook) == MAKER_NUM
-
-    cj_amount = int(1.1 * 10**8)
-    # mixdepth, amount, counterparties, dest_addr, waittime, rounding
-    schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0, NO_ROUNDING)]
-    taker = create_taker(wallet_services[-1], schedule, monkeypatch)
-
-    active_orders, maker_data = init_coinjoin(taker, makers,
-                                              orderbook, cj_amount)
-
-    txdata = taker.receive_utxos(maker_data)
-    assert txdata[0], "taker.receive_utxos error"
-
-    taker_final_result = do_tx_signing(taker, makers, active_orders, txdata)
-    assert taker_final_result is not False
-    assert taker.on_finished_callback.status is not False
-
-
-def test_coinjoin_mixdepth_wrap_taker(monkeypatch, tmpdir, setup_cj):
-    def raise_exit(i):
-        raise Exception("sys.exit called")
-    monkeypatch.setattr(sys, 'exit', raise_exit)
-    set_commitment_file(str(tmpdir.join('commitments.json')))
-
-    MAKER_NUM = 3
-    wallet_services = make_wallets_to_list(make_wallets(
-        MAKER_NUM + 1,
-        wallet_structures=[[4, 0, 0, 0, 0]] * MAKER_NUM + [[0, 0, 0, 0, 3]],
-        mean_amt=1))
-
-    for wallet_service in wallet_services:
-        assert wallet_service.max_mixdepth == 4
-
-    jm_single().bc_interface.tickchain()
-    jm_single().bc_interface.tickchain()
-
-    sync_wallets(wallet_services)
-
-    cj_fee = 2000
-    makers = [YieldGeneratorBasic(
-        wallet_services[i],
-        [0, cj_fee, 0, absoffer_type_map[SegwitWallet], 10**7, None, None, None]) for i in range(MAKER_NUM)]
-    create_orders(makers)
-
-    orderbook = create_orderbook(makers)
-    assert len(orderbook) == MAKER_NUM
-
-    cj_amount = int(1.1 * 10**8)
-    # mixdepth, amount, counterparties, dest_addr, waittime, rounding
-    schedule = [(4, cj_amount, MAKER_NUM, 'INTERNAL', 0, NO_ROUNDING)]
-    taker = create_taker(wallet_services[-1], schedule, monkeypatch)
-
-    active_orders, maker_data = init_coinjoin(taker, makers,
-                                              orderbook, cj_amount)
-
-    txdata = taker.receive_utxos(maker_data)
-    assert txdata[0], "taker.receive_utxos error"
-
-    taker_final_result = do_tx_signing(taker, makers, active_orders, txdata)
-    assert taker_final_result is not False
-
-    tx = btc.CMutableTransaction.deserialize(txdata[2])
-
-    wallet_service = wallet_services[-1]
-    # TODO change for new tx monitoring:
-    wallet_service.remove_old_utxos(tx)
-    wallet_service.add_new_utxos(tx)
-
-    balances = wallet_service.get_balance_by_mixdepth()
-    assert balances[0] == cj_amount
-    # <= because of tx fee
-    assert balances[4] <= 3 * 10**8 - cj_amount - (cj_fee * MAKER_NUM)
-
-
-def test_coinjoin_mixdepth_wrap_maker(monkeypatch, tmpdir, setup_cj):
-    def raise_exit(i):
-        raise Exception("sys.exit called")
-    monkeypatch.setattr(sys, 'exit', raise_exit)
-    set_commitment_file(str(tmpdir.join('commitments.json')))
-
-    MAKER_NUM = 2
-    wallet_services = make_wallets_to_list(make_wallets(
-        MAKER_NUM + 1,
-        wallet_structures=[[0, 0, 0, 0, 4]] * MAKER_NUM + [[3, 0, 0, 0, 0]],
-        mean_amt=1))
-
-    for wallet_service in wallet_services:
-        assert wallet_service.max_mixdepth == 4
-
-    jm_single().bc_interface.tickchain()
-    jm_single().bc_interface.tickchain()
-
-    sync_wallets(wallet_services)
-
-    cj_fee = 2000
-    makers = [YieldGeneratorBasic(
-        wallet_services[i],
-        [0, cj_fee, 0, absoffer_type_map[SegwitWallet], 10**7, None, None, None]) for i in range(MAKER_NUM)]
-    create_orders(makers)
-    orderbook = create_orderbook(makers)
-    assert len(orderbook) == MAKER_NUM
-
-    cj_amount = int(1.1 * 10**8)
-    # mixdepth, amount, counterparties, dest_addr, waittime, rounding
-    schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0, NO_ROUNDING)]
-    taker = create_taker(wallet_services[-1], schedule, monkeypatch)
-
-    active_orders, maker_data = init_coinjoin(taker, makers,
-                                              orderbook, cj_amount)
-
-    txdata = taker.receive_utxos(maker_data)
-    assert txdata[0], "taker.receive_utxos error"
-
-    taker_final_result = do_tx_signing(taker, makers, active_orders, txdata)
-    assert taker_final_result is not False
-
-    tx = btc.CMutableTransaction.deserialize(txdata[2])
-
-    for i in range(MAKER_NUM):
-        wallet_service = wallet_services[i]
-        # TODO as above re: monitoring
-        wallet_service.remove_old_utxos(tx)
+#@pytest.mark.usefixtures("setup_cj")
+class AsyncioTestCase(IsolatedAsyncioTestCase, ParametrizedTestCase):
+
+    def setUp(self):
+        self.tmpdir = tempfile.mkdtemp()
+        load_test_config()
+        jm_single().config.set('POLICY', 'tx_broadcast', 'self')
+        jm_single().bc_interface.tick_forward_chain_interval = 5
+        jm_single().bc_interface.simulate_blocks()
+        sys._exit_ = sys.exit
+
+    def tearDown(self):
+        monkeypatch = MonkeyPatch()
+        monkeypatch.setattr(sys, 'exit', sys._exit_)
+        for dc in reactor.getDelayedCalls():
+            dc.cancel()
+        shutil.rmtree(self.tmpdir)
+
+    @parametrize(
+        'wallet_cls',
+        [
+            (LegacyWallet,),
+            (SegwitLegacyWallet,),
+            (SegwitWallet,),
+        ])
+    async def test_simple_coinjoin(self, wallet_cls):
+        def raise_exit(i):
+            raise Exception("sys.exit called")
+        monkeypatch = MonkeyPatch()
+        monkeypatch.setattr(sys, 'exit', raise_exit)
+        commitment_file = os.path.join(self.tmpdir, 'commitments.json')
+        set_commitment_file(commitment_file)
+
+        MAKER_NUM = 3
+        wallets = await make_wallets(
+            MAKER_NUM + 1, wallet_structures=[[4, 0, 0, 0, 0]] * (MAKER_NUM + 1),
+            mean_amt=1, wallet_cls=wallet_cls)
+        wallet_services = make_wallets_to_list(wallets)
+
+        jm_single().bc_interface.tickchain()
+        jm_single().bc_interface.tickchain()
+
+        await sync_wallets(wallet_services)
+
+        makers = [
+            YieldGeneratorBasic(
+                wallet_services[i],
+                [0, 2000, 0, absoffer_type_map[wallet_cls],
+                 10**7, None, None, None])
+            for i in range(MAKER_NUM)]
+        await create_orders(makers)
+        orderbook = create_orderbook(makers)
+        assert len(orderbook) == MAKER_NUM
+
+        cj_amount = int(1.1 * 10**8)
+        # mixdepth, amount, counterparties, dest_addr, waittime, rounding
+        schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0, NO_ROUNDING)]
+        taker = create_taker(wallet_services[-1], schedule, monkeypatch)
+
+        active_orders, maker_data = await init_coinjoin(
+            taker, makers, orderbook, cj_amount)
+
+        txdata = await taker.receive_utxos(maker_data)
+        assert txdata[0], "taker.receive_utxos error"
+
+        taker_final_result = await do_tx_signing(
+            taker, makers, active_orders, txdata)
+        assert taker_final_result is not False
+        assert taker.on_finished_callback.status is not False
+
+    async def test_coinjoin_mixdepth_wrap_taker(self):
+        def raise_exit(i):
+            raise Exception("sys.exit called")
+        monkeypatch = MonkeyPatch()
+        monkeypatch.setattr(sys, 'exit', raise_exit)
+        commitment_file = os.path.join(self.tmpdir, 'commitments.json')
+        set_commitment_file(commitment_file)
+
+        MAKER_NUM = 3
+        wallets = await make_wallets(
+            MAKER_NUM + 1,
+            wallet_structures=[[4, 0, 0, 0, 0]] * MAKER_NUM + [[0, 0, 0, 0, 3]],
+            mean_amt=1)
+        wallet_services = make_wallets_to_list(wallets)
+
+        for wallet_service in wallet_services:
+            assert wallet_service.max_mixdepth == 4
+
+        jm_single().bc_interface.tickchain()
+        jm_single().bc_interface.tickchain()
+
+        await sync_wallets(wallet_services)
+
+        cj_fee = 2000
+        makers = [
+            YieldGeneratorBasic(
+                wallet_services[i],
+                [0, cj_fee, 0, absoffer_type_map[SegwitWallet],
+                 10**7, None, None, None])
+            for i in range(MAKER_NUM)]
+        await create_orders(makers)
+        orderbook = create_orderbook(makers)
+        assert len(orderbook) == MAKER_NUM
+
+        cj_amount = int(1.1 * 10**8)
+        # mixdepth, amount, counterparties, dest_addr, waittime, rounding
+        schedule = [(4, cj_amount, MAKER_NUM, 'INTERNAL', 0, NO_ROUNDING)]
+        taker = create_taker(wallet_services[-1], schedule, monkeypatch)
+
+        active_orders, maker_data = await init_coinjoin(
+            taker, makers, orderbook, cj_amount)
+
+        txdata = await taker.receive_utxos(maker_data)
+        assert txdata[0], "taker.receive_utxos error"
+
+        taker_final_result = await do_tx_signing(
+            taker, makers, active_orders, txdata)
+        assert taker_final_result is not False
+
+        tx = btc.CMutableTransaction.deserialize(txdata[2])
+
+        wallet_service = wallet_services[-1]
+        # TODO change for new tx monitoring:
+        await wallet_service.remove_old_utxos(tx)
         wallet_service.add_new_utxos(tx)
 
         balances = wallet_service.get_balance_by_mixdepth()
         assert balances[0] == cj_amount
-        assert balances[4] == 4 * 10**8 - cj_amount + cj_fee
-
-
-@pytest.fixture(scope='module')
-def setup_cj():
-    load_test_config()
-    jm_single().config.set('POLICY', 'tx_broadcast', 'self')
-    jm_single().bc_interface.tick_forward_chain_interval = 5
-    jm_single().bc_interface.simulate_blocks()
-    yield None
-    # teardown
-    for dc in reactor.getDelayedCalls():
-        dc.cancel()
+        # <= because of tx fee
+        assert balances[4] <= 3 * 10**8 - cj_amount - (cj_fee * MAKER_NUM)
+
+    async def test_coinjoin_mixdepth_wrap_maker(self):
+        def raise_exit(i):
+            raise Exception("sys.exit called")
+        monkeypatch = MonkeyPatch()
+        monkeypatch.setattr(sys, 'exit', raise_exit)
+        commitment_file = os.path.join(self.tmpdir, 'commitments.json')
+        set_commitment_file(commitment_file)
+
+        MAKER_NUM = 2
+        wallets = await make_wallets(
+            MAKER_NUM + 1,
+            wallet_structures=[[0, 0, 0, 0, 4]] * MAKER_NUM + [[3, 0, 0, 0, 0]],
+            mean_amt=1)
+        wallet_services = make_wallets_to_list(wallets)
+
+        for wallet_service in wallet_services:
+            assert wallet_service.max_mixdepth == 4
+
+        jm_single().bc_interface.tickchain()
+        jm_single().bc_interface.tickchain()
+
+        await sync_wallets(wallet_services)
+
+        cj_fee = 2000
+        makers = [
+            YieldGeneratorBasic(
+                wallet_services[i],
+                [0, cj_fee, 0, absoffer_type_map[SegwitWallet],
+                 10**7, None, None, None])
+            for i in range(MAKER_NUM)]
+        await create_orders(makers)
+        orderbook = create_orderbook(makers)
+        assert len(orderbook) == MAKER_NUM
+
+        cj_amount = int(1.1 * 10**8)
+        # mixdepth, amount, counterparties, dest_addr, waittime, rounding
+        schedule = [(0, cj_amount, MAKER_NUM, 'INTERNAL', 0, NO_ROUNDING)]
+        taker = create_taker(wallet_services[-1], schedule, monkeypatch)
+
+        active_orders, maker_data = await init_coinjoin(
+            taker, makers, orderbook, cj_amount)
+
+        txdata = await taker.receive_utxos(maker_data)
+        assert txdata[0], "taker.receive_utxos error"
+
+        taker_final_result = await do_tx_signing(
+            taker, makers, active_orders, txdata)
+        assert taker_final_result is not False
+
+        tx = btc.CMutableTransaction.deserialize(txdata[2])
+
+        for i in range(MAKER_NUM):
+            wallet_service = wallet_services[i]
+            # TODO as above re: monitoring
+            await wallet_service.remove_old_utxos(tx)
+            wallet_service.add_new_utxos(tx)
+
+            balances = wallet_service.get_balance_by_mixdepth()
+            assert balances[0] == cj_amount
+            assert balances[4] == 4 * 10**8 - cj_amount + cj_fee

test/jmclient/test_core_nohistory_sync.py

@@ -3,53 +3,66 @@
 
 """BitcoinCoreNoHistoryInterface functionality tests."""
 
-from commontest import create_wallet_for_sync
+from unittest import IsolatedAsyncioTestCase
+
+from unittest_parametrize import parametrize, ParametrizedTestCase
+
+import jmclient  # install asyncioreactor
+from twisted.internet import reactor
 
 import pytest
 from jmbase import get_log
 from jmclient import (load_test_config, SegwitLegacyWallet,
                       SegwitWallet, jm_single, BaseWallet)
 from jmbitcoin import select_chain_params
+from commontest import create_wallet_for_sync
 
 pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
 
 log = get_log()
 
-def test_fast_sync_unavailable(setup_sync):
-    wallet_service = create_wallet_for_sync([0, 0, 0, 0, 0],
-        ['test_fast_sync_unavailable'])
-    with pytest.raises(RuntimeError) as e_info:
-        wallet_service.sync_wallet(fast=True)
-
-@pytest.mark.parametrize('internal, wallet_cls',
-    [(BaseWallet.ADDRESS_TYPE_EXTERNAL, SegwitLegacyWallet),
-    (BaseWallet.ADDRESS_TYPE_INTERNAL, SegwitLegacyWallet),
-    (BaseWallet.ADDRESS_TYPE_EXTERNAL, SegwitWallet),
-    (BaseWallet.ADDRESS_TYPE_INTERNAL, SegwitWallet)])
-def test_sync(setup_sync, internal, wallet_cls):
-    used_count = [1, 3, 6, 2, 23]
-    wallet_service = create_wallet_for_sync(used_count, ['test_sync'],
-        populate_internal=internal, wallet_cls=wallet_cls)
-    ##the gap limit should be not zero before sync
-    assert wallet_service.gap_limit > 0
-    for md in range(len(used_count)):
-        ##obtaining an address should be possible without error before sync
-        wallet_service.get_new_script(md, internal)
-
-    # TODO bci should probably not store this state globally,
-    # in case syncing is needed for multiple wallets (as in this test):
-    jm_single().bc_interface.import_addresses_call_count = 0
-    wallet_service.sync_wallet(fast=False)
-
-    for md in range(len(used_count)):
-        ##plus one to take into account the one new script obtained above
-        assert used_count[md] + 1 == wallet_service.get_next_unused_index(md,
-            internal)
-    #gap limit is zero after sync
-    assert wallet_service.gap_limit == 0
-    #obtaining an address leads to an error after sync
-    with pytest.raises(RuntimeError) as e_info:
-        wallet_service.get_new_script(0, internal)
+@pytest.mark.usefixtures("setup_sync")
+class AsyncioTestCase(IsolatedAsyncioTestCase, ParametrizedTestCase):
+
+    async def test_fast_sync_unavailable(setup_sync):
+        wallet_service = await create_wallet_for_sync(
+            [0, 0, 0, 0, 0], ['test_fast_sync_unavailable'])
+        with pytest.raises(RuntimeError) as e_info:
+            await wallet_service.sync_wallet(fast=True)
+
+    @parametrize(
+        'internal, wallet_cls',
+        [
+            (BaseWallet.ADDRESS_TYPE_EXTERNAL, SegwitLegacyWallet),
+            (BaseWallet.ADDRESS_TYPE_INTERNAL, SegwitLegacyWallet),
+            (BaseWallet.ADDRESS_TYPE_EXTERNAL, SegwitWallet),
+            (BaseWallet.ADDRESS_TYPE_INTERNAL, SegwitWallet),
+        ])
+    async def test_sync(setup_sync, internal, wallet_cls):
+        used_count = [1, 3, 6, 2, 23]
+        wallet_service = await create_wallet_for_sync(
+            used_count, ['test_sync'],
+            populate_internal=internal, wallet_cls=wallet_cls)
+        ##the gap limit should be not zero before sync
+        assert wallet_service.gap_limit > 0
+        for md in range(len(used_count)):
+            ##obtaining an address should be possible without error before sync
+            await wallet_service.get_new_script(md, internal)
+
+        # TODO bci should probably not store this state globally,
+        # in case syncing is needed for multiple wallets (as in this test):
+        jm_single().bc_interface.import_addresses_call_count = 0
+        await wallet_service.sync_wallet(fast=False)
+
+        for md in range(len(used_count)):
+            ##plus one to take into account the one new script obtained above
+            assert used_count[md] + 1 == wallet_service.get_next_unused_index(
+                md, internal)
+        #gap limit is zero after sync
+        assert wallet_service.gap_limit == 0
+        #obtaining an address leads to an error after sync
+        with pytest.raises(RuntimeError) as e_info:
+            await wallet_service.get_new_script(0, internal)
 
 
 @pytest.fixture(scope='module')

test/jmclient/test_maker.py

@@ -1,5 +1,10 @@
 import datetime
 
+from unittest import IsolatedAsyncioTestCase
+
+import jmclient  # install asyncioreactor
+from twisted.internet import reactor
+
 import jmbitcoin as btc
 from jmclient import Maker, load_test_config, jm_single, WalletService, VolatileStorage, \
     SegwitWalletFidelityBonds, get_network
@@ -11,6 +16,7 @@ import struct
 import binascii
 from itertools import chain
 import pytest
+from _pytest.monkeypatch import MonkeyPatch
 
 
 class OfflineMaker(Maker):
@@ -101,91 +107,102 @@ def create_tx_and_offerlist(cj_addr, cj_change_addr, other_output_addrs,
     return tx, offerlist
 
 
-def test_verify_unsigned_tx_sw_valid(setup_env_nodeps):
-    jm_single().config.set("POLICY", "segwit", "true")
-
-    p2sh_gen = address_p2sh_generator()
-    p2pkh_gen = address_p2pkh_generator()
+class AsyncioTestCase(IsolatedAsyncioTestCase):
 
-    wallet = DummyWallet()
-    maker = OfflineMaker(WalletService(wallet))
+    def setUp(self):
+        jmclient.configure._get_bc_interface_instance_ = \
+            jmclient.configure.get_blockchain_interface_instance
+        monkeypatch = MonkeyPatch()
+        monkeypatch.setattr(jmclient.configure,
+                            'get_blockchain_interface_instance',
+                            lambda x: DummyBlockchainInterface())
+        btc.select_chain_params("bitcoin/regtest")
+        load_test_config()
 
-    cj_addr, cj_script = next(p2sh_gen)
-    changeaddr, cj_change_script = next(p2sh_gen)
+    def tearDown(self):
+        monkeypatch = MonkeyPatch()
+        monkeypatch.setattr(jmclient.configure,
+                            'get_blockchain_interface_instance',
+                            jmclient.configure._get_bc_interface_instance_)
 
-    # test standard cj
-    tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr,
-        [next(p2sh_gen)[0] for s in range(4)])
+    async def test_verify_unsigned_tx_sw_valid(self):
+        jm_single().config.set("POLICY", "segwit", "true")
 
-    assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "standard sw cj"
+        p2sh_gen = address_p2sh_generator()
+        p2pkh_gen = address_p2pkh_generator()
 
-    # test cj with mixed outputs
-    tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr,
-        list(chain((next(p2sh_gen)[0] for s in range(3)),
-                   (next(p2pkh_gen)[0] for s in range(1)))))
+        wallet = DummyWallet()
+        await wallet.async_init(wallet.storage)
+        maker = OfflineMaker(WalletService(wallet))
 
-    assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "sw cj with p2pkh output"
+        cj_addr, cj_script = next(p2sh_gen)
+        changeaddr, cj_change_script = next(p2sh_gen)
 
-    # test cj with only p2pkh outputs
-    tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr,
-        [next(p2pkh_gen)[0] for s in range(4)])
+        # test standard cj
+        tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr,
+            [next(p2sh_gen)[0] for s in range(4)])
 
-    assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "sw cj with only p2pkh outputs"
+        assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "standard sw cj"
 
+        # test cj with mixed outputs
+        tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr,
+            list(chain((next(p2sh_gen)[0] for s in range(3)),
+                       (next(p2pkh_gen)[0] for s in range(1)))))
 
-def test_verify_unsigned_tx_nonsw_valid(setup_env_nodeps):
-    jm_single().config.set("POLICY", "segwit", "false")
+        assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "sw cj with p2pkh output"
 
-    p2sh_gen = address_p2sh_generator()
-    p2pkh_gen = address_p2pkh_generator()
+        # test cj with only p2pkh outputs
+        tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr,
+            [next(p2pkh_gen)[0] for s in range(4)])
 
-    wallet = DummyWallet()
-    maker = OfflineMaker(WalletService(wallet))
+        assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "sw cj with only p2pkh outputs"
 
-    cj_addr, cj_script = next(p2pkh_gen)
-    changeaddr, cj_change_script = next(p2pkh_gen)
+    async def test_verify_unsigned_tx_nonsw_valid(self):
+        jm_single().config.set("POLICY", "segwit", "false")
 
-    # test standard cj
-    tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr,
-        [next(p2pkh_gen)[0] for s in range(4)], offertype='reloffer')
+        p2sh_gen = address_p2sh_generator()
+        p2pkh_gen = address_p2pkh_generator()
 
-    assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "standard nonsw cj"
+        wallet = DummyWallet()
+        await wallet.async_init(wallet.storage)
+        maker = OfflineMaker(WalletService(wallet))
 
-    # test cj with mixed outputs
-    tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr,
-        list(chain((next(p2sh_gen)[0] for s in range(1)),
-                   (next(p2pkh_gen)[0] for s in range(3)))), offertype='reloffer')
+        cj_addr, cj_script = next(p2pkh_gen)
+        changeaddr, cj_change_script = next(p2pkh_gen)
 
-    assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "nonsw cj with p2sh output"
+        # test standard cj
+        tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr,
+            [next(p2pkh_gen)[0] for s in range(4)], offertype='reloffer')
 
-    # test cj with only p2sh outputs
-    tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr,
-        [next(p2sh_gen)[0] for s in range(4)], offertype='reloffer')
+        assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "standard nonsw cj"
 
-    assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "nonsw cj with only p2sh outputs"
+        # test cj with mixed outputs
+        tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr,
+            list(chain((next(p2sh_gen)[0] for s in range(1)),
+                       (next(p2pkh_gen)[0] for s in range(3)))), offertype='reloffer')
 
+        assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "nonsw cj with p2sh output"
 
-def test_freeze_timelocked_utxos(setup_env_nodeps):
-    storage = VolatileStorage()
-    SegwitWalletFidelityBonds.initialize(storage, get_network())
-    wallet = SegwitWalletFidelityBonds(storage)
-    ts = wallet.datetime_to_time_number(
-        datetime.datetime.strptime("2021-07", "%Y-%m"))
-    tl_path = wallet.get_path(
-        wallet.FIDELITY_BOND_MIXDEPTH, wallet.BIP32_TIMELOCK_ID, ts)
-    tl_script = wallet.get_script_from_path(tl_path)
-    utxo = (b'a'*32, 0)
-    wallet.add_utxo(utxo[0], utxo[1], tl_script, 100000000)
-    assert not wallet._utxos.is_disabled(*utxo)
+        # test cj with only p2sh outputs
+        tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr,
+            [next(p2sh_gen)[0] for s in range(4)], offertype='reloffer')
 
-    maker = OfflineMaker(WalletService(wallet))
-    maker.freeze_timelocked_utxos()
-    assert wallet._utxos.is_disabled(*utxo)
+        assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "nonsw cj with only p2sh outputs"
 
+    async def test_freeze_timelocked_utxos(self):
+        storage = VolatileStorage()
+        SegwitWalletFidelityBonds.initialize(storage, get_network())
+        wallet = SegwitWalletFidelityBonds(storage)
+        await wallet.async_init(storage)
+        ts = wallet.datetime_to_time_number(
+            datetime.datetime.strptime("2021-07", "%Y-%m"))
+        tl_path = wallet.get_path(
+            wallet.FIDELITY_BOND_MIXDEPTH, wallet.BIP32_TIMELOCK_ID, ts)
+        tl_script = await wallet.get_script_from_path(tl_path)
+        utxo = (b'a'*32, 0)
+        wallet.add_utxo(utxo[0], utxo[1], tl_script, 100000000)
+        assert not wallet._utxos.is_disabled(*utxo)
 
-@pytest.fixture
-def setup_env_nodeps(monkeypatch):
-    monkeypatch.setattr(jmclient.configure, 'get_blockchain_interface_instance',
-                        lambda x: DummyBlockchainInterface())
-    btc.select_chain_params("bitcoin/regtest")
-    load_test_config()
+        maker = OfflineMaker(WalletService(wallet))
+        await maker.freeze_timelocked_utxos()
+        assert wallet._utxos.is_disabled(*utxo)

test/jmclient/test_payjoin.py

@@ -6,8 +6,8 @@ Test doing payjoins over tcp client/server
 import os
 
 import pytest
-from twisted.internet import reactor
-from twisted.web.server import Site
+from twisted.internet import reactor, defer
+from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.client import readBody
 from twisted.web.http_headers import Headers
 from twisted.trial import unittest
@@ -25,7 +25,7 @@ from jmclient import (load_test_config, jm_single,
                       JMBIP78ReceiverManager)
 from jmclient.payjoin import make_payjoin_request_params, make_payment_psbt
 from jmclient.payjoin import process_payjoin_proposal_from_server
-from commontest import make_wallets
+from commontest import make_wallets, TrialTestCase
 from test_coinjoin import make_wallets_to_list, sync_wallets
 
 pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
@@ -47,12 +47,19 @@ class DummyBIP78ReceiverResource(JMHTTPResource):
     def render_POST(self, request):
         proposed_tx = request.content
         payment_psbt_base64 = proposed_tx.read().decode("utf-8")
-        retval = self.bip78_receiver_manager.receive_proposal_from_sender(
-            payment_psbt_base64, request.args)
-        assert retval[0]
-        content = retval[1].encode("utf-8")
-        request.setHeader(b"content-length", ("%d" % len(content)))
-        return content
+        d = defer.Deferred.fromCoroutine(
+            self.bip78_receiver_manager.receive_proposal_from_sender(
+                payment_psbt_base64, request.args))
+
+        def _delayedRender(result, request):
+            assert result[0]
+            content = result[1].encode("utf-8")
+            request.setHeader(b"content-length", ("%d" % len(content)))
+            request.write(content)
+            request.finish()
+
+        d.addCallback(_delayedRender, request)
+        return NOT_DONE_YET
 
 class PayjoinTestBase(object):
     """ This tests that a payjoin invoice and
@@ -70,18 +77,20 @@ class PayjoinTestBase(object):
         jm_single().bc_interface.tick_forward_chain_interval = 5
         jm_single().bc_interface.simulate_blocks()
 
-    def do_test_payment(self, wc1, wc2, amt=1.1):
+    async def do_test_payment(self, wc1, wc2, amt=1.1):
         wallet_structures = [self.wallet_structure] * 2
         wallet_cls = (wc1, wc2)
         self.wallet_services = []
-        self.wallet_services.append(make_wallets_to_list(make_wallets(
+        wallets = await make_wallets(
             1, wallet_structures=[wallet_structures[0]],
-            mean_amt=self.mean_amt, wallet_cls=wallet_cls[0]))[0])
-        self.wallet_services.append(make_wallets_to_list(make_wallets(
+            mean_amt=self.mean_amt, wallet_cls=wallet_cls[0])
+        self.wallet_services.append(make_wallets_to_list(wallets)[0])
+        wallets = await make_wallets(
                 1, wallet_structures=[wallet_structures[1]],
-                mean_amt=self.mean_amt, wallet_cls=wallet_cls[1]))[0])
+                mean_amt=self.mean_amt, wallet_cls=wallet_cls[1])
+        self.wallet_services.append(make_wallets_to_list(wallets)[0])
         jm_single().bc_interface.tickchain()
-        sync_wallets(self.wallet_services)
+        await sync_wallets(self.wallet_services)
 
         # For accounting purposes, record the balances
         # at the start.
@@ -93,6 +102,7 @@ class PayjoinTestBase(object):
             return self.port.stopListening()
         b78rm = JMBIP78ReceiverManager(self.wallet_services[0], 0,
                                        self.cj_amount, 47083)
+        await b78rm.async_init(self.wallet_services[0], 0, self.cj_amount)
         resource = DummyBIP78ReceiverResource(jmprint, cbStopListening, b78rm)
         self.site = Site(resource)
         self.site.displayTracebacks = False
@@ -109,7 +119,7 @@ class PayjoinTestBase(object):
                                 safe=":/")
         self.manager = parse_payjoin_setup(bip78_uri, self.wallet_services[1], 0)
         self.manager.mode = "testing"
-        success, msg = make_payment_psbt(self.manager)
+        success, msg = await make_payment_psbt(self.manager)
         assert success, msg
         params = make_payjoin_request_params(self.manager)
         # avoiding backend daemon (testing only jmclient code here),
@@ -120,38 +130,52 @@ class PayjoinTestBase(object):
         url_parts = list(wrapped_urlparse(serv))
         url_parts[4] = urlencode(params).encode("utf-8")
         destination_url = urlparse.urlunparse(url_parts)
-        d = agent.request(b"POST", destination_url,
-                          Headers({"Content-Type": ["text/plain"]}),
-                          bodyProducer=body)
-        d.addCallback(bip78_receiver_response, self.manager)
-        return d
+        response = await agent.request(
+            b"POST", destination_url,
+            Headers({"Content-Type": ["text/plain"]}),
+            bodyProducer=body)
+        await bip78_receiver_response(response, self.manager)
+        return response
 
     def tearDown(self):
         for dc in reactor.getDelayedCalls():
             dc.cancel()
-        res = final_checks(self.wallet_services, self.cj_amount,
-                           self.manager.final_psbt.get_fee(),
-                           self.ssb, self.rsb)
-        assert res, "final checks failed"
+        d = defer.ensureDeferred(
+            final_checks(self.wallet_services, self.cj_amount,
+                         self.manager.final_psbt.get_fee(),
+                         self.ssb, self.rsb))
+        assert d, "final checks failed"
+
+
+class TrialTestPayjoin1(PayjoinTestBase, TrialTestCase):
 
-class TrialTestPayjoin1(PayjoinTestBase, unittest.TestCase):
     def test_payment(self):
-        return self.do_test_payment(SegwitLegacyWallet, SegwitLegacyWallet)
+        coro = self.do_test_payment(SegwitLegacyWallet, SegwitLegacyWallet)
+        d = defer.Deferred.fromCoroutine(coro)
+        return d
+
+class TrialTestPayjoin2(PayjoinTestBase, TrialTestCase):
 
-class TrialTestPayjoin2(PayjoinTestBase, unittest.TestCase):
     def test_bech32_payment(self):
-        return self.do_test_payment(SegwitWallet, SegwitWallet)
+        coro = self.do_test_payment(SegwitWallet, SegwitWallet)
+        d = defer.Deferred.fromCoroutine(coro)
+        return d
+
+class TrialTestPayjoin3(PayjoinTestBase, TrialTestCase):
 
-class TrialTestPayjoin3(PayjoinTestBase, unittest.TestCase):
     def test_multi_input(self):
         # wallet structure and amt are chosen so that the sender
         # will need 3 utxos rather than 1 (to pay 4.5 from 2,2,2).
         self.wallet_structure = [3, 1, 0, 0, 0]
-        return self.do_test_payment(SegwitWallet, SegwitWallet, amt=4.5)
+        coro = self.do_test_payment(SegwitWallet, SegwitWallet, amt=4.5)
+        d = defer.Deferred.fromCoroutine(coro)
+        return d
+
+class TrialTestPayjoin4(PayjoinTestBase, TrialTestCase):
 
-class TrialTestPayjoin4(PayjoinTestBase, unittest.TestCase):
     def reset_fee(self, res):
         jm_single().config.set("POLICY", "txfees", self.old_txfees)
+
     def test_low_feerate(self):
         self.old_txfees = jm_single().config.get("POLICY", "tx_fees")
         # To set such that randomization cannot pull it below minfeerate
@@ -160,25 +184,27 @@ class TrialTestPayjoin4(PayjoinTestBase, unittest.TestCase):
         # as noted in https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/babad1963992965e933924b6c306ad9da89989e0/jmclient/jmclient/payjoin.py#L802-L809
         # , we increase from that by 2%.
         jm_single().config.set("POLICY", "tx_fees", "1404")
-        d = self.do_test_payment(SegwitWallet, SegwitWallet)
+        coro = self.do_test_payment(SegwitWallet, SegwitWallet)
+        d = defer.Deferred.fromCoroutine(coro)
         d.addCallback(self.reset_fee)
         return d
 
-def bip78_receiver_response(response, manager):
-    d = readBody(response)
+async def bip78_receiver_response(response, manager):
+    body = await readBody(response)
     # if the response code is not 200 OK, we must assume payjoin
     # attempt has failed, and revert to standard payment.
     if int(response.code) != 200:
-        d.addCallback(process_receiver_errormsg, response.code)
+        await process_receiver_errormsg(body, response.code)
         return
-    d.addCallback(process_receiver_psbt, manager)
+    await process_receiver_psbt(body, manager)
 
 def process_receiver_errormsg(r, c):
     print("Failed: r, c: ", r, c)
     assert False
 
-def process_receiver_psbt(response, manager):
-    process_payjoin_proposal_from_server(response.decode("utf-8"), manager)
+async def process_receiver_psbt(response, manager):
+    await process_payjoin_proposal_from_server(
+        response.decode("utf-8"), manager)
 
 def getbals(wallet_service, mixdepth):
     """ Retrieves balances for a mixdepth and the 'next'
@@ -186,7 +212,8 @@ def getbals(wallet_service, mixdepth):
     bbm = wallet_service.get_balance_by_mixdepth()
     return (bbm[mixdepth], bbm[(mixdepth + 1) % (wallet_service.mixdepth + 1)])
 
-def final_checks(wallet_services, amount, txfee, ssb, rsb, source_mixdepth=0):
+async def final_checks(wallet_services, amount, txfee, ssb, rsb,
+                       source_mixdepth=0):
     """We use this to check that the wallet contents are
     as we've expected according to the test case.
     amount is the payment amount going from spender to receiver.
@@ -195,7 +222,7 @@ def final_checks(wallet_services, amount, txfee, ssb, rsb, source_mixdepth=0):
     of two entries, source and destination mixdepth respectively.
     """
     jm_single().bc_interface.tickchain()
-    sync_wallets(wallet_services)
+    await sync_wallets(wallet_services)
     spenderbals = getbals(wallet_services[1], source_mixdepth)
     receiverbals = getbals(wallet_services[0], source_mixdepth)
     # is the payment received?

test/jmclient/test_podle.py

@@ -1,11 +1,14 @@
 #! /usr/bin/env python
 '''Tests of Proof of discrete log equivalence commitments.'''
 import os
-import jmbitcoin as bitcoin
 import struct
 import json
 import pytest
 import copy
+
+from unittest import IsolatedAsyncioTestCase
+
+import jmbitcoin as bitcoin
 from jmbase import get_log, bintohex
 from jmclient import load_test_config, jm_single, generate_podle,\
     generate_podle_error_string, get_commitment_file, PoDLE,\
@@ -17,31 +20,6 @@ pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
 
 log = get_log()
 
-def test_commitments_empty(setup_podle):
-    """Ensure that empty commitments file
-    results in {}
-    """
-    assert get_podle_commitments() == ([], {})
-
-def test_commitment_retries(setup_podle):
-    """Assumes no external commitments available.
-    Generate pretend priv/utxo pairs and check that they can be used
-    taker_utxo_retries times.
-    """
-    allowed = jm_single().config.getint("POLICY", "taker_utxo_retries")
-    #make some pretend commitments
-    dummy_priv_utxo_pairs = [(bitcoin.Hash(os.urandom(10)),
-           bitcoin.b2x(bitcoin.Hash(os.urandom(10)))+":0") for _ in range(10)]
-    #test a single commitment request of all 10
-    for x in dummy_priv_utxo_pairs:
-        p = generate_podle([x], allowed)
-        assert p
-    #At this point slot 0 has been taken by all 10.
-    for i in range(allowed-1):
-        p = generate_podle(dummy_priv_utxo_pairs[:1], allowed)
-        assert p
-    p = generate_podle(dummy_priv_utxo_pairs[:1], allowed)
-    assert p is None
 
 def generate_single_podle_sig(priv, i):
     """Make a podle entry for key priv at index i, using a dummy utxo value.
@@ -54,184 +32,213 @@ def generate_single_podle_sig(priv, i):
     return (r['P'], r['P2'], r['sig'],
             r['e'], r['commit'])
 
-def test_rand_commitments(setup_podle):
-    for i in range(20):
-        priv = os.urandom(32)+b"\x01"
-        Pser, P2ser, s, e, commitment = generate_single_podle_sig(priv, 1 + i%5)
-        assert verify_podle(Pser, P2ser, s, e, commitment)
-        #tweak commitments to verify failure
-        tweaked = [x[::-1] for x in [Pser, P2ser, s, e, commitment]]
-        for i in range(5):
-            #Check failure on garbling of each parameter
-            y = [Pser, P2ser, s, e, commitment]
-            y[i] = tweaked[i]
-            fail = False
-            try: 
-                fail = verify_podle(*y)
-            except:
-                pass
-            finally:
-                assert not fail
-
-def test_nums_verify(setup_podle):
-    """Check that the NUMS precomputed values are
-    valid according to the code; assertion check
-    implicit.
-    """
-    verify_all_NUMS(True)
 
-def test_external_commitments(setup_podle):
-    """Add this generated commitment to the external list
-    {txid:N:{'P':pubkey, 'reveal':{1:{'P2':P2,'s':s,'e':e}, 2:{..},..}}}
-    Note we do this *after* the sendpayment test so that the external
-    commitments will not erroneously used (they are fake).
-    """
-    #ensure the file exists even if empty
-    update_commitments()
-    ecs = {}
-    tries = jm_single().config.getint("POLICY","taker_utxo_retries")
-    for i in range(10):
-        priv = os.urandom(32)
-        dummy_utxo = (bitcoin.Hash(priv), 2)
-        ecs[dummy_utxo] = {}
-        ecs[dummy_utxo]['reveal']={}
-        for j in range(tries):
-            P, P2, s, e, commit = generate_single_podle_sig(priv, j)
-            if 'P' not in ecs[dummy_utxo]:
-                ecs[dummy_utxo]['P']=P
-            ecs[dummy_utxo]['reveal'][j] = {'P2':P2, 's':s, 'e':e}
-    add_external_commitments(ecs)
-    used, external = get_podle_commitments()
-    for  u in external:
-        assert external[u]['P'] == ecs[u]['P']
-        for i in range(tries):
-            for x in ['P2', 's', 'e']:
-                assert external[u]['reveal'][i][x] == ecs[u]['reveal'][i][x]
-    
-    #add a dummy used commitment, then try again
-    update_commitments(commitment=b"\xab"*32)
-    ecs = {}
-    known_commits = []
-    known_utxos = []
-    tries = 3
-    for i in range(1, 6):
-        u = (struct.pack(b'B', i)*32, i+3)
-        known_utxos.append(u)
-        priv = struct.pack(b'B', i)*32+b"\x01"
-        ecs[u] = {}
-        ecs[u]['reveal']={}
-        for j in range(tries):
-            P, P2, s, e, commit = generate_single_podle_sig(priv, j)
-            known_commits.append(commit)
-            if 'P' not in ecs[u]:
-                ecs[u]['P'] = P
-            ecs[u]['reveal'][j] = {'P2':P2, 's':s, 'e':e}
-    add_external_commitments(ecs)
-    #simulate most of those external being already used
-    for c in known_commits[:-1]:
-        update_commitments(commitment=c)
-    #this should find the remaining one utxo and return from it
-    assert generate_podle([], max_tries=tries, allow_external=known_utxos)
-    #test commitment removal
-    tru = (struct.pack(b"B", 3)*32, 3+3)
-    to_remove = {tru: ecs[tru]}
-    update_commitments(external_to_remove=to_remove)
-    #test that an incorrectly formatted file raises
-    with open(get_commitment_file(), "rb") as f:
-        validjson = json.loads(f.read().decode('utf-8'))
-    corruptjson = copy.deepcopy(validjson)
-    del corruptjson['used']
-    with open(get_commitment_file(), "wb") as f:
-        f.write(json.dumps(corruptjson, indent=4).encode('utf-8'))
-    with pytest.raises(PoDLEError) as e_info:
-        get_podle_commitments()
-    #clean up
-    with open(get_commitment_file(), "wb") as f:
-        f.write(json.dumps(validjson, indent=4).encode('utf-8'))
-
-
-
-def test_podle_constructor(setup_podle):
-    """Tests rules about construction of PoDLE object
-    are conformed to.
-    """
-    priv  = b"\xaa"*32
-    #pub and priv together not allowed
-    with pytest.raises(PoDLEError) as e_info:
-        p = PoDLE(priv=priv, P="dummypub")
-    #no pub or priv is allowed, i forget if this is useful for something
-    p = PoDLE()
-    #create from priv
-    p = PoDLE(priv=priv+b"\x01", u=(struct.pack(b"B", 7)*32, 4))
-    pdict = p.generate_podle(2)
-    assert all([k in pdict for k in ['used', 'utxo', 'P', 'P2', 'commit', 'sig', 'e']])
-    #using the valid data, serialize/deserialize test
-    deser = p.deserialize_revelation(p.serialize_revelation())
-    assert all([deser[x] == pdict[x] for x in ['utxo', 'P', 'P2', 'sig', 'e']])
-    #deserialization must fail for wrong number of items
-    with pytest.raises(PoDLEError) as e_info:
-        p.deserialize_revelation(':'.join([str(x) for x in range(4)]), separator=':')
-    #reveal() must work without pre-generated commitment
-    p.commitment = None
-    pdict2 = p.reveal()
-    assert pdict2 == pdict
-    #corrupt P2, cannot commit:
-    p.P2 = "blah"
-    with pytest.raises(PoDLEError) as e_info:
-        p.get_commitment()
-    #generation fails without a utxo
-    p = PoDLE(priv=priv)
-    with pytest.raises(PoDLEError) as e_info:
-        p.generate_podle(0)
-    #Test construction from pubkey
-    pub = bitcoin.privkey_to_pubkey(priv+b"\x01")
-    p = PoDLE(P=pub)
-    with pytest.raises(PoDLEError) as e_info:
-        p.get_commitment()
-    with pytest.raises(PoDLEError) as e_info:
-        p.verify("dummycommitment", range(3))
-
-def test_podle_error_string(setup_podle):
-    example_utxos = [(b"\x00"*32, i) for i in range(6)]
-    priv_utxo_pairs = [('fakepriv1', example_utxos[0]),
-                             ('fakepriv2', example_utxos[1])]
-    to = example_utxos[2:4]
-    ts = example_utxos[4:6]
-    wallet_service = make_wallets(1, [[1, 0, 0, 0, 0]])[0]['wallet']
-    cjamt = 100
-    tua = "3"
-    tuamtper = "20"
-    errmgsheader, errmsg = generate_podle_error_string(priv_utxo_pairs,
-                                                       to,
-                                                       ts,
-                                                       wallet_service,
-                                                       cjamt,
-                                                       tua,
-                                                       tuamtper)
-    assert errmgsheader == ("Failed to source a commitment; this debugging information"
-                            " may help:\n\n")
-    y = [bintohex(x[0]) for x in example_utxos]
-    assert all([errmsg.find(x) != -1 for x in y])
-    #ensure OK with nothing
-    errmgsheader, errmsg = generate_podle_error_string([], [], [], wallet_service,
-                                                       cjamt, tua, tuamtper)
-
-@pytest.fixture(scope="module")
-def setup_podle(request):
-    load_test_config()
-    if not os.path.exists("cmtdata"):
-        os.mkdir("cmtdata")
-    prev_commits = False
-    #back up any existing commitments
-    pcf = get_commitment_file()
-    log.debug("Podle file: " + pcf)
-    if os.path.exists(pcf):
-        os.rename(pcf, pcf + ".bak")
-        prev_commits = True
-    def teardown():
-        if prev_commits:
+class AsyncioTestCase(IsolatedAsyncioTestCase):
+
+    async def asyncSetUp(self):
+        load_test_config()
+        if not os.path.exists("cmtdata"):
+            os.mkdir("cmtdata")
+        self.prev_commits = False
+        #back up any existing commitments
+        pcf = get_commitment_file()
+        log.debug("Podle file: " + pcf)
+        if os.path.exists(pcf):
+            os.rename(pcf, pcf + ".bak")
+            self.prev_commits = True
+
+    async def asyncTearDown(self):
+        pcf = get_commitment_file()
+        if self.prev_commits:
             os.rename(pcf + ".bak", pcf)
         else:
             if os.path.exists(pcf):
                 os.remove(pcf)
-    request.addfinalizer(teardown)
+
+    async def test_commitments_empty(self):
+        """Ensure that empty commitments file
+        results in {}
+        """
+        assert get_podle_commitments() == ([], {})
+
+    async def test_commitment_retries(self):
+        """Assumes no external commitments available.
+        Generate pretend priv/utxo pairs and check that they can be used
+        taker_utxo_retries times.
+        """
+        allowed = jm_single().config.getint("POLICY", "taker_utxo_retries")
+        #make some pretend commitments
+        dummy_priv_utxo_pairs = [(bitcoin.Hash(os.urandom(10)),
+               bitcoin.b2x(bitcoin.Hash(os.urandom(10)))+":0") for _ in range(10)]
+        #test a single commitment request of all 10
+        for x in dummy_priv_utxo_pairs:
+            p = generate_podle([x], allowed)
+            assert p
+        #At this point slot 0 has been taken by all 10.
+        for i in range(allowed-1):
+            p = generate_podle(dummy_priv_utxo_pairs[:1], allowed)
+            assert p
+        p = generate_podle(dummy_priv_utxo_pairs[:1], allowed)
+        assert p is None
+
+    async def test_rand_commitments(self):
+        for i in range(20):
+            priv = os.urandom(32)+b"\x01"
+            Pser, P2ser, s, e, commitment = generate_single_podle_sig(priv, 1 + i%5)
+            assert verify_podle(Pser, P2ser, s, e, commitment)
+            #tweak commitments to verify failure
+            tweaked = [x[::-1] for x in [Pser, P2ser, s, e, commitment]]
+            for i in range(5):
+                #Check failure on garbling of each parameter
+                y = [Pser, P2ser, s, e, commitment]
+                y[i] = tweaked[i]
+                fail = False
+                try: 
+                    fail = verify_podle(*y)
+                except:
+                    pass
+                finally:
+                    assert not fail
+
+    async def test_nums_verify(self):
+        """Check that the NUMS precomputed values are
+        valid according to the code; assertion check
+        implicit.
+        """
+        verify_all_NUMS(True)
+
+    async def test_external_commitments(self):
+        """Add this generated commitment to the external list
+        {txid:N:{'P':pubkey, 'reveal':{1:{'P2':P2,'s':s,'e':e}, 2:{..},..}}}
+        Note we do this *after* the sendpayment test so that the external
+        commitments will not erroneously used (they are fake).
+        """
+        #ensure the file exists even if empty
+        update_commitments()
+        ecs = {}
+        tries = jm_single().config.getint("POLICY","taker_utxo_retries")
+        for i in range(10):
+            priv = os.urandom(32)
+            dummy_utxo = (bitcoin.Hash(priv), 2)
+            ecs[dummy_utxo] = {}
+            ecs[dummy_utxo]['reveal']={}
+            for j in range(tries):
+                P, P2, s, e, commit = generate_single_podle_sig(priv, j)
+                if 'P' not in ecs[dummy_utxo]:
+                    ecs[dummy_utxo]['P']=P
+                ecs[dummy_utxo]['reveal'][j] = {'P2':P2, 's':s, 'e':e}
+        add_external_commitments(ecs)
+        used, external = get_podle_commitments()
+        for  u in external:
+            assert external[u]['P'] == ecs[u]['P']
+            for i in range(tries):
+                for x in ['P2', 's', 'e']:
+                    assert external[u]['reveal'][i][x] == ecs[u]['reveal'][i][x]
+        
+        #add a dummy used commitment, then try again
+        update_commitments(commitment=b"\xab"*32)
+        ecs = {}
+        known_commits = []
+        known_utxos = []
+        tries = 3
+        for i in range(1, 6):
+            u = (struct.pack(b'B', i)*32, i+3)
+            known_utxos.append(u)
+            priv = struct.pack(b'B', i)*32+b"\x01"
+            ecs[u] = {}
+            ecs[u]['reveal']={}
+            for j in range(tries):
+                P, P2, s, e, commit = generate_single_podle_sig(priv, j)
+                known_commits.append(commit)
+                if 'P' not in ecs[u]:
+                    ecs[u]['P'] = P
+                ecs[u]['reveal'][j] = {'P2':P2, 's':s, 'e':e}
+        add_external_commitments(ecs)
+        #simulate most of those external being already used
+        for c in known_commits[:-1]:
+            update_commitments(commitment=c)
+        #this should find the remaining one utxo and return from it
+        assert generate_podle([], max_tries=tries, allow_external=known_utxos)
+        #test commitment removal
+        tru = (struct.pack(b"B", 3)*32, 3+3)
+        to_remove = {tru: ecs[tru]}
+        update_commitments(external_to_remove=to_remove)
+        #test that an incorrectly formatted file raises
+        with open(get_commitment_file(), "rb") as f:
+            validjson = json.loads(f.read().decode('utf-8'))
+        corruptjson = copy.deepcopy(validjson)
+        del corruptjson['used']
+        with open(get_commitment_file(), "wb") as f:
+            f.write(json.dumps(corruptjson, indent=4).encode('utf-8'))
+        with pytest.raises(PoDLEError) as e_info:
+            get_podle_commitments()
+        #clean up
+        with open(get_commitment_file(), "wb") as f:
+            f.write(json.dumps(validjson, indent=4).encode('utf-8'))
+
+    async def test_podle_constructor(self):
+        """Tests rules about construction of PoDLE object
+        are conformed to.
+        """
+        priv  = b"\xaa"*32
+        #pub and priv together not allowed
+        with pytest.raises(PoDLEError) as e_info:
+            p = PoDLE(priv=priv, P="dummypub")
+        #no pub or priv is allowed, i forget if this is useful for something
+        p = PoDLE()
+        #create from priv
+        p = PoDLE(priv=priv+b"\x01", u=(struct.pack(b"B", 7)*32, 4))
+        pdict = p.generate_podle(2)
+        assert all([k in pdict for k in ['used', 'utxo', 'P', 'P2', 'commit', 'sig', 'e']])
+        #using the valid data, serialize/deserialize test
+        deser = p.deserialize_revelation(p.serialize_revelation())
+        assert all([deser[x] == pdict[x] for x in ['utxo', 'P', 'P2', 'sig', 'e']])
+        #deserialization must fail for wrong number of items
+        with pytest.raises(PoDLEError) as e_info:
+            p.deserialize_revelation(':'.join([str(x) for x in range(4)]), separator=':')
+        #reveal() must work without pre-generated commitment
+        p.commitment = None
+        pdict2 = p.reveal()
+        assert pdict2 == pdict
+        #corrupt P2, cannot commit:
+        p.P2 = "blah"
+        with pytest.raises(PoDLEError) as e_info:
+            p.get_commitment()
+        #generation fails without a utxo
+        p = PoDLE(priv=priv)
+        with pytest.raises(PoDLEError) as e_info:
+            p.generate_podle(0)
+        #Test construction from pubkey
+        pub = bitcoin.privkey_to_pubkey(priv+b"\x01")
+        p = PoDLE(P=pub)
+        with pytest.raises(PoDLEError) as e_info:
+            p.get_commitment()
+        with pytest.raises(PoDLEError) as e_info:
+            p.verify("dummycommitment", range(3))
+
+    async def test_podle_error_string(self):
+        example_utxos = [(b"\x00"*32, i) for i in range(6)]
+        priv_utxo_pairs = [('fakepriv1', example_utxos[0]),
+                                 ('fakepriv2', example_utxos[1])]
+        to = example_utxos[2:4]
+        ts = example_utxos[4:6]
+        wallets = await make_wallets(1, [[1, 0, 0, 0, 0]])
+        wallet_service = wallets[0]['wallet']
+        cjamt = 100
+        tua = "3"
+        tuamtper = "20"
+        errmgsheader, errmsg = await generate_podle_error_string(
+            priv_utxo_pairs,
+            to,
+            ts,
+            wallet_service,
+            cjamt,
+            tua,
+            tuamtper)
+        assert errmgsheader == ("Failed to source a commitment; this debugging information"
+                                " may help:\n\n")
+        y = [bintohex(x[0]) for x in example_utxos]
+        assert all([errmsg.find(x) != -1 for x in y])
+        #ensure OK with nothing
+        errmgsheader, errmsg = await generate_podle_error_string(
+            [], [], [], wallet_service, cjamt, tua, tuamtper)

test/jmclient/test_psbt_wallet.py

@@ -8,6 +8,13 @@
 
 import copy
 import base64
+from unittest import IsolatedAsyncioTestCase
+
+from unittest_parametrize import parametrize, ParametrizedTestCase
+
+import jmclient  # install asyncioreactor
+from twisted.internet import reactor
+
 from commontest import make_wallets, dummy_accept_callback, dummy_info_callback
 
 import jmbitcoin as bitcoin
@@ -22,384 +29,15 @@ pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
 
 log = get_log()
 
-def create_volatile_wallet(seedphrase, wallet_cls=SegwitWallet):
+async def create_volatile_wallet(seedphrase, wallet_cls=SegwitWallet):
     storage = VolatileStorage()
     wallet_cls.initialize(storage, get_network(), max_mixdepth=4,
                           entropy=wallet_cls.entropy_from_mnemonic(seedphrase))
     storage.save()
-    return wallet_cls(storage)
-
-@pytest.mark.parametrize('walletseed, xpub, spktype_wallet, spktype_destn, partial, psbt', [
-    ("prosper diamond marriage spy across start shift elevator job lunar edge gallery",
-     "tpubDChjiEhsafnW2LcmK1C77XiEAgZddi6xZyxjMujBzUqZPTMRwsv3e5vSBYsdiPtCyc6TtoHTCjkxBjtF22tf8Z5ABRdeBUNwHCsqEyzR5wT",
-     "p2wpkh", "p2sh-p2wpkh", False,
-     "cHNidP8BAMQCAAAAA7uEliZeXLPfjeUiRBw6e5oZV1DtBrDmLthfDC4oaHQLAAAAAAD/////+X1Exketc4o5b9BPxsj70O+VlGvgiZz0KP1OMRtVLUQAAAAAAP////9r5ylMhQyxbJvCbU8aNE3NOPoXJwUaUZm4H3iT4RnaSwAAAAAA/////wKMz/AIAAAAABepFJwmRAefvZS7VQStD4k52Rn0k71Gh4zP8AgAAAAAFgAUA2shnTVftDXq+ssPwzml2UKdu1QAAAAAAAEBHwDh9QUAAAAAFgAUqw1Ifto4LztwcsxV6q+sQThIdloiBgMDZ5u3RN6Xum+OLkgAzwLFXGWFLwBUraMi7Oin4fYfrwzvYoLxAAAAAAAAAAAAAQEfAOH1BQAAAAAWABQpSCwoeMSghUoVflvtTPiqBPi+5yIGA/tAH4kVpqd3wzidaTNFxtwdpHTydkmB825us2w/3cAVDO9igvEAAAAAAQAAAAABAR8A4fUFAAAAABYAFEqM0KJ5FJ7ak2NL8PDqOPI0I1PaIgYD84aDwOqXKfGvEbre+bpNpuT0uZv6syESzz5PMu4RyLkM72KC8QAAAAACAAAAAAEAF6kUnCZEB5+9lLtVBK0PiTnZGfSTvUaHACICAw8k2gGGcF5sR8yKO5JeAkrkH15rmtCq8sCoDYbywTNzEO9igvEMAAAAIgAAAJQCAAAA"),
-    ("prosper diamond marriage spy across start shift elevator job lunar edge gallery",
-     "tpubDChjiEhsafnW2LcmK1C77XiEAgZddi6xZyxjMujBzUqZPTMRwsv3e5vSBYsdiPtCyc6TtoHTCjkxBjtF22tf8Z5ABRdeBUNwHCsqEyzR5wT",
-     "p2wpkh", "p2wpkh", False,
-     "cHNidP8BAMMCAAAAA7uEliZeXLPfjeUiRBw6e5oZV1DtBrDmLthfDC4oaHQLAAAAAAD/////+X1Exketc4o5b9BPxsj70O+VlGvgiZz0KP1OMRtVLUQAAAAAAP////9r5ylMhQyxbJvCbU8aNE3NOPoXJwUaUZm4H3iT4RnaSwAAAAAA/////wKMz/AIAAAAABYAFBaOTObQIdtCaryiPxaDV5rsGYWUjM/wCAAAAAAWABR9TJm5rcSoIMW7bE1bnj7REL/eygAAAAAAAQEfAOH1BQAAAAAWABSrDUh+2jgvO3ByzFXqr6xBOEh2WiIGAwNnm7dE3pe6b44uSADPAsVcZYUvAFStoyLs6Kfh9h+vDO9igvEAAAAAAAAAAAABAR8A4fUFAAAAABYAFClILCh4xKCFShV+W+1M+KoE+L7nIgYD+0AfiRWmp3fDOJ1pM0XG3B2kdPJ2SYHzbm6zbD/dwBUM72KC8QAAAAABAAAAAAEBHwDh9QUAAAAAFgAUSozQonkUntqTY0vw8Oo48jQjU9oiBgPzhoPA6pcp8a8Rut75uk2m5PS5m/qzIRLPPk8y7hHIuQzvYoLxAAAAAAIAAAAAACICAuqCicVUfcM5IiVSiB/0ZemodybG5Im9Fu8MLorQSE4UEO9igvEMAAAAIgAAAOkAAAAA"),
-    ("prosper diamond marriage spy across start shift elevator job lunar edge gallery",
-     "tpubDChjiEhsafnW2LcmK1C77XiEAgZddi6xZyxjMujBzUqZPTMRwsv3e5vSBYsdiPtCyc6TtoHTCjkxBjtF22tf8Z5ABRdeBUNwHCsqEyzR5wT",
-     "p2wpkh", "p2wpkh", True,
-     "cHNidP8BAMMCAAAAA7uEliZeXLPfjeUiRBw6e5oZV1DtBrDmLthfDC4oaHQLAAAAAAD/////+X1Exketc4o5b9BPxsj70O+VlGvgiZz0KP1OMRtVLUQAAAAAAP////9r5ylMhQyxbJvCbU8aNE3NOPoXJwUaUZm4H3iT4RnaSwAAAAAA/////wKMz/AIAAAAABYAFLH/IL11rTJ3wX1NcmUIsJ/T4j4jjM/wCAAAAAAWABR8GPNb1HUpCz8PKOc8aQXLD1wjcAAAAAAAAQEfAOH1BQAAAAAWABSrDUh+2jgvO3ByzFXqr6xBOEh2WiIGAwNnm7dE3pe6b44uSADPAsVcZYUvAFStoyLs6Kfh9h+vDE5vcGUAAAAAAAAAAAABAR8A4fUFAAAAABYAFClILCh4xKCFShV+W+1M+KoE+L7nIgYD+0AfiRWmp3fDOJ1pM0XG3B2kdPJ2SYHzbm6zbD/dwBUM72KC8QAAAAABAAAAAAEBHwDh9QUAAAAAFgAUSozQonkUntqTY0vw8Oo48jQjU9oiBgPzhoPA6pcp8a8Rut75uk2m5PS5m/qzIRLPPk8y7hHIuQzvYoLxAAAAAAIAAAAAACICAsQ7ZvU9tsbBoSje5rIJQBStlUkQaRCssKylEixre3AYEO9igvEMAAAAIgAAABcCAAAA"),
-])
-def test_sign_external_psbt(setup_psbt_wallet, walletseed, xpub,
-                            spktype_wallet, spktype_destn, partial, psbt):
-    bitcoin.select_chain_params("bitcoin")
-    wallet_cls = SegwitWallet if spktype_wallet == "p2wpkh" else SegwitLegacyWallet
-    wallet = create_volatile_wallet(walletseed, wallet_cls=wallet_cls)
-    # if we want to actually sign, our wallet has to recognize the fake utxos
-    # as being in the wallet, so we inject them:
-    class DummyUtxoManager(object):
-        _utxo = {0:{}}
-        def add_utxo(self, utxo, path, value, height):
-            self._utxo[0][utxo] = (path, value, height)
-    wallet._index_cache[0][0] = 1000
-    wallet._utxos = DummyUtxoManager()
-    p0, p1, p2 = (wallet.get_path(0, 0, i) for i in range(3))
-    if not partial:
-        wallet._utxos.add_utxo(utxostr_to_utxo(
-            "0b7468282e0c5fd82ee6b006ed5057199a7b3a1c4422e58ddfb35c5e269684bb:0"),
-                               p0, 10000, 1)
-    wallet._utxos.add_utxo(utxostr_to_utxo(
-        "442d551b314efd28f49c89e06b9495efd0fbc8c64fd06f398a73ad47c6447df9:0"),
-                           p1, 10000, 1)
-    wallet._utxos.add_utxo(utxostr_to_utxo(
-        "4bda19e193781fb899511a052717fa38cd4d341a4f6dc29b6cb10c854c29e76b:0"),
-                           p2, 10000, 1)
-    signresult_and_signedpsbt, err = wallet.sign_psbt(base64.b64decode(
-        psbt.encode("ascii")),with_sign_result=True)
-    assert not err
-    signresult, signedpsbt = signresult_and_signedpsbt
-    if partial:
-        assert not signresult.is_final
-        assert signresult.num_inputs_signed == 2
-        assert signresult.num_inputs_final == 2
-    else:
-        assert signresult.is_final
-        assert signresult.num_inputs_signed == 3
-        assert signresult.num_inputs_final == 3
-    print(PSBTWalletMixin.human_readable_psbt(signedpsbt))
-    bitcoin.select_chain_params("bitcoin/regtest")
-
-def test_create_and_sign_psbt_with_legacy(setup_psbt_wallet):
-    """ The purpose of this test is to check that we can create and
-    then partially sign a PSBT where we own one input and the other input
-    is of legacy p2pkh type.
-    """
-    wallet_service = make_wallets(1, [[1,0,0,0,0]], 1)[0]['wallet']
-    wallet_service.sync_wallet(fast=True)
-    utxos = wallet_service.select_utxos(0, bitcoin.coins_to_satoshi(0.5))
-    assert len(utxos) == 1
-    # create a legacy address and make a payment into it
-    legacy_addr = bitcoin.CCoinAddress.from_scriptPubKey(
-        bitcoin.pubkey_to_p2pkh_script(
-            bitcoin.privkey_to_pubkey(b"\x01"*33)))
-    tx = direct_send(wallet_service, 0,
-                     [(str(legacy_addr), bitcoin.coins_to_satoshi(0.3))],
-                     accept_callback=dummy_accept_callback,
-                     info_callback=dummy_info_callback,
-                     return_transaction=True)
-    assert tx
-    # this time we will have one utxo worth <~ 0.7
-    my_utxos = wallet_service.select_utxos(0, bitcoin.coins_to_satoshi(0.5))
-    assert len(my_utxos) == 1
-    # find the outpoint for the legacy address we're spending
-    n = -1
-    for i, t in enumerate(tx.vout):
-        if bitcoin.CCoinAddress.from_scriptPubKey(t.scriptPubKey) == legacy_addr:
-            n = i
-    assert n > -1
-    utxos = copy.deepcopy(my_utxos)
-    utxos[(tx.GetTxid()[::-1], n)] ={"script": legacy_addr.to_scriptPubKey(),
-              "value": bitcoin.coins_to_satoshi(0.3)}
-    outs = [{"value": bitcoin.coins_to_satoshi(0.998),
-                 "address": wallet_service.get_addr(0,0,0)}]
-    tx2 = bitcoin.mktx(list(utxos.keys()), outs)
-    spent_outs = wallet_service.witness_utxos_to_psbt_utxos(my_utxos)
-    spent_outs.append(tx)
-    new_psbt = wallet_service.create_psbt_from_tx(tx2, spent_outs,
-                                                  force_witness_utxo=False)
-    signed_psbt_and_signresult, err = wallet_service.sign_psbt(
-        new_psbt.serialize(), with_sign_result=True)
-    assert err is None
-    signresult, signed_psbt = signed_psbt_and_signresult
-    assert signresult.num_inputs_signed == 1
-    assert signresult.num_inputs_final == 1
-    assert not signresult.is_final
-
-@pytest.mark.parametrize('unowned_utxo, wallet_cls', [
-    (True, SegwitLegacyWallet),
-    (False, SegwitLegacyWallet),
-    (True, SegwitWallet),
-    (False, SegwitWallet),
-    (True, LegacyWallet),
-    (False, LegacyWallet),
-])
-def test_create_psbt_and_sign(setup_psbt_wallet, unowned_utxo, wallet_cls):
-    """ Plan of test:
-    1. Create a wallet and source 3 destination addresses.
-    2. Make, and confirm, transactions that fund the 3 addrs.
-    3. Create a new tx spending 2 of those 3 utxos and spending
-       another utxo we don't own (extra is optional per `unowned_utxo`).
-    4. Create a psbt using the above transaction and corresponding
-       `spent_outs` field to fill in the redeem script.
-    5. Compare resulting PSBT with expected structure.
-    6. Use the wallet's sign_psbt method to sign the whole psbt, which
-       means signing each input we own.
-    7. Check that each input is finalized as per expected. Check that the whole
-       PSBT is or is not finalized as per whether there is an unowned utxo.
-    8. In case where whole psbt is finalized, attempt to broadcast the tx.
-    """
-    # steps 1 and 2:
-    wallet_service = make_wallets(1, [[3,0,0,0,0]], 1,
-                                  wallet_cls=wallet_cls)[0]['wallet']
-    wallet_service.sync_wallet(fast=True)
-    utxos = wallet_service.select_utxos(0, bitcoin.coins_to_satoshi(1.5))
-    # for legacy wallets, psbt creation requires querying for the spending
-    # transaction:
-    if wallet_cls == LegacyWallet:
-        fulltxs = []
-        for utxo, v in utxos.items():
-            fulltxs.append(jm_single().bc_interface.get_deser_from_gettransaction(
-                jm_single().bc_interface.get_transaction(utxo[0])))
-
-    assert len(utxos) == 2
-    u_utxos = {}
-    if unowned_utxo:
-        # note: tx creation uses the key only; psbt creation uses the value,
-        # which can be fake here; we do not intend to attempt to fully
-        # finalize a psbt with an unowned input. See
-        # https://github.com/Simplexum/python-bitcointx/issues/30
-        # the redeem script creation (which is artificial) will be
-        # avoided in future.
-        priv = b"\xaa"*32 + b"\x01"
-        pub = bitcoin.privkey_to_pubkey(priv)
-        script = bitcoin.pubkey_to_p2sh_p2wpkh_script(pub)
-        redeem_script = bitcoin.pubkey_to_p2wpkh_script(pub)
-        u_utxos[(b"\xaa"*32, 12)] = {"value": 1000, "script": script}
-    utxos.update(u_utxos)
-    # outputs aren't interesting for this test (we selected 1.5 but will get 2):
-    outs = [{"value": bitcoin.coins_to_satoshi(1.999),
-             "address": wallet_service.get_addr(0,0,0)}]
-    tx = bitcoin.mktx(list(utxos.keys()), outs)
-
-    if wallet_cls != LegacyWallet:
-        spent_outs = wallet_service.witness_utxos_to_psbt_utxos(utxos)
-        force_witness_utxo=True
-    else:
-        spent_outs = fulltxs
-        # the extra input is segwit:
-        if unowned_utxo:
-            spent_outs.extend(
-                wallet_service.witness_utxos_to_psbt_utxos(u_utxos))
-        force_witness_utxo=False
-    newpsbt = wallet_service.create_psbt_from_tx(tx, spent_outs,
-                            force_witness_utxo=force_witness_utxo)
-    # see note above
-    if unowned_utxo:
-        newpsbt.inputs[-1].redeem_script = redeem_script
-    print(bintohex(newpsbt.serialize()))
-    print("human readable: ")
-    print(wallet_service.human_readable_psbt(newpsbt))
-    # we cannot compare with a fixed expected result due to wallet randomization, but we can
-    # check psbt structure:
-    expected_inputs_length = 3 if unowned_utxo else 2
-    assert len(newpsbt.inputs) == expected_inputs_length
-    assert len(newpsbt.outputs) == 1
-    # note: redeem_script field is a CScript which is a bytes instance,
-    # so checking length is best way to check for existence (comparison
-    # with None does not work):
-    if wallet_cls == SegwitLegacyWallet:
-        assert len(newpsbt.inputs[0].redeem_script) != 0
-        assert len(newpsbt.inputs[1].redeem_script) != 0
-    if unowned_utxo:
-        assert newpsbt.inputs[2].redeem_script == redeem_script
-
-    signed_psbt_and_signresult, err = wallet_service.sign_psbt(
-        newpsbt.serialize(), with_sign_result=True)
-    assert err is None
-    signresult, signed_psbt = signed_psbt_and_signresult
-    expected_signed_inputs = len(utxos) if not unowned_utxo else len(utxos)-1
-    assert signresult.num_inputs_signed == expected_signed_inputs
-    assert signresult.num_inputs_final == expected_signed_inputs
-
-    if not unowned_utxo:
-        assert signresult.is_final
-        # only in case all signed do we try to broadcast:
-        extracted_tx = signed_psbt.extract_transaction().serialize()
-        assert jm_single().bc_interface.pushtx(extracted_tx)
-    else:
-        # transaction extraction must fail for not-fully-signed psbts:
-        with pytest.raises(ValueError) as e:
-            extracted_tx = signed_psbt.extract_transaction()
-
-@pytest.mark.parametrize('payment_amt, wallet_cls_sender, wallet_cls_receiver', [
-    (0.05, SegwitLegacyWallet, SegwitLegacyWallet),
-    #(0.95, SegwitLegacyWallet, SegwitWallet),
-    #(0.05, SegwitWallet, SegwitLegacyWallet),
-    #(0.95, SegwitWallet, SegwitWallet),
-])
-def test_payjoin_workflow(setup_psbt_wallet, payment_amt, wallet_cls_sender,
-                          wallet_cls_receiver):
-    """ Workflow step 1:
-    Create a payment from a wallet, and create a finalized PSBT.
-    This step is fairly trivial as the functionality is built-in to
-    PSBTWalletMixin.
-    Note that only Segwit* wallets are supported for PayJoin.
-
-        Workflow step 2:
-    Receiver creates a new partially signed PSBT with the same amount
-    and at least one more utxo.
-
-        Workflow step 3:
-    Given a partially signed PSBT created by a receiver, here the sender
-    completes (co-signs) the PSBT they are given. Note this code is a PSBT
-    functionality check, and does NOT include the detailed checks that
-    the sender should perform before agreeing to sign (see:
-    https://github.com/btcpayserver/btcpayserver-doc/blob/eaac676866a4d871eda5fd7752b91b88fdf849ff/Payjoin-spec.md#receiver-side
-    ).
-    """
-
-    wallet_r = make_wallets(1, [[3,0,0,0,0]], 1,
-                    wallet_cls=wallet_cls_receiver)[0]["wallet"]
-    wallet_s = make_wallets(1, [[3,0,0,0,0]], 1,
-                        wallet_cls=wallet_cls_sender)[0]["wallet"]
-    for w in [wallet_r, wallet_s]:
-        w.sync_wallet(fast=True)
-
-    # destination address for payment:
-    destaddr = str(bitcoin.CCoinAddress.from_scriptPubKey(
-        bitcoin.pubkey_to_p2wpkh_script(bitcoin.privkey_to_pubkey(b"\x01"*33))))
-
-    payment_amt = bitcoin.coins_to_satoshi(payment_amt)
-
-    # *** STEP 1 ***
-    # **************
-
-    # create a normal tx from the sender wallet:
-    payment_psbt = direct_send(wallet_s, 0,
-                    [(destaddr, payment_amt)],
-                    accept_callback=dummy_accept_callback,
-                    info_callback=dummy_info_callback,
-                    with_final_psbt=True)
-
-    print("Initial payment PSBT created:\n{}".format(
-        wallet_s.human_readable_psbt(payment_psbt)))
-    # ensure that the payemnt amount is what was intended:
-    out_amts = [x.nValue for x in payment_psbt.unsigned_tx.vout]
-    # NOTE this would have to change for more than 2 outputs:
-    assert any([out_amts[i] == payment_amt for i in [0, 1]])
-
-    # ensure that we can actually broadcast the created tx:
-    # (note that 'extract_transaction' represents an implicit
-    # PSBT finality check).
-    extracted_tx = payment_psbt.extract_transaction().serialize()
-    # don't want to push the tx right now, because of test structure
-    # (in production code this isn't really needed, we will not
-    # produce invalid payment transactions).
-    assert jm_single().bc_interface.testmempoolaccept(bintohex(extracted_tx)),\
-        "Payment transaction was rejected from mempool."
-
-    # *** STEP 2 ***
-    # **************
-
-    # Simple receiver utxo choice heuristic.
-    # For more generality we test with two receiver-utxos, not one.
-    all_receiver_utxos = wallet_r.get_all_utxos()
-    # TODO is there a less verbose way to get any 2 utxos from the dict?
-    receiver_utxos_keys = list(all_receiver_utxos.keys())[:2]
-    receiver_utxos = {k: v for k, v in all_receiver_utxos.items(
-        ) if k in receiver_utxos_keys}
-
-    # receiver will do other checks as discussed above, including payment
-    # amount; as discussed above, this is out of the scope of this PSBT test.
-
-    # construct unsigned tx for payjoin-psbt:
-    payjoin_tx_inputs = [(x.prevout.hash[::-1],
-                x.prevout.n) for x in payment_psbt.unsigned_tx.vin]
-    payjoin_tx_inputs.extend(receiver_utxos.keys())
-    # find payment output and change output
-    pay_out = None
-    change_out = None
-    for o in payment_psbt.unsigned_tx.vout:
-        jm_out_fmt = {"value": o.nValue,
-        "address": str(bitcoin.CCoinAddress.from_scriptPubKey(
-        o.scriptPubKey))}
-        if o.nValue == payment_amt:
-            assert pay_out is None
-            pay_out = jm_out_fmt
-        else:
-            assert change_out is None
-            change_out = jm_out_fmt
-
-    # we now know there were two outputs and know which is payment.
-    # bump payment output with our input:
-    outs = [pay_out, change_out]
-    our_inputs_val = sum([v["value"] for _, v in receiver_utxos.items()])
-    pay_out["value"] += our_inputs_val
-    print("we bumped the payment output value by: ", our_inputs_val)
-    print("It is now: ", pay_out["value"])
-    unsigned_payjoin_tx = bitcoin.make_shuffled_tx(payjoin_tx_inputs, outs,
-                                version=payment_psbt.unsigned_tx.nVersion,
-                                locktime=payment_psbt.unsigned_tx.nLockTime)
-    print("we created this unsigned tx: ")
-    print(bitcoin.human_readable_transaction(unsigned_payjoin_tx))
-    # to create the PSBT we need the spent_outs for each input,
-    # in the right order:
-    spent_outs = []
-    for i, inp in enumerate(unsigned_payjoin_tx.vin):
-        input_found = False
-        for j, inp2 in enumerate(payment_psbt.unsigned_tx.vin):
-            if inp.prevout == inp2.prevout:
-                spent_outs.append(payment_psbt.inputs[j].utxo)
-                input_found = True
-                break
-        if input_found:
-            continue
-        # if we got here this input is ours, we must find
-        # it from our original utxo choice list:
-        for ru in receiver_utxos.keys():
-            if (inp.prevout.hash[::-1], inp.prevout.n) == ru:
-                spent_outs.append(
-                    wallet_r.witness_utxos_to_psbt_utxos(
-                        {ru: receiver_utxos[ru]})[0])
-                input_found = True
-                break
-        # there should be no other inputs:
-        assert input_found
-
-    r_payjoin_psbt = wallet_r.create_psbt_from_tx(unsigned_payjoin_tx,
-                                                  spent_outs=spent_outs)
-    print("Receiver created payjoin PSBT:\n{}".format(
-        wallet_r.human_readable_psbt(r_payjoin_psbt)))
-
-    signresultandpsbt, err = wallet_r.sign_psbt(r_payjoin_psbt.serialize(),
-                                                with_sign_result=True)
-    assert not err, err
-    signresult, receiver_signed_psbt = signresultandpsbt
-    assert signresult.num_inputs_final == len(receiver_utxos)
-    assert not signresult.is_final
-
-    print("Receiver signing successful. Payjoin PSBT is now:\n{}".format(
-        wallet_r.human_readable_psbt(receiver_signed_psbt)))
-
-    # *** STEP 3 ***
-    # **************
-
-    # take the half-signed PSBT, validate and co-sign:
-
-    signresultandpsbt, err = wallet_s.sign_psbt(
-        receiver_signed_psbt.serialize(), with_sign_result=True)
-    assert not err, err
-    signresult, sender_signed_psbt =  signresultandpsbt
-    print("Sender's final signed PSBT is:\n{}".format(
-        wallet_s.human_readable_psbt(sender_signed_psbt)))
-    assert signresult.is_final
-
-    # broadcast the tx
-    extracted_tx = sender_signed_psbt.extract_transaction().serialize()
-    assert jm_single().bc_interface.pushtx(extracted_tx)
+    wallet = wallet_cls(storage)
+    await wallet.async_init(storage)
+    return wallet
+
 
 """ test vector data for human readable parsing only,
 they are taken from bitcointx/tests/test_psbt.py and in turn
@@ -430,12 +68,403 @@ hr_test_vectors = {
 "proprietary-values": '70736274ff0100550200000001ab0949a08c5af7c49b8212f417e2f15ab3f5c33dcf153821a8139f877a5b7be40100000000feffffff018e240000000000001976a9146f4620b553fa095e721b9ee0efe9fa039cca459788ac0000000015fc0a676c6f62616c5f706678016d756c7469706c790563686965660001012000e1f5050000000017a9143545e6e33b832c47050f24d3eeb93c9c03948bc787010416001485d13537f2e265405a34dbafa9e3dda01fb823080ffc06696e5f706678fde80377686174056672616d650afc00fe40420f0061736b077361746f7368690012fc076f75745f706678feffffff01636f726e05746967657217fc076f75745f706678ffffffffffffffffff707570707905647269766500'
 }
 
-def test_hr_psbt(setup_psbt_wallet):
-    bitcoin.select_chain_params("bitcoin")
-    for k, v in hr_test_vectors.items():
-        print(PSBTWalletMixin.human_readable_psbt(
-            bitcoin.PartiallySignedTransaction.from_binary(hextobin(v))))
-    bitcoin.select_chain_params("bitcoin/regtest")
+@pytest.mark.usefixtures("setup_psbt_wallet")
+class AsyncioTestCase(IsolatedAsyncioTestCase, ParametrizedTestCase):
+
+    @parametrize(
+        'walletseed, xpub, spktype_wallet, spktype_destn, partial, psbt',
+        [
+            ("prosper diamond marriage spy across start shift elevator job lunar edge gallery",
+             "tpubDChjiEhsafnW2LcmK1C77XiEAgZddi6xZyxjMujBzUqZPTMRwsv3e5vSBYsdiPtCyc6TtoHTCjkxBjtF22tf8Z5ABRdeBUNwHCsqEyzR5wT",
+             "p2wpkh", "p2sh-p2wpkh", False,
+             "cHNidP8BAMQCAAAAA7uEliZeXLPfjeUiRBw6e5oZV1DtBrDmLthfDC4oaHQLAAAAAAD/////+X1Exketc4o5b9BPxsj70O+VlGvgiZz0KP1OMRtVLUQAAAAAAP////9r5ylMhQyxbJvCbU8aNE3NOPoXJwUaUZm4H3iT4RnaSwAAAAAA/////wKMz/AIAAAAABepFJwmRAefvZS7VQStD4k52Rn0k71Gh4zP8AgAAAAAFgAUA2shnTVftDXq+ssPwzml2UKdu1QAAAAAAAEBHwDh9QUAAAAAFgAUqw1Ifto4LztwcsxV6q+sQThIdloiBgMDZ5u3RN6Xum+OLkgAzwLFXGWFLwBUraMi7Oin4fYfrwzvYoLxAAAAAAAAAAAAAQEfAOH1BQAAAAAWABQpSCwoeMSghUoVflvtTPiqBPi+5yIGA/tAH4kVpqd3wzidaTNFxtwdpHTydkmB825us2w/3cAVDO9igvEAAAAAAQAAAAABAR8A4fUFAAAAABYAFEqM0KJ5FJ7ak2NL8PDqOPI0I1PaIgYD84aDwOqXKfGvEbre+bpNpuT0uZv6syESzz5PMu4RyLkM72KC8QAAAAACAAAAAAEAF6kUnCZEB5+9lLtVBK0PiTnZGfSTvUaHACICAw8k2gGGcF5sR8yKO5JeAkrkH15rmtCq8sCoDYbywTNzEO9igvEMAAAAIgAAAJQCAAAA"),
+            ("prosper diamond marriage spy across start shift elevator job lunar edge gallery",
+             "tpubDChjiEhsafnW2LcmK1C77XiEAgZddi6xZyxjMujBzUqZPTMRwsv3e5vSBYsdiPtCyc6TtoHTCjkxBjtF22tf8Z5ABRdeBUNwHCsqEyzR5wT",
+             "p2wpkh", "p2wpkh", False,
+             "cHNidP8BAMMCAAAAA7uEliZeXLPfjeUiRBw6e5oZV1DtBrDmLthfDC4oaHQLAAAAAAD/////+X1Exketc4o5b9BPxsj70O+VlGvgiZz0KP1OMRtVLUQAAAAAAP////9r5ylMhQyxbJvCbU8aNE3NOPoXJwUaUZm4H3iT4RnaSwAAAAAA/////wKMz/AIAAAAABYAFBaOTObQIdtCaryiPxaDV5rsGYWUjM/wCAAAAAAWABR9TJm5rcSoIMW7bE1bnj7REL/eygAAAAAAAQEfAOH1BQAAAAAWABSrDUh+2jgvO3ByzFXqr6xBOEh2WiIGAwNnm7dE3pe6b44uSADPAsVcZYUvAFStoyLs6Kfh9h+vDO9igvEAAAAAAAAAAAABAR8A4fUFAAAAABYAFClILCh4xKCFShV+W+1M+KoE+L7nIgYD+0AfiRWmp3fDOJ1pM0XG3B2kdPJ2SYHzbm6zbD/dwBUM72KC8QAAAAABAAAAAAEBHwDh9QUAAAAAFgAUSozQonkUntqTY0vw8Oo48jQjU9oiBgPzhoPA6pcp8a8Rut75uk2m5PS5m/qzIRLPPk8y7hHIuQzvYoLxAAAAAAIAAAAAACICAuqCicVUfcM5IiVSiB/0ZemodybG5Im9Fu8MLorQSE4UEO9igvEMAAAAIgAAAOkAAAAA"),
+            ("prosper diamond marriage spy across start shift elevator job lunar edge gallery",
+             "tpubDChjiEhsafnW2LcmK1C77XiEAgZddi6xZyxjMujBzUqZPTMRwsv3e5vSBYsdiPtCyc6TtoHTCjkxBjtF22tf8Z5ABRdeBUNwHCsqEyzR5wT",
+             "p2wpkh", "p2wpkh", True,
+             "cHNidP8BAMMCAAAAA7uEliZeXLPfjeUiRBw6e5oZV1DtBrDmLthfDC4oaHQLAAAAAAD/////+X1Exketc4o5b9BPxsj70O+VlGvgiZz0KP1OMRtVLUQAAAAAAP////9r5ylMhQyxbJvCbU8aNE3NOPoXJwUaUZm4H3iT4RnaSwAAAAAA/////wKMz/AIAAAAABYAFLH/IL11rTJ3wX1NcmUIsJ/T4j4jjM/wCAAAAAAWABR8GPNb1HUpCz8PKOc8aQXLD1wjcAAAAAAAAQEfAOH1BQAAAAAWABSrDUh+2jgvO3ByzFXqr6xBOEh2WiIGAwNnm7dE3pe6b44uSADPAsVcZYUvAFStoyLs6Kfh9h+vDE5vcGUAAAAAAAAAAAABAR8A4fUFAAAAABYAFClILCh4xKCFShV+W+1M+KoE+L7nIgYD+0AfiRWmp3fDOJ1pM0XG3B2kdPJ2SYHzbm6zbD/dwBUM72KC8QAAAAABAAAAAAEBHwDh9QUAAAAAFgAUSozQonkUntqTY0vw8Oo48jQjU9oiBgPzhoPA6pcp8a8Rut75uk2m5PS5m/qzIRLPPk8y7hHIuQzvYoLxAAAAAAIAAAAAACICAsQ7ZvU9tsbBoSje5rIJQBStlUkQaRCssKylEixre3AYEO9igvEMAAAAIgAAABcCAAAA"),
+        ])
+    async def test_sign_external_psbt(self, walletseed, xpub, spktype_wallet,
+                                      spktype_destn, partial, psbt):
+        bitcoin.select_chain_params("bitcoin")
+        wallet_cls = SegwitWallet if spktype_wallet == "p2wpkh" else SegwitLegacyWallet
+        wallet = await create_volatile_wallet(
+            walletseed, wallet_cls=wallet_cls)
+        # if we want to actually sign, our wallet has to recognize the fake utxos
+        # as being in the wallet, so we inject them:
+        class DummyUtxoManager(object):
+            _utxo = {0:{}}
+            def add_utxo(self, utxo, path, value, height):
+                self._utxo[0][utxo] = (path, value, height)
+        wallet._index_cache[0][0] = 1000
+        wallet._utxos = DummyUtxoManager()
+        p0, p1, p2 = (wallet.get_path(0, 0, i) for i in range(3))
+        if not partial:
+            wallet._utxos.add_utxo(utxostr_to_utxo(
+                "0b7468282e0c5fd82ee6b006ed5057199a7b3a1c4422e58ddfb35c5e269684bb:0"),
+                                   p0, 10000, 1)
+        wallet._utxos.add_utxo(utxostr_to_utxo(
+            "442d551b314efd28f49c89e06b9495efd0fbc8c64fd06f398a73ad47c6447df9:0"),
+                               p1, 10000, 1)
+        wallet._utxos.add_utxo(utxostr_to_utxo(
+            "4bda19e193781fb899511a052717fa38cd4d341a4f6dc29b6cb10c854c29e76b:0"),
+                               p2, 10000, 1)
+        signresult_and_signedpsbt, err = await wallet.sign_psbt(
+            base64.b64decode(psbt.encode("ascii")), with_sign_result=True)
+        assert not err
+        signresult, signedpsbt = signresult_and_signedpsbt
+        if partial:
+            assert not signresult.is_final
+            assert signresult.num_inputs_signed == 2
+            assert signresult.num_inputs_final == 2
+        else:
+            assert signresult.is_final
+            assert signresult.num_inputs_signed == 3
+            assert signresult.num_inputs_final == 3
+        print(PSBTWalletMixin.human_readable_psbt(signedpsbt))
+        bitcoin.select_chain_params("bitcoin/regtest")
+
+    async def test_create_and_sign_psbt_with_legacy(self):
+        """ The purpose of this test is to check that we can create and
+        then partially sign a PSBT where we own one input and the other input
+        is of legacy p2pkh type.
+        """
+        wallets = await make_wallets(1, [[1,0,0,0,0]], 1)
+        wallet_service = wallets[0]['wallet']
+        await wallet_service.sync_wallet(fast=True)
+        utxos = await wallet_service.select_utxos(
+            0, bitcoin.coins_to_satoshi(0.5))
+        assert len(utxos) == 1
+        # create a legacy address and make a payment into it
+        legacy_addr = bitcoin.CCoinAddress.from_scriptPubKey(
+            bitcoin.pubkey_to_p2pkh_script(
+                bitcoin.privkey_to_pubkey(b"\x01"*33)))
+        tx = await direct_send(
+            wallet_service, 0,
+            [(str(legacy_addr), bitcoin.coins_to_satoshi(0.3))],
+            accept_callback=dummy_accept_callback,
+            info_callback=dummy_info_callback,
+            return_transaction=True)
+        assert tx
+        # this time we will have one utxo worth <~ 0.7
+        my_utxos = await wallet_service.select_utxos(
+            0, bitcoin.coins_to_satoshi(0.5))
+        assert len(my_utxos) == 1
+        # find the outpoint for the legacy address we're spending
+        n = -1
+        for i, t in enumerate(tx.vout):
+            if bitcoin.CCoinAddress.from_scriptPubKey(t.scriptPubKey) == legacy_addr:
+                n = i
+        assert n > -1
+        utxos = copy.deepcopy(my_utxos)
+        utxos[(tx.GetTxid()[::-1], n)] ={"script": legacy_addr.to_scriptPubKey(),
+                  "value": bitcoin.coins_to_satoshi(0.3)}
+        outs = [{"value": bitcoin.coins_to_satoshi(0.998),
+                     "address": await wallet_service.get_addr(0,0,0)}]
+        tx2 = bitcoin.mktx(list(utxos.keys()), outs)
+        spent_outs = wallet_service.witness_utxos_to_psbt_utxos(my_utxos)
+        spent_outs.append(tx)
+        new_psbt = await wallet_service.create_psbt_from_tx(
+            tx2, spent_outs, force_witness_utxo=False)
+        signed_psbt_and_signresult, err = await wallet_service.sign_psbt(
+            new_psbt.serialize(), with_sign_result=True)
+        assert err is None
+        signresult, signed_psbt = signed_psbt_and_signresult
+        assert signresult.num_inputs_signed == 1
+        assert signresult.num_inputs_final == 1
+        assert not signresult.is_final
+
+    @parametrize(
+        'unowned_utxo, wallet_cls',
+        [
+            (True, SegwitLegacyWallet),
+            (False, SegwitLegacyWallet),
+            (True, SegwitWallet),
+            (False, SegwitWallet),
+            (True, LegacyWallet),
+            (False, LegacyWallet),
+        ])
+    async def test_create_psbt_and_sign(self, unowned_utxo, wallet_cls):
+        """ Plan of test:
+        1. Create a wallet and source 3 destination addresses.
+        2. Make, and confirm, transactions that fund the 3 addrs.
+        3. Create a new tx spending 2 of those 3 utxos and spending
+           another utxo we don't own (extra is optional per `unowned_utxo`).
+        4. Create a psbt using the above transaction and corresponding
+           `spent_outs` field to fill in the redeem script.
+        5. Compare resulting PSBT with expected structure.
+        6. Use the wallet's sign_psbt method to sign the whole psbt, which
+           means signing each input we own.
+        7. Check that each input is finalized as per expected. Check that the whole
+           PSBT is or is not finalized as per whether there is an unowned utxo.
+        8. In case where whole psbt is finalized, attempt to broadcast the tx.
+        """
+        # steps 1 and 2:
+        wallets = await make_wallets(
+            1, [[3,0,0,0,0]], 1, wallet_cls=wallet_cls)
+        wallet_service = wallets[0]['wallet']
+        await wallet_service.sync_wallet(fast=True)
+        utxos = await wallet_service.select_utxos(
+            0, bitcoin.coins_to_satoshi(1.5))
+        # for legacy wallets, psbt creation requires querying for the spending
+        # transaction:
+        if wallet_cls == LegacyWallet:
+            fulltxs = []
+            for utxo, v in utxos.items():
+                fulltxs.append(jm_single().bc_interface.get_deser_from_gettransaction(
+                    jm_single().bc_interface.get_transaction(utxo[0])))
+
+        assert len(utxos) == 2
+        u_utxos = {}
+        if unowned_utxo:
+            # note: tx creation uses the key only; psbt creation uses the value,
+            # which can be fake here; we do not intend to attempt to fully
+            # finalize a psbt with an unowned input. See
+            # https://github.com/Simplexum/python-bitcointx/issues/30
+            # the redeem script creation (which is artificial) will be
+            # avoided in future.
+            priv = b"\xaa"*32 + b"\x01"
+            pub = bitcoin.privkey_to_pubkey(priv)
+            script = bitcoin.pubkey_to_p2sh_p2wpkh_script(pub)
+            redeem_script = bitcoin.pubkey_to_p2wpkh_script(pub)
+            u_utxos[(b"\xaa"*32, 12)] = {"value": 1000, "script": script}
+        utxos.update(u_utxos)
+        # outputs aren't interesting for this test (we selected 1.5 but will get 2):
+        outs = [{"value": bitcoin.coins_to_satoshi(1.999),
+                 "address": await wallet_service.get_addr(0,0,0)}]
+        tx = bitcoin.mktx(list(utxos.keys()), outs)
+
+        if wallet_cls != LegacyWallet:
+            spent_outs = wallet_service.witness_utxos_to_psbt_utxos(utxos)
+            force_witness_utxo=True
+        else:
+            spent_outs = fulltxs
+            # the extra input is segwit:
+            if unowned_utxo:
+                spent_outs.extend(
+                    wallet_service.witness_utxos_to_psbt_utxos(u_utxos))
+            force_witness_utxo=False
+        newpsbt = await wallet_service.create_psbt_from_tx(
+            tx, spent_outs, force_witness_utxo=force_witness_utxo)
+        # see note above
+        if unowned_utxo:
+            newpsbt.inputs[-1].redeem_script = redeem_script
+        print(bintohex(newpsbt.serialize()))
+        print("human readable: ")
+        print(wallet_service.human_readable_psbt(newpsbt))
+        # we cannot compare with a fixed expected result due to wallet randomization, but we can
+        # check psbt structure:
+        expected_inputs_length = 3 if unowned_utxo else 2
+        assert len(newpsbt.inputs) == expected_inputs_length
+        assert len(newpsbt.outputs) == 1
+        # note: redeem_script field is a CScript which is a bytes instance,
+        # so checking length is best way to check for existence (comparison
+        # with None does not work):
+        if wallet_cls == SegwitLegacyWallet:
+            assert len(newpsbt.inputs[0].redeem_script) != 0
+            assert len(newpsbt.inputs[1].redeem_script) != 0
+        if unowned_utxo:
+            assert newpsbt.inputs[2].redeem_script == redeem_script
+
+        signed_psbt_and_signresult, err = await wallet_service.sign_psbt(
+            newpsbt.serialize(), with_sign_result=True)
+        assert err is None
+        signresult, signed_psbt = signed_psbt_and_signresult
+        expected_signed_inputs = len(utxos) if not unowned_utxo else len(utxos)-1
+        assert signresult.num_inputs_signed == expected_signed_inputs
+        assert signresult.num_inputs_final == expected_signed_inputs
+
+        if not unowned_utxo:
+            assert signresult.is_final
+            # only in case all signed do we try to broadcast:
+            extracted_tx = signed_psbt.extract_transaction().serialize()
+            assert jm_single().bc_interface.pushtx(extracted_tx)
+        else:
+            # transaction extraction must fail for not-fully-signed psbts:
+            with pytest.raises(ValueError) as e:
+                extracted_tx = signed_psbt.extract_transaction()
+
+    @parametrize(
+        'payment_amt, wallet_cls_sender, wallet_cls_receiver',
+        [
+            (0.05, SegwitLegacyWallet, SegwitLegacyWallet),
+            #(0.95, SegwitLegacyWallet, SegwitWallet),
+            #(0.05, SegwitWallet, SegwitLegacyWallet),
+            #(0.95, SegwitWallet, SegwitWallet),
+        ])
+    async def test_payjoin_workflow(self, payment_amt, wallet_cls_sender,
+                                    wallet_cls_receiver):
+        """ Workflow step 1:
+        Create a payment from a wallet, and create a finalized PSBT.
+        This step is fairly trivial as the functionality is built-in to
+        PSBTWalletMixin.
+        Note that only Segwit* wallets are supported for PayJoin.
+
+            Workflow step 2:
+        Receiver creates a new partially signed PSBT with the same amount
+        and at least one more utxo.
+
+            Workflow step 3:
+        Given a partially signed PSBT created by a receiver, here the sender
+        completes (co-signs) the PSBT they are given. Note this code is a PSBT
+        functionality check, and does NOT include the detailed checks that
+        the sender should perform before agreeing to sign (see:
+        https://github.com/btcpayserver/btcpayserver-doc/blob/eaac676866a4d871eda5fd7752b91b88fdf849ff/Payjoin-spec.md#receiver-side
+        ).
+        """
+
+        wallets = await make_wallets(
+            1, [[3,0,0,0,0]], 1, wallet_cls=wallet_cls_receiver)
+        wallet_r = wallets[0]["wallet"]
+        wallets = await make_wallets(
+            1, [[3,0,0,0,0]], 1, wallet_cls=wallet_cls_sender)
+        wallet_s = wallets[0]["wallet"]
+        for w in [wallet_r, wallet_s]:
+            await w.sync_wallet(fast=True)
+
+        # destination address for payment:
+        destaddr = str(bitcoin.CCoinAddress.from_scriptPubKey(
+            bitcoin.pubkey_to_p2wpkh_script(bitcoin.privkey_to_pubkey(b"\x01"*33))))
+
+        payment_amt = bitcoin.coins_to_satoshi(payment_amt)
+
+        # *** STEP 1 ***
+        # **************
+
+        # create a normal tx from the sender wallet:
+        payment_psbt = await direct_send(
+            wallet_s, 0,
+            [(destaddr, payment_amt)],
+            accept_callback=dummy_accept_callback,
+            info_callback=dummy_info_callback,
+            with_final_psbt=True)
+
+        print("Initial payment PSBT created:\n{}".format(
+            wallet_s.human_readable_psbt(payment_psbt)))
+        # ensure that the payemnt amount is what was intended:
+        out_amts = [x.nValue for x in payment_psbt.unsigned_tx.vout]
+        # NOTE this would have to change for more than 2 outputs:
+        assert any([out_amts[i] == payment_amt for i in [0, 1]])
+
+        # ensure that we can actually broadcast the created tx:
+        # (note that 'extract_transaction' represents an implicit
+        # PSBT finality check).
+        extracted_tx = payment_psbt.extract_transaction().serialize()
+        # don't want to push the tx right now, because of test structure
+        # (in production code this isn't really needed, we will not
+        # produce invalid payment transactions).
+        assert jm_single().bc_interface.testmempoolaccept(bintohex(extracted_tx)),\
+            "Payment transaction was rejected from mempool."
+
+        # *** STEP 2 ***
+        # **************
+
+        # Simple receiver utxo choice heuristic.
+        # For more generality we test with two receiver-utxos, not one.
+        all_receiver_utxos = await wallet_r.get_all_utxos()
+        # TODO is there a less verbose way to get any 2 utxos from the dict?
+        receiver_utxos_keys = list(all_receiver_utxos.keys())[:2]
+        receiver_utxos = {k: v for k, v in all_receiver_utxos.items(
+            ) if k in receiver_utxos_keys}
+
+        # receiver will do other checks as discussed above, including payment
+        # amount; as discussed above, this is out of the scope of this PSBT test.
+
+        # construct unsigned tx for payjoin-psbt:
+        payjoin_tx_inputs = [(x.prevout.hash[::-1],
+                    x.prevout.n) for x in payment_psbt.unsigned_tx.vin]
+        payjoin_tx_inputs.extend(receiver_utxos.keys())
+        # find payment output and change output
+        pay_out = None
+        change_out = None
+        for o in payment_psbt.unsigned_tx.vout:
+            jm_out_fmt = {"value": o.nValue,
+            "address": str(bitcoin.CCoinAddress.from_scriptPubKey(
+            o.scriptPubKey))}
+            if o.nValue == payment_amt:
+                assert pay_out is None
+                pay_out = jm_out_fmt
+            else:
+                assert change_out is None
+                change_out = jm_out_fmt
+
+        # we now know there were two outputs and know which is payment.
+        # bump payment output with our input:
+        outs = [pay_out, change_out]
+        our_inputs_val = sum([v["value"] for _, v in receiver_utxos.items()])
+        pay_out["value"] += our_inputs_val
+        print("we bumped the payment output value by: ", our_inputs_val)
+        print("It is now: ", pay_out["value"])
+        unsigned_payjoin_tx = bitcoin.make_shuffled_tx(payjoin_tx_inputs, outs,
+                                    version=payment_psbt.unsigned_tx.nVersion,
+                                    locktime=payment_psbt.unsigned_tx.nLockTime)
+        print("we created this unsigned tx: ")
+        print(bitcoin.human_readable_transaction(unsigned_payjoin_tx))
+        # to create the PSBT we need the spent_outs for each input,
+        # in the right order:
+        spent_outs = []
+        for i, inp in enumerate(unsigned_payjoin_tx.vin):
+            input_found = False
+            for j, inp2 in enumerate(payment_psbt.unsigned_tx.vin):
+                if inp.prevout == inp2.prevout:
+                    spent_outs.append(payment_psbt.inputs[j].utxo)
+                    input_found = True
+                    break
+            if input_found:
+                continue
+            # if we got here this input is ours, we must find
+            # it from our original utxo choice list:
+            for ru in receiver_utxos.keys():
+                if (inp.prevout.hash[::-1], inp.prevout.n) == ru:
+                    spent_outs.append(
+                        wallet_r.witness_utxos_to_psbt_utxos(
+                            {ru: receiver_utxos[ru]})[0])
+                    input_found = True
+                    break
+            # there should be no other inputs:
+            assert input_found
+
+        r_payjoin_psbt = await wallet_r.create_psbt_from_tx(
+            unsigned_payjoin_tx, spent_outs=spent_outs)
+        print("Receiver created payjoin PSBT:\n{}".format(
+            wallet_r.human_readable_psbt(r_payjoin_psbt)))
+
+        signresultandpsbt, err = await wallet_r.sign_psbt(
+            r_payjoin_psbt.serialize(), with_sign_result=True)
+        assert not err, err
+        signresult, receiver_signed_psbt = signresultandpsbt
+        assert signresult.num_inputs_final == len(receiver_utxos)
+        assert not signresult.is_final
+
+        print("Receiver signing successful. Payjoin PSBT is now:\n{}".format(
+            wallet_r.human_readable_psbt(receiver_signed_psbt)))
+
+        # *** STEP 3 ***
+        # **************
+
+        # take the half-signed PSBT, validate and co-sign:
+
+        signresultandpsbt, err = await wallet_s.sign_psbt(
+            receiver_signed_psbt.serialize(), with_sign_result=True)
+        assert not err, err
+        signresult, sender_signed_psbt =  signresultandpsbt
+        print("Sender's final signed PSBT is:\n{}".format(
+            wallet_s.human_readable_psbt(sender_signed_psbt)))
+        assert signresult.is_final
+
+        # broadcast the tx
+        extracted_tx = sender_signed_psbt.extract_transaction().serialize()
+        assert jm_single().bc_interface.pushtx(extracted_tx)
+
+    async def test_hr_psbt(self):
+        bitcoin.select_chain_params("bitcoin")
+        for k, v in hr_test_vectors.items():
+            print(PSBTWalletMixin.human_readable_psbt(
+                bitcoin.PartiallySignedTransaction.from_binary(hextobin(v))))
+        bitcoin.select_chain_params("bitcoin/regtest")
 
 @pytest.fixture(scope="module")
 def setup_psbt_wallet():

test/jmclient/test_snicker.py

@@ -3,6 +3,12 @@
    wallets as defined in jmclient.wallet.'''
 
 import pytest
+from unittest import IsolatedAsyncioTestCase
+
+from unittest_parametrize import parametrize, ParametrizedTestCase
+
+import jmclient  # install asyncioreactor
+from twisted.internet import reactor
 
 from commontest import make_wallets, dummy_accept_callback, dummy_info_callback
 import jmbitcoin as btc
@@ -14,105 +20,112 @@ pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
 
 log = get_log()
 
-@pytest.mark.parametrize(
-    "nw, wallet_structures, mean_amt, sdev_amt, amt, net_transfer", [
-        (2, [[1, 0, 0, 0, 0]] * 2, 4, 0, 20000000, 1000),
-    ])
-def test_snicker_e2e(setup_snicker, nw, wallet_structures,
-                     mean_amt, sdev_amt, amt, net_transfer):
-    """ Test strategy:
-    1. create two wallets.
-    2. with wallet 1 (Receiver), create a single transaction
-    tx1, from mixdepth 0 to 1.
-    3. with wallet 2 (Proposer), take pubkey of all inputs from tx1, and use
-    them to create snicker proposals to the non-change out of tx1,
-    in base64 and place in proposals.txt.
-    4. Receiver polls for proposals in the file manually (instead of twisted
-    LoopingCall) and processes them.
-    5. Check for valid final transaction with broadcast.
-    """
-
-    # TODO: Make this test work with native segwit wallets
-    wallets = make_wallets(nw, wallet_structures, mean_amt, sdev_amt)
-    for w in wallets.values():
-        w['wallet'].sync_wallet(fast=True)
-    print(wallets)
-    wallet_r = wallets[0]['wallet']
-    wallet_p = wallets[1]['wallet']
-    # next, create a tx from the receiver wallet
-    our_destn_script = wallet_r.get_new_script(1, BaseWallet.ADDRESS_TYPE_INTERNAL)
-    tx = direct_send(wallet_r, 0,
-                     [(
-                        wallet_r.script_to_addr(our_destn_script),
-                        btc.coins_to_satoshi(0.3)
-                     )],
-                     accept_callback=dummy_accept_callback,
-                     info_callback=dummy_info_callback,
-                     return_transaction=True)    
-    
-    assert tx, "Failed to spend from receiver wallet"
-    print("Parent transaction OK. It was: ")
-    print(btc.human_readable_transaction(tx))
-    wallet_r.process_new_tx(tx)
-    # we must identify the receiver's output we're going to use;
-    # it can be destination or change, that's up to the proposer
-    # to guess successfully; here we'll just choose index 0.
-    txid1 = tx.GetTxid()[::-1]
-    txid1_index = 0
-
-    receiver_start_bal = sum([x['value'] for x in wallet_r.get_all_utxos(
-        ).values()])
-
-    # Now create a proposal for every input index in tx1
-    # (version 1 proposals mean we source keys from the/an
-    # ancestor transaction)
-    propose_keys = []
-    for i in range(len(tx.vin)):
-        # todo check access to pubkey
-        sig, pub = [a for a in iter(tx.wit.vtxinwit[i].scriptWitness)]
-        propose_keys.append(pub)
-    # the proposer wallet needs to choose a single
-    # utxo that is bigger than the output amount of tx1
-    prop_m_utxos = wallet_p.get_utxos_by_mixdepth()[0]
-    prop_utxo = prop_m_utxos[list(prop_m_utxos)[0]]
-    # get the private key for that utxo
-    priv = wallet_p.get_key_from_addr(
-        wallet_p.script_to_addr(prop_utxo['script']))
-    prop_input_amt = prop_utxo['value']
-    # construct the arguments for the snicker proposal:
-    our_input = list(prop_m_utxos)[0] # should be (txid, index)
-    their_input = (txid1, txid1_index)
-    our_input_utxo = btc.CMutableTxOut(prop_utxo['value'],
-                                       prop_utxo['script'])
-    fee_est = estimate_tx_fee(len(tx.vin), 2,
-                              txtype=wallet_p.get_txtype())
-    change_spk = wallet_p.get_new_script(0, BaseWallet.ADDRESS_TYPE_INTERNAL)
-
-    encrypted_proposals = []
-
-    for p in propose_keys:
-        # TODO: this can be a loop over all outputs,
-        # not just one guessed output, if desired.
-        encrypted_proposals.append(
-            wallet_p.create_snicker_proposal(
-            [our_input], their_input,
-            [our_input_utxo],
-            tx.vout[txid1_index],
-            net_transfer,
-            fee_est,
-            priv,
-            p,
-            prop_utxo['script'],
-            change_spk,
-            version_byte=1) + b"," + bintohex(p).encode('utf-8'))
-    sR = SNICKERReceiver(wallet_r)
-    sR.process_proposals([x.decode("utf-8") for x in encrypted_proposals])
-    assert len(sR.successful_txs) == 1
-    wallet_r.process_new_tx(sR.successful_txs[0])
-    end_utxos = wallet_r.get_all_utxos()
-    print("At end the receiver has these utxos: ", end_utxos)
-    receiver_end_bal = sum([x['value'] for x in end_utxos.values()])
-    assert receiver_end_bal == receiver_start_bal + net_transfer
+
+@pytest.mark.usefixtures("setup_snicker")
+class AsyncioTestCase(IsolatedAsyncioTestCase, ParametrizedTestCase):
+
+    @parametrize(
+        "nw, wallet_structures, mean_amt, sdev_amt, amt, net_transfer",
+        [
+            (2, [[1, 0, 0, 0, 0]] * 2, 4, 0, 20000000, 1000),
+        ])
+    async def test_snicker_e2e(self, nw, wallet_structures,
+                               mean_amt, sdev_amt, amt, net_transfer):
+        """ Test strategy:
+        1. create two wallets.
+        2. with wallet 1 (Receiver), create a single transaction
+        tx1, from mixdepth 0 to 1.
+        3. with wallet 2 (Proposer), take pubkey of all inputs from tx1, and use
+        them to create snicker proposals to the non-change out of tx1,
+        in base64 and place in proposals.txt.
+        4. Receiver polls for proposals in the file manually (instead of twisted
+        LoopingCall) and processes them.
+        5. Check for valid final transaction with broadcast.
+        """
+
+        # TODO: Make this test work with native segwit wallets
+        wallets = await make_wallets(nw, wallet_structures, mean_amt, sdev_amt)
+        for w in wallets.values():
+            await w['wallet'].sync_wallet(fast=True)
+        print(wallets)
+        wallet_r = wallets[0]['wallet']
+        wallet_p = wallets[1]['wallet']
+        # next, create a tx from the receiver wallet
+        our_destn_script = await wallet_r.get_new_script(
+            1, BaseWallet.ADDRESS_TYPE_INTERNAL)
+        tx = await direct_send(
+            wallet_r, 0,
+            [(await wallet_r.script_to_addr(our_destn_script),
+              btc.coins_to_satoshi(0.3))],
+            accept_callback=dummy_accept_callback,
+            info_callback=dummy_info_callback,
+            return_transaction=True)
+
+        assert tx, "Failed to spend from receiver wallet"
+        print("Parent transaction OK. It was: ")
+        print(btc.human_readable_transaction(tx))
+        await wallet_r.process_new_tx(tx)
+        # we must identify the receiver's output we're going to use;
+        # it can be destination or change, that's up to the proposer
+        # to guess successfully; here we'll just choose index 0.
+        txid1 = tx.GetTxid()[::-1]
+        txid1_index = 0
+
+        receiver_start_bal = sum(
+            [x['value'] for x in (await wallet_r.get_all_utxos()).values()])
+
+        # Now create a proposal for every input index in tx1
+        # (version 1 proposals mean we source keys from the/an
+        # ancestor transaction)
+        propose_keys = []
+        for i in range(len(tx.vin)):
+            # todo check access to pubkey
+            sig, pub = [a for a in iter(tx.wit.vtxinwit[i].scriptWitness)]
+            propose_keys.append(pub)
+        # the proposer wallet needs to choose a single
+        # utxo that is bigger than the output amount of tx1
+        prop_m_utxos = (await wallet_p.get_utxos_by_mixdepth())[0]
+        prop_utxo = prop_m_utxos[list(prop_m_utxos)[0]]
+        # get the private key for that utxo
+        addr = await wallet_p.script_to_addr(prop_utxo['script'])
+        priv = wallet_p.get_key_from_addr(addr)
+        prop_input_amt = prop_utxo['value']
+        # construct the arguments for the snicker proposal:
+        our_input = list(prop_m_utxos)[0] # should be (txid, index)
+        their_input = (txid1, txid1_index)
+        our_input_utxo = btc.CMutableTxOut(prop_utxo['value'],
+                                           prop_utxo['script'])
+        fee_est = estimate_tx_fee(len(tx.vin), 2,
+                                  txtype=wallet_p.get_txtype())
+        change_spk = await wallet_p.get_new_script(
+            0, BaseWallet.ADDRESS_TYPE_INTERNAL)
+
+        encrypted_proposals = []
+
+        for p in propose_keys:
+            # TODO: this can be a loop over all outputs,
+            # not just one guessed output, if desired.
+            encrypted_proposals.append(
+                await wallet_p.create_snicker_proposal(
+                    [our_input], their_input,
+                    [our_input_utxo],
+                    tx.vout[txid1_index],
+                    net_transfer,
+                    fee_est,
+                    priv,
+                    p,
+                    prop_utxo['script'],
+                    change_spk,
+                    version_byte=1) + b"," + bintohex(p).encode('utf-8'))
+        sR = SNICKERReceiver(wallet_r)
+        await sR.process_proposals(
+            [x.decode("utf-8") for x in encrypted_proposals])
+        assert len(sR.successful_txs) == 1
+        await wallet_r.process_new_tx(sR.successful_txs[0])
+        end_utxos = await wallet_r.get_all_utxos()
+        print("At end the receiver has these utxos: ", end_utxos)
+        receiver_end_bal = sum([x['value'] for x in end_utxos.values()])
+        assert receiver_end_bal == receiver_start_bal + net_transfer
 
 @pytest.fixture(scope="module")
 def setup_snicker(request):

test/jmclient/test_tx_creation.py

@@ -5,11 +5,16 @@
    p2(w)sh tests, these have been removed since Joinmarket
    does not use this feature.'''
 
+import pytest
 import struct
+from unittest import IsolatedAsyncioTestCase
+
+import jmclient  # install asyncioreactor
+from twisted.internet import reactor
+
 from commontest import make_wallets, make_sign_and_push, ensure_bip65_activated
 
 import jmbitcoin as bitcoin
-import pytest
 from jmbase import get_log
 from jmclient import load_test_config, jm_single, direct_send, estimate_tx_fee, compute_tx_locktime
 
@@ -25,250 +30,266 @@ vpubs = ["03e9a06e539d6bf5cf1ca5c41b59121fa3df07a338322405a312c67b6349a707e9",
          "028a2f126e3999ff66d01dcb101ab526d3aa1bf5cbdc4bde14950a4cead95f6fcb",
          "02bea84d70e74f7603746b62d79bf035e16d982b56e6a1ee07dfd3b9130e8a2ad9"]
 
-def test_all_same_priv(setup_tx_creation):
-    #recipient
-    priv = b"\xaa"*32 + b"\x01"
-    pub = bitcoin.privkey_to_pubkey(priv)
-    addr = str(bitcoin.CCoinAddress.from_scriptPubKey(
-        bitcoin.CScript([bitcoin.OP_0, bitcoin.Hash160(pub)])))
-    wallet_service = make_wallets(1, [[1,0,0,0,0]], 1)[0]['wallet']
-    #make another utxo on the same address
-    addrinwallet = wallet_service.get_addr(0,0,0)
-    jm_single().bc_interface.grab_coins(addrinwallet, 1)
-    wallet_service.sync_wallet(fast=True)
-    insfull = wallet_service.select_utxos(0, 110000000)
-    outs = [{"address": addr, "value": 1000000}]
-    ins = list(insfull.keys())
-    tx = bitcoin.mktx(ins, outs)
-    scripts = {}
-    for i, j in enumerate(ins):
-        scripts[i] = (insfull[j]["script"], insfull[j]["value"])
-    success, msg = wallet_service.sign_tx(tx, scripts)
-    assert success, msg
 
-def test_verify_tx_input(setup_tx_creation):
-    priv = b"\xaa"*32 + b"\x01"
-    pub = bitcoin.privkey_to_pubkey(priv)
-    script = bitcoin.pubkey_to_p2sh_p2wpkh_script(pub)
-    addr = str(bitcoin.CCoinAddress.from_scriptPubKey(script))
-    wallet_service = make_wallets(1, [[2,0,0,0,0]], 1)[0]['wallet']
-    wallet_service.sync_wallet(fast=True)
-    insfull = wallet_service.select_utxos(0, 110000000)
-    outs = [{"address": addr, "value": 1000000}]
-    ins = list(insfull.keys())
-    tx = bitcoin.mktx(ins, outs)
-    scripts = {0: (insfull[ins[0]]["script"], bitcoin.coins_to_satoshi(1))}
-    success, msg = wallet_service.sign_tx(tx, scripts)
-    assert success, msg
-    # testing Joinmarket's ability to verify transaction inputs
-    # of others: pretend we don't have a wallet owning the transaction,
-    # and instead verify an input using the (sig, pub, scriptCode) data
-    # that is sent by counterparties:
-    cScrWit = tx.wit.vtxinwit[0].scriptWitness
-    sig = cScrWit.stack[0]
-    pub = cScrWit.stack[1]
-    scriptSig = tx.vin[0].scriptSig
-    tx2 = bitcoin.mktx(ins, outs)
-    res = bitcoin.verify_tx_input(tx2, 0, scriptSig,
-                            bitcoin.pubkey_to_p2wpkh_script(pub),
-                            amount = bitcoin.coins_to_satoshi(1),
-                            witness = bitcoin.CScriptWitness([sig, pub]))
-    assert res
+@pytest.mark.usefixtures("setup_tx_creation")
+class AsyncioTestCase(IsolatedAsyncioTestCase):
 
-def test_absurd_fees(setup_tx_creation):
-    """Test triggering of ValueError exception
-    if the transaction fees calculated from the blockchain
-    interface exceed the limit set in the config.
-    """
-    jm_single().bc_interface.absurd_fees = True
-    #pay into it
-    wallet_service = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet']
-    wallet_service.sync_wallet(fast=True)
-    amount = 350000000
-    ins_full = wallet_service.select_utxos(0, amount)
-    with pytest.raises(ValueError) as e_info:
-        txid = make_sign_and_push(ins_full, wallet_service, amount, estimate_fee=True)
-    jm_single().bc_interface.absurd_fees = False
+    async def test_all_same_priv(self):
+        #recipient
+        priv = b"\xaa"*32 + b"\x01"
+        pub = bitcoin.privkey_to_pubkey(priv)
+        addr = str(bitcoin.CCoinAddress.from_scriptPubKey(
+            bitcoin.CScript([bitcoin.OP_0, bitcoin.Hash160(pub)])))
+        wallets = await make_wallets(1, [[1,0,0,0,0]], 1)
+        wallet_service = wallets[0]['wallet']
+        #make another utxo on the same address
+        addrinwallet = await wallet_service.get_addr(0,0,0)
+        jm_single().bc_interface.grab_coins(addrinwallet, 1)
+        await wallet_service.sync_wallet(fast=True)
+        insfull = await wallet_service.select_utxos(0, 110000000)
+        outs = [{"address": addr, "value": 1000000}]
+        ins = list(insfull.keys())
+        tx = bitcoin.mktx(ins, outs)
+        scripts = {}
+        for i, j in enumerate(ins):
+            scripts[i] = (insfull[j]["script"], insfull[j]["value"])
+        success, msg = await wallet_service.sign_tx(tx, scripts)
+        assert success, msg
+
+    async def test_verify_tx_input(self):
+        priv = b"\xaa"*32 + b"\x01"
+        pub = bitcoin.privkey_to_pubkey(priv)
+        script = bitcoin.pubkey_to_p2sh_p2wpkh_script(pub)
+        addr = str(bitcoin.CCoinAddress.from_scriptPubKey(script))
+        wallets = await make_wallets(1, [[2,0,0,0,0]], 1)
+        wallet_service = wallets[0]['wallet']
+        await wallet_service.sync_wallet(fast=True)
+        insfull = await wallet_service.select_utxos(0, 110000000)
+        outs = [{"address": addr, "value": 1000000}]
+        ins = list(insfull.keys())
+        tx = bitcoin.mktx(ins, outs)
+        scripts = {0: (insfull[ins[0]]["script"], bitcoin.coins_to_satoshi(1))}
+        success, msg = await wallet_service.sign_tx(tx, scripts)
+        assert success, msg
+        # testing Joinmarket's ability to verify transaction inputs
+        # of others: pretend we don't have a wallet owning the transaction,
+        # and instead verify an input using the (sig, pub, scriptCode) data
+        # that is sent by counterparties:
+        cScrWit = tx.wit.vtxinwit[0].scriptWitness
+        sig = cScrWit.stack[0]
+        pub = cScrWit.stack[1]
+        scriptSig = tx.vin[0].scriptSig
+        tx2 = bitcoin.mktx(ins, outs)
+        res = bitcoin.verify_tx_input(tx2, 0, scriptSig,
+                                bitcoin.pubkey_to_p2wpkh_script(pub),
+                                amount = bitcoin.coins_to_satoshi(1),
+                                witness = bitcoin.CScriptWitness([sig, pub]))
+        assert res
 
-def test_create_sighash_txs(setup_tx_creation):
-    #non-standard hash codes:
-    for sighash in [bitcoin.SIGHASH_ANYONECANPAY + bitcoin.SIGHASH_SINGLE,
-                    bitcoin.SIGHASH_NONE, bitcoin.SIGHASH_SINGLE]:
-        wallet_service = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet']
-        wallet_service.sync_wallet(fast=True)
+    async def test_absurd_fees(self):
+        """Test triggering of ValueError exception
+        if the transaction fees calculated from the blockchain
+        interface exceed the limit set in the config.
+        """
+        jm_single().bc_interface.absurd_fees = True
+        #pay into it
+        wallets = await make_wallets(1, [[2, 0, 0, 0, 1]], 3)
+        wallet_service = wallets[0]['wallet']
+        await wallet_service.sync_wallet(fast=True)
         amount = 350000000
-        ins_full = wallet_service.select_utxos(0, amount)
-        txid = make_sign_and_push(ins_full, wallet_service, amount, hashcode=sighash)
-        assert txid
+        ins_full = await wallet_service.select_utxos(0, amount)
+        with pytest.raises(ValueError) as e_info:
+            txid = await make_sign_and_push(
+                ins_full, wallet_service, amount, estimate_fee=True)
+        jm_single().bc_interface.absurd_fees = False
 
-    #trigger insufficient funds
-    with pytest.raises(Exception) as e_info:
-        fake_utxos = wallet_service.select_utxos(4, 1000000000)
+    async def test_create_sighash_txs(self):
+        #non-standard hash codes:
+        for sighash in [bitcoin.SIGHASH_ANYONECANPAY + bitcoin.SIGHASH_SINGLE,
+                        bitcoin.SIGHASH_NONE, bitcoin.SIGHASH_SINGLE]:
+            wallets = await make_wallets(1, [[2, 0, 0, 0, 1]], 3)
+            wallet_service = wallets[0]['wallet']
+            await wallet_service.sync_wallet(fast=True)
+            amount = 350000000
+            ins_full = await wallet_service.select_utxos(0, amount)
+            txid = await make_sign_and_push(
+                ins_full, wallet_service, amount, hashcode=sighash)
+            assert txid
 
-def test_spend_p2wpkh(setup_tx_creation):
-    #make 3 p2wpkh outputs from 3 privs
-    privs = [struct.pack(b'B', x) * 32 + b'\x01' for x in range(1, 4)]
-    pubs = [bitcoin.privkey_to_pubkey(priv) for priv in privs]
-    scriptPubKeys = [bitcoin.pubkey_to_p2wpkh_script(pub) for pub in pubs]
-    addresses = [str(bitcoin.CCoinAddress.from_scriptPubKey(
-        spk)) for spk in scriptPubKeys]
-    #pay into it
-    wallet_service = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet']
-    wallet_service.sync_wallet(fast=True)
-    amount = 35000000
-    p2wpkh_ins = []
-    for i, addr in enumerate(addresses):
-        ins_full = wallet_service.select_utxos(0, amount)
-        txid = make_sign_and_push(ins_full, wallet_service, amount, output_addr=addr)
-        assert txid
-        p2wpkh_ins.append((txid, 0))
-        txhex = jm_single().bc_interface.get_transaction(txid)
-        #wait for mining
-        jm_single().bc_interface.tick_forward_chain(1)
-    #random output address
-    output_addr = wallet_service.get_internal_addr(1)
-    amount2 = amount*3 - 50000
-    outs = [{'value': amount2, 'address': output_addr}]
-    tx = bitcoin.mktx(p2wpkh_ins, outs)
+        #trigger insufficient funds
+        with pytest.raises(Exception) as e_info:
+            fake_utxos = await wallet_service.select_utxos(4, 1000000000)
+
+    async def test_spend_p2wpkh(self):
+        #make 3 p2wpkh outputs from 3 privs
+        privs = [struct.pack(b'B', x) * 32 + b'\x01' for x in range(1, 4)]
+        pubs = [bitcoin.privkey_to_pubkey(priv) for priv in privs]
+        scriptPubKeys = [bitcoin.pubkey_to_p2wpkh_script(pub) for pub in pubs]
+        addresses = [str(bitcoin.CCoinAddress.from_scriptPubKey(
+            spk)) for spk in scriptPubKeys]
+        #pay into it
+        wallets = await make_wallets(1, [[3, 0, 0, 0, 0]], 3)
+        wallet_service = wallets[0]['wallet']
+        await wallet_service.sync_wallet(fast=True)
+        amount = 35000000
+        p2wpkh_ins = []
+        for i, addr in enumerate(addresses):
+            ins_full = await wallet_service.select_utxos(0, amount)
+            txid = await make_sign_and_push(
+                ins_full, wallet_service, amount, output_addr=addr)
+            assert txid
+            p2wpkh_ins.append((txid, 0))
+            txhex = jm_single().bc_interface.get_transaction(txid)
+            #wait for mining
+            jm_single().bc_interface.tick_forward_chain(1)
+        #random output address
+        output_addr = await wallet_service.get_internal_addr(1)
+        amount2 = amount*3 - 50000
+        outs = [{'value': amount2, 'address': output_addr}]
+        tx = bitcoin.mktx(p2wpkh_ins, outs)
 
-    for i, priv in enumerate(privs):
-        # sign each of 3 inputs; note that bitcoin.sign
-        # automatically validates each signature it creates.
-        sig, msg = bitcoin.sign(tx, i, priv, amount=amount, native="p2wpkh")
-        if not sig:
-            assert False, msg
-    txid = jm_single().bc_interface.pushtx(tx.serialize())
-    assert txid
+        for i, priv in enumerate(privs):
+            # sign each of 3 inputs; note that bitcoin.sign
+            # automatically validates each signature it creates.
+            sig, msg = bitcoin.sign(tx, i, priv, amount=amount, native="p2wpkh")
+            if not sig:
+                assert False, msg
+        txid = jm_single().bc_interface.pushtx(tx.serialize())
+        assert txid
 
-def test_spend_then_rbf(setup_tx_creation):
-    """ Test plan: first, create a normal spend with
-    rbf enabled in direct_send, then broadcast but
-    do not mine a block. Then create a re-spend of
-    the same utxos with a higher fee and check
-    that broadcast succeeds.
-    """
-    # First phase: broadcast with RBF enabled.
-    #
-    # set a baseline feerate:
-    old_feerate = jm_single().config.get("POLICY", "tx_fees")
-    jm_single().config.set("POLICY", "tx_fees", "20000")
-    # set up a single wallet with some coins:
-    wallet_service = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet']
-    wallet_service.sync_wallet(fast=True)
-    # ensure selection of two utxos, doesn't really matter
-    # but a more general case than only one:
-    amount = 350000000
-    # destination doesn't matter; this is easiest:
-    destn = wallet_service.get_internal_addr(1)
-    # While `direct_send` usually encapsulates utxo selection
-    # for user, here we need to know what was chosen, hence
-    # we return the transaction object, not directly broadcast.
-    tx1 = direct_send(wallet_service, 0, [(destn, amount)],
-                      answeryes=True,
-                      return_transaction=True)
-    assert tx1
-    # record the utxos for reuse:
-    assert isinstance(tx1, bitcoin.CTransaction)
-    utxos_objs = (x.prevout for x in tx1.vin)
-    utxos = [(x.hash[::-1], x.n) for x in utxos_objs]
-    # in order to sign on those utxos, we need their script and value.
-    scrs = {}
-    vals = {}
-    for u, details in wallet_service.get_utxos_by_mixdepth()[0].items():
-        if u in utxos:
-            scrs[u] = details["script"]
-            vals[u] = details["value"]
-    assert len(scrs.keys()) == 2
-    assert len(vals.keys()) == 2
+    async def test_spend_then_rbf(self):
+        """ Test plan: first, create a normal spend with
+        rbf enabled in direct_send, then broadcast but
+        do not mine a block. Then create a re-spend of
+        the same utxos with a higher fee and check
+        that broadcast succeeds.
+        """
+        # First phase: broadcast with RBF enabled.
+        #
+        # set a baseline feerate:
+        old_feerate = jm_single().config.get("POLICY", "tx_fees")
+        jm_single().config.set("POLICY", "tx_fees", "20000")
+        # set up a single wallet with some coins:
+        wallets = await make_wallets(1, [[2, 0, 0, 0, 1]], 3)
+        wallet_service = wallets[0]['wallet']
+        await wallet_service.sync_wallet(fast=True)
+        # ensure selection of two utxos, doesn't really matter
+        # but a more general case than only one:
+        amount = 350000000
+        # destination doesn't matter; this is easiest:
+        destn = await wallet_service.get_internal_addr(1)
+        # While `direct_send` usually encapsulates utxo selection
+        # for user, here we need to know what was chosen, hence
+        # we return the transaction object, not directly broadcast.
+        tx1 = await direct_send(wallet_service, 0, [(destn, amount)],
+                                answeryes=True,
+                                return_transaction=True)
+        assert tx1
+        # record the utxos for reuse:
+        assert isinstance(tx1, bitcoin.CTransaction)
+        utxos_objs = (x.prevout for x in tx1.vin)
+        utxos = [(x.hash[::-1], x.n) for x in utxos_objs]
+        # in order to sign on those utxos, we need their script and value.
+        scrs = {}
+        vals = {}
+        for u, details in (
+                await wallet_service.get_utxos_by_mixdepth())[0].items():
+            if u in utxos:
+                scrs[u] = details["script"]
+                vals[u] = details["value"]
+        assert len(scrs.keys()) == 2
+        assert len(vals.keys()) == 2
 
-    # This will go to mempool but not get mined because
-    # we don't call `tick_forward_chain`.
-    push_succeed = jm_single().bc_interface.pushtx(tx1.serialize())
-    if push_succeed:
-        # mimics real operations with transaction monitor:
-        wallet_service.process_new_tx(tx1)
-    else:
-        assert False
+        # This will go to mempool but not get mined because
+        # we don't call `tick_forward_chain`.
+        push_succeed = jm_single().bc_interface.pushtx(tx1.serialize())
+        if push_succeed:
+            # mimics real operations with transaction monitor:
+            await wallet_service.process_new_tx(tx1)
+        else:
+            assert False
 
-    # Second phase: bump fee.
-    #
-    # we set a larger fee rate.
-    jm_single().config.set("POLICY", "tx_fees", "30000")
-    # just a different destination to avoid confusion:
-    destn2 = wallet_service.get_internal_addr(2)
-    # We reuse *both* utxos so total fees are comparable
-    # (modulo tiny 1 byte differences in signatures).
-    # Ordinary wallet operations would remove the first-spent utxos,
-    # so for now we build a PSBT using the code from #921 to select
-    # the same utxos (it could be done other ways).
-    # Then we broadcast the PSBT and check it is allowed
+        # Second phase: bump fee.
+        #
+        # we set a larger fee rate.
+        jm_single().config.set("POLICY", "tx_fees", "30000")
+        # just a different destination to avoid confusion:
+        destn2 = await wallet_service.get_internal_addr(2)
+        # We reuse *both* utxos so total fees are comparable
+        # (modulo tiny 1 byte differences in signatures).
+        # Ordinary wallet operations would remove the first-spent utxos,
+        # so for now we build a PSBT using the code from #921 to select
+        # the same utxos (it could be done other ways).
+        # Then we broadcast the PSBT and check it is allowed
 
-    # before constructing the outputs, we need a good fee estimate,
-    # using the bumped feerate:
-    fee = estimate_tx_fee(2, 2, wallet_service.get_txtype())
-    # reset the feerate:
-    total_input_val = sum(vals.values())
-    jm_single().config.set("POLICY", "tx_fees", old_feerate)
-    outs = [{"address": destn2, "value": 1000000},
-            {"address": wallet_service.get_internal_addr(0),
-             "value": total_input_val - 1000000 - fee}]
-    tx2 = bitcoin.mktx(utxos, outs, version=2,
-                       locktime=compute_tx_locktime())
-    spent_outs = []
-    for u in utxos:
-        spent_outs.append(bitcoin.CTxOut(nValue=vals[u],
-                            scriptPubKey=scrs[u]))
-    psbt_unsigned = wallet_service.create_psbt_from_tx(tx2,
-                                    spent_outs=spent_outs)
-    signresultandpsbt, err = wallet_service.sign_psbt(
-        psbt_unsigned.serialize(), with_sign_result=True)
-    assert not err
-    signresult, psbt_signed = signresultandpsbt
-    tx2_signed = psbt_signed.extract_transaction()
-    # the following assertion is sufficient, because
-    # tx broadcast would fail if the replacement were
-    # not allowed by Core:
-    assert jm_single().bc_interface.pushtx(tx2_signed.serialize())
+        # before constructing the outputs, we need a good fee estimate,
+        # using the bumped feerate:
+        fee = estimate_tx_fee(2, 2, wallet_service.get_txtype())
+        # reset the feerate:
+        total_input_val = sum(vals.values())
+        jm_single().config.set("POLICY", "tx_fees", old_feerate)
+        outs = [{"address": destn2, "value": 1000000},
+                {"address": await wallet_service.get_internal_addr(0),
+                 "value": total_input_val - 1000000 - fee}]
+        tx2 = bitcoin.mktx(utxos, outs, version=2,
+                           locktime=compute_tx_locktime())
+        spent_outs = []
+        for u in utxos:
+            spent_outs.append(bitcoin.CTxOut(nValue=vals[u],
+                                scriptPubKey=scrs[u]))
+        psbt_unsigned = await wallet_service.create_psbt_from_tx(
+            tx2, spent_outs=spent_outs)
+        signresultandpsbt, err = await wallet_service.sign_psbt(
+            psbt_unsigned.serialize(), with_sign_result=True)
+        assert not err
+        signresult, psbt_signed = signresultandpsbt
+        tx2_signed = psbt_signed.extract_transaction()
+        # the following assertion is sufficient, because
+        # tx broadcast would fail if the replacement were
+        # not allowed by Core:
+        assert jm_single().bc_interface.pushtx(tx2_signed.serialize())
 
-def test_spend_freeze_script(setup_tx_creation):
-    ensure_bip65_activated()
+    async def test_spend_freeze_script(self):
+        ensure_bip65_activated()
 
-    wallet_service = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet']
-    wallet_service.sync_wallet(fast=True)
+        wallets = await make_wallets(1, [[3, 0, 0, 0, 0]], 3)
+        wallet_service = wallets[0]['wallet']
+        await wallet_service.sync_wallet(fast=True)
 
-    mediantime = jm_single().bc_interface.get_best_block_median_time()
+        mediantime = jm_single().bc_interface.get_best_block_median_time()
 
-    timeoffset_success_tests = [(2, False), (-60*60*24*30, True), (60*60*24*30, False)]
+        timeoffset_success_tests = [(2, False), (-60*60*24*30, True), (60*60*24*30, False)]
 
-    for timeoffset, required_success in timeoffset_success_tests:
-        #generate keypair
-        priv = b"\xaa"*32 + b"\x01"
-        pub = bitcoin.privkey_to_pubkey(priv)
-        addr_locktime = mediantime + timeoffset
-        redeem_script = bitcoin.mk_freeze_script(pub, addr_locktime)
-        script_pub_key = bitcoin.redeem_script_to_p2wsh_script(redeem_script)
-        # cannot convert to address within wallet service, as not known
-        # to wallet; use engine directly:
-        addr = wallet_service._ENGINE.script_to_address(script_pub_key)
+        for timeoffset, required_success in timeoffset_success_tests:
+            #generate keypair
+            priv = b"\xaa"*32 + b"\x01"
+            pub = bitcoin.privkey_to_pubkey(priv)
+            addr_locktime = mediantime + timeoffset
+            redeem_script = bitcoin.mk_freeze_script(pub, addr_locktime)
+            script_pub_key = bitcoin.redeem_script_to_p2wsh_script(redeem_script)
+            # cannot convert to address within wallet service, as not known
+            # to wallet; use engine directly:
+            addr = wallet_service._ENGINE.script_to_address(script_pub_key)
 
-        #fund frozen funds address
-        amount = 100000000
-        funding_ins_full = wallet_service.select_utxos(0, amount)
-        funding_txid = make_sign_and_push(funding_ins_full, wallet_service, amount, output_addr=addr)
-        assert funding_txid
+            #fund frozen funds address
+            amount = 100000000
+            funding_ins_full = await wallet_service.select_utxos(0, amount)
+            funding_txid = await make_sign_and_push(
+                funding_ins_full, wallet_service, amount, output_addr=addr)
+            assert funding_txid
 
-        #spend frozen funds
-        frozen_in = (funding_txid, 0)
-        output_addr = wallet_service.get_internal_addr(1)
-        miner_fee = 5000
-        outs = [{'value': amount - miner_fee, 'address': output_addr}]
-        tx = bitcoin.mktx([frozen_in], outs, locktime=addr_locktime+1)
-        i = 0
-        sig, success = bitcoin.sign(tx, i, priv, amount=amount,
-                                    native=redeem_script)
-        assert success
-        push_success = jm_single().bc_interface.pushtx(tx.serialize())
-        assert push_success == required_success
+            #spend frozen funds
+            frozen_in = (funding_txid, 0)
+            output_addr = await wallet_service.get_internal_addr(1)
+            miner_fee = 5000
+            outs = [{'value': amount - miner_fee, 'address': output_addr}]
+            tx = bitcoin.mktx([frozen_in], outs, locktime=addr_locktime+1)
+            i = 0
+            sig, success = bitcoin.sign(tx, i, priv, amount=amount,
+                                        native=redeem_script)
+            assert success
+            push_success = jm_single().bc_interface.pushtx(tx.serialize())
+            assert push_success == required_success
 
 @pytest.fixture(scope="module")
 def setup_tx_creation():

test/jmclient/test_utxomanager.py

@@ -1,8 +1,13 @@
 
-from jmclient.wallet import UTXOManager
 from test_storage import MockStorage
 import pytest
+from _pytest.monkeypatch import MonkeyPatch
+from unittest import IsolatedAsyncioTestCase
+
+import jmclient  # install asyncioreactor
+from twisted.internet import reactor
 
+from jmclient.wallet import UTXOManager
 from jmclient import load_test_config
 import jmclient
 from commontest import DummyBlockchainInterface
@@ -12,111 +17,120 @@ def select(unspent, value):
     return unspent
 
 
-def test_utxomanager_persist(setup_env_nodeps):
-    """ Tests that the utxo manager's data is correctly
-    persisted and can be recreated from storage.
-    This persistence is currently only used for metadata
-    (specifically, disabling coins for coin control).
-    """
-
-    storage = MockStorage(None, 'wallet.jmdat', None, create=True)
-    UTXOManager.initialize(storage)
-    um = UTXOManager(storage, select)
-
-    txid = b'\x00' * UTXOManager.TXID_LEN
-    index = 0
-    path = (0,)
-    mixdepth = 0
-    value = 500
-
-    um.add_utxo(txid, index, path, value, mixdepth, 1)
-    um.add_utxo(txid, index+1, path, value, mixdepth+1, 2)
-    # the third utxo will be disabled and we'll check if
-    # the disablement persists in the storage across UM instances
-    um.add_utxo(txid, index+2, path, value, mixdepth+1, 3)
-    um.disable_utxo(txid, index+2)
-    um.save()
-
-    # Remove and recreate the UM from the same storage.
-
-    del um
-
-    um = UTXOManager(storage, select)
-
-    assert um.have_utxo(txid, index) == mixdepth
-    assert um.have_utxo(txid, index+1) == mixdepth + 1
-    # The third should not be registered as present given flag:
-    assert um.have_utxo(txid, index+2, include_disabled=False) == False
-    # check is_disabled works:
-    assert not um.is_disabled(txid, index)
-    assert not um.is_disabled(txid, index+1)
-    assert um.is_disabled(txid, index+2)
-    # check re-enabling works
-    um.enable_utxo(txid, index+2)
-    assert not um.is_disabled(txid, index+2)
-    um.disable_utxo(txid, index+2)
-
-    assert len(um.get_utxos_at_mixdepth(mixdepth)) == 1
-    assert len(um.get_utxos_at_mixdepth(mixdepth+1)) == 2
-    assert len(um.get_utxos_at_mixdepth(mixdepth+2)) == 0
-
-    assert um.get_balance_at_mixdepth(mixdepth) == value
-    assert um.get_balance_at_mixdepth(mixdepth+1) == value * 2
-
-    um.remove_utxo(txid, index, mixdepth)
-    assert um.have_utxo(txid, index) == False
-    # check that removing a utxo does not remove the metadata
-    um.remove_utxo(txid, index+2, mixdepth+1)
-    assert um.is_disabled(txid, index+2)
-
-    um.save()
-    del um
-
-    um = UTXOManager(storage, select)
-
-    assert um.have_utxo(txid, index) == False
-    assert um.have_utxo(txid, index+1) == mixdepth + 1
-
-    assert len(um.get_utxos_at_mixdepth(mixdepth)) == 0
-    assert len(um.get_utxos_at_mixdepth(mixdepth+1)) == 1
-
-    assert um.get_balance_at_mixdepth(mixdepth) == 0
-    assert um.get_balance_at_mixdepth(mixdepth+1) == value
-    assert um.get_balance_at_mixdepth(mixdepth+2) == 0
-
-
-def test_utxomanager_select(setup_env_nodeps):
-    storage = MockStorage(None, 'wallet.jmdat', None, create=True)
-    UTXOManager.initialize(storage)
-    um = UTXOManager(storage, select)
-
-    txid = b'\x00' * UTXOManager.TXID_LEN
-    index = 0
-    path = (0,)
-    mixdepth = 0
-    value = 500
-
-    um.add_utxo(txid, index, path, value, mixdepth, 100)
-
-    assert len(um.select_utxos(mixdepth, value)) == 1
-    assert len(um.select_utxos(mixdepth+1, value)) == 0
-
-    um.add_utxo(txid, index+1, path, value, mixdepth, None)
-    assert len(um.select_utxos(mixdepth, value)) == 2
-
-    # ensure that added utxos that are disabled do not
-    # get used by the selector
-    um.add_utxo(txid, index+2, path, value, mixdepth, 101)
-    um.disable_utxo(txid, index+2)
-    assert len(um.select_utxos(mixdepth, value)) == 2
-
-    # ensure that unconfirmed coins are not selected if
-    # dis-requested:
-    assert len(um.select_utxos(mixdepth, value, maxheight=105)) == 1
-
-
-@pytest.fixture
-def setup_env_nodeps(monkeypatch):
-    monkeypatch.setattr(jmclient.configure, 'get_blockchain_interface_instance',
-                        lambda x: DummyBlockchainInterface())
-    load_test_config()
+class AsyncioTestCase(IsolatedAsyncioTestCase):
+
+    def setUp(self):
+        jmclient.configure._get_bc_interface_instance_ = \
+            jmclient.configure.get_blockchain_interface_instance
+        monkeypatch = MonkeyPatch()
+        monkeypatch.setattr(jmclient.configure,
+                            'get_blockchain_interface_instance',
+                            lambda x: DummyBlockchainInterface())
+        load_test_config()
+
+    def tearDown(self):
+        monkeypatch = MonkeyPatch()
+        monkeypatch.setattr(jmclient.configure,
+                            'get_blockchain_interface_instance',
+                            jmclient.configure._get_bc_interface_instance_)
+
+    async def test_utxomanager_persist(self):
+        """ Tests that the utxo manager's data is correctly
+        persisted and can be recreated from storage.
+        This persistence is currently only used for metadata
+        (specifically, disabling coins for coin control).
+        """
+
+        storage = MockStorage(None, 'wallet.jmdat', None, create=True)
+        UTXOManager.initialize(storage)
+        um = UTXOManager(storage, select)
+
+        txid = b'\x00' * UTXOManager.TXID_LEN
+        index = 0
+        path = (0,)
+        mixdepth = 0
+        value = 500
+
+        um.add_utxo(txid, index, path, value, mixdepth, 1)
+        um.add_utxo(txid, index+1, path, value, mixdepth+1, 2)
+        # the third utxo will be disabled and we'll check if
+        # the disablement persists in the storage across UM instances
+        um.add_utxo(txid, index+2, path, value, mixdepth+1, 3)
+        um.disable_utxo(txid, index+2)
+        um.save()
+
+        # Remove and recreate the UM from the same storage.
+
+        del um
+
+        um = UTXOManager(storage, select)
+
+        assert um.have_utxo(txid, index) == mixdepth
+        assert um.have_utxo(txid, index+1) == mixdepth + 1
+        # The third should not be registered as present given flag:
+        assert um.have_utxo(txid, index+2, include_disabled=False) == False
+        # check is_disabled works:
+        assert not um.is_disabled(txid, index)
+        assert not um.is_disabled(txid, index+1)
+        assert um.is_disabled(txid, index+2)
+        # check re-enabling works
+        um.enable_utxo(txid, index+2)
+        assert not um.is_disabled(txid, index+2)
+        um.disable_utxo(txid, index+2)
+
+        assert len(await um.get_utxos_at_mixdepth(mixdepth)) == 1
+        assert len(await um.get_utxos_at_mixdepth(mixdepth+1)) == 2
+        assert len(await um.get_utxos_at_mixdepth(mixdepth+2)) == 0
+
+        assert um.get_balance_at_mixdepth(mixdepth) == value
+        assert um.get_balance_at_mixdepth(mixdepth+1) == value * 2
+
+        um.remove_utxo(txid, index, mixdepth)
+        assert um.have_utxo(txid, index) == False
+        # check that removing a utxo does not remove the metadata
+        um.remove_utxo(txid, index+2, mixdepth+1)
+        assert um.is_disabled(txid, index+2)
+
+        um.save()
+        del um
+
+        um = UTXOManager(storage, select)
+
+        assert um.have_utxo(txid, index) == False
+        assert um.have_utxo(txid, index+1) == mixdepth + 1
+
+        assert len(await um.get_utxos_at_mixdepth(mixdepth)) == 0
+        assert len(await um.get_utxos_at_mixdepth(mixdepth+1)) == 1
+
+        assert um.get_balance_at_mixdepth(mixdepth) == 0
+        assert um.get_balance_at_mixdepth(mixdepth+1) == value
+        assert um.get_balance_at_mixdepth(mixdepth+2) == 0
+
+    async def test_utxomanager_select(self):
+        storage = MockStorage(None, 'wallet.jmdat', None, create=True)
+        UTXOManager.initialize(storage)
+        um = UTXOManager(storage, select)
+
+        txid = b'\x00' * UTXOManager.TXID_LEN
+        index = 0
+        path = (0,)
+        mixdepth = 0
+        value = 500
+
+        um.add_utxo(txid, index, path, value, mixdepth, 100)
+
+        assert len(await um.select_utxos(mixdepth, value)) == 1
+        assert len(await um.select_utxos(mixdepth+1, value)) == 0
+
+        um.add_utxo(txid, index+1, path, value, mixdepth, None)
+        assert len(await um.select_utxos(mixdepth, value)) == 2
+
+        # ensure that added utxos that are disabled do not
+        # get used by the selector
+        um.add_utxo(txid, index+2, path, value, mixdepth, 101)
+        um.disable_utxo(txid, index+2)
+        assert len(await um.select_utxos(mixdepth, value)) == 2
+
+        # ensure that unconfirmed coins are not selected if
+        # dis-requested:
+        assert len(await um.select_utxos(mixdepth, value, maxheight=105)) == 1

test/jmclient/test_wallet_rpc.py

@@ -4,11 +4,13 @@ import functools
 import json
 import os
 
+import jmclient  # install asyncioreactor
+from twisted.internet import reactor
+
 import jwt
 import pytest
-from twisted.internet import reactor, defer, task
+from twisted.internet import defer, task
 from twisted.web.client import readBody, Headers
-from twisted.trial import unittest
 from autobahn.twisted.websocket import WebSocketClientFactory, \
     connectWS
 
@@ -26,7 +28,7 @@ from jmclient import (
     storage,
 )
 from jmclient.wallet_rpc import api_version_string, CJ_MAKER_RUNNING, CJ_NOT_RUNNING
-from commontest import make_wallets
+from commontest import make_wallets, TrialAsyncioTestCase
 from test_coinjoin import make_wallets_to_list, sync_wallets
 
 from test_websocket import ClientTProtocol, test_tx_hex_1, test_tx_hex_txid
@@ -45,7 +47,8 @@ class JMWalletDaemonT(JMWalletDaemon):
             return True
         return super().check_cookie(request, *args, **kwargs)
 
-class WalletRPCTestBase(object):
+
+class WalletRPCTestBase(TrialAsyncioTestCase):
     """ Base class for set up of tests of the
     Wallet RPC calls using the wallet_rpc.JMWalletDaemon service.
     """
@@ -62,7 +65,7 @@ class WalletRPCTestBase(object):
     # wallet type
     wallet_cls = SegwitWallet
 
-    def setUp(self):
+    async def asyncSetUp(self):
         load_test_config()
         self.clean_out_wallet_files()
         jm_single().bc_interface.tick_forward_chain_interval = 5
@@ -94,11 +97,12 @@ class WalletRPCTestBase(object):
         self.listener_rpc = r
         self.listener_ws = s
         wallet_structures = [self.wallet_structure] * 2
-        self.daemon.services["wallet"] = make_wallets_to_list(make_wallets(
+        wallets = await make_wallets(
             1, wallet_structures=[wallet_structures[0]],
-            mean_amt=self.mean_amt, wallet_cls=self.wallet_cls))[0]
+            mean_amt=self.mean_amt, wallet_cls=self.wallet_cls)
+        self.daemon.services["wallet"] = make_wallets_to_list(wallets)[0]
         jm_single().bc_interface.tickchain()
-        sync_wallets([self.daemon.services["wallet"]])
+        await sync_wallets([self.daemon.services["wallet"]])
         # dummy tx example to force a notification event:
         self.test_tx = CTransaction.deserialize(hextobin(test_tx_hex_1))
         # auth token is not set at the start
@@ -168,6 +172,7 @@ class WalletRPCTestBase(object):
 
     def tearDown(self):
         self.clean_out_wallet_files()
+        reactor.disconnectAll()
         for dc in reactor.getDelayedCalls():
             if not dc.cancelled:
                 dc.cancel()
@@ -198,7 +203,7 @@ class ClientNotifTestFactory(WebSocketClientFactory):
             self.callbackfn = kwargs.pop("callbackfn", None)
         super().__init__(*args, **kwargs)
 
-class TrialTestWRPC_WS(WalletRPCTestBase, unittest.TestCase):
+class TrialTestWRPC_WS(WalletRPCTestBase):
     """ class for testing websocket subscriptions/events etc.
     """
 
@@ -240,7 +245,7 @@ class TrialTestWRPC_WS(WalletRPCTestBase, unittest.TestCase):
         self.daemon.wss_factory.sendTxNotification(self.test_tx,
                                             test_tx_hex_txid)
 
-class TrialTestWRPC_FB(WalletRPCTestBaseFB, unittest.TestCase):
+class TrialTestWRPC_FB(WalletRPCTestBaseFB):
     @defer.inlineCallbacks
     def test_gettimelockaddress(self):
         self.daemon.auth_disabled = True
@@ -294,7 +299,7 @@ class TrialTestWRPC_FB(WalletRPCTestBaseFB, unittest.TestCase):
         # be MAKER_RUNNING since no non-TL-type coin existed:
         assert self.daemon.coinjoin_state == CJ_NOT_RUNNING
 
-class TrialTestWRPC_DisplayWallet(WalletRPCTestBase, unittest.TestCase):
+class TrialTestWRPC_DisplayWallet(WalletRPCTestBase):
 
     @defer.inlineCallbacks
     def do_session_request(self, agent, addr, handler=None, token=None):
@@ -749,7 +754,7 @@ class TrialTestWRPC_DisplayWallet(WalletRPCTestBase, unittest.TestCase):
         assert json_body["seedphrase"]
 
 
-class TrialTestWRPC_JWT(WalletRPCTestBase, unittest.TestCase):
+class TrialTestWRPC_JWT(WalletRPCTestBase):
     @defer.inlineCallbacks
     def do_request(self, agent, method, addr, body, handler, token):
         headers = Headers({"Authorization": ["Bearer " + token]})

test/jmclient/test_wallets.py

@@ -7,6 +7,8 @@ import binascii
 from commontest import create_wallet_for_sync, make_sign_and_push
 import json
 
+from unittest import IsolatedAsyncioTestCase
+
 import pytest
 from jmbase import get_log, hextobin
 from jmclient import (
@@ -20,78 +22,23 @@ testdir = os.path.dirname(os.path.realpath(__file__))
 log = get_log()
 
 
-def do_tx(wallet_service, amount):
-    ins_full = wallet_service.select_utxos(0, amount)
-    cj_addr = wallet_service.get_internal_addr(1)
-    change_addr = wallet_service.get_internal_addr(0)
+async def do_tx(wallet_service, amount):
+    ins_full = await wallet_service.select_utxos(0, amount)
+    cj_addr = await wallet_service.get_internal_addr(1)
+    change_addr = await wallet_service.get_internal_addr(0)
     wallet_service.save_wallet()
-    txid = make_sign_and_push(ins_full,
-                              wallet_service,
-                              amount,
-                              output_addr=cj_addr,
-                              change_addr=change_addr,
-                              estimate_fee=True)
+    txid = await make_sign_and_push(ins_full,
+                                    wallet_service,
+                                    amount,
+                                    output_addr=cj_addr,
+                                    change_addr=change_addr,
+                                    estimate_fee=True)
     assert txid
     time.sleep(2)  #blocks
     wallet_service.sync_unspent()
     return txid
 
 
-def test_query_utxo_set(setup_wallets):
-    load_test_config()
-    jm_single().bc_interface.tick_forward_chain_interval = 1
-    wallet_service = create_wallet_for_sync([2, 3, 0, 0, 0],
-                                    ["wallet4utxo.json", "4utxo", [2, 3]])
-    wallet_service.sync_wallet(fast=True)
-    txid = do_tx(wallet_service, 90000000)
-    txid2 = do_tx(wallet_service, 20000000)
-    print("Got txs: ", txid, txid2)
-    res1 = jm_single().bc_interface.query_utxo_set(
-        (txid, 0), include_mempool=True)
-    res2 = jm_single().bc_interface.query_utxo_set(
-        [(txid, 0), (txid2, 1)],
-        includeconfs=True, include_mempool=True)
-    assert len(res1) == 1
-    assert len(res2) == 2
-    assert all([x in res1[0] for x in ['script', 'value']])
-    assert not 'confirms' in res1[0]
-    assert 'confirms' in res2[0]
-    assert 'confirms' in res2[1]
-    res3 = jm_single().bc_interface.query_utxo_set((b"\xee" * 32, 25))
-    assert res3 == [None]
-
-
-"""Purely blockchaininterface related error condition tests"""
-
-
-def test_wrong_network_bci(setup_wallets):
-    rpc = jm_single().bc_interface.jsonRpc
-    with pytest.raises(Exception) as e_info:
-        x = BitcoinCoreInterface(rpc, 'mainnet')
-
-
-def test_pushtx_errors(setup_wallets):
-    """Ensure pushtx fails return False
-    """
-    badtx = b"\xaa\xaa"
-    assert not jm_single().bc_interface.pushtx(badtx)
-    #Break the authenticated jsonrpc and try again
-    jm_single().bc_interface.jsonRpc.port = 18333
-    assert not jm_single().bc_interface.pushtx(hextobin(t_raw_signed_tx))
-    #rebuild a valid jsonrpc inside the bci
-    load_test_config()
-
-
-"""Tests mainly for wallet.py"""
-
-
-def test_absurd_fee(setup_wallets):
-    jm_single().config.set("POLICY", "absurd_fee_per_kb", "1000")
-    with pytest.raises(ValueError) as e_info:
-        estimate_tx_fee(10, 2)
-    load_test_config()
-
-
 def check_bip39_case(vectors, language="english"):
     mnemo = Mnemonic(language)
     for v in vectors:
@@ -102,21 +49,75 @@ def check_bip39_case(vectors, language="english"):
         assert v[1] == code
         assert v[2] == seed
 
-"""
-Sanity check of basic bip39 functionality for 12 words seed, copied from
-https://github.com/trezor/python-mnemonic/blob/master/tests/test_mnemonic.py
-"""
-def test_bip39_vectors(setup_wallets):
-    with open(os.path.join(testdir, 'bip39vectors.json'), 'r') as f:
-        vectors_full = json.load(f)
-    vectors = vectors_full['english']
-    #default from-file cases use passphrase 'TREZOR'; TODO add other
-    #extensions, but note there is coverage of that in the below test
-    for v in vectors:
-        v.append("TREZOR")
-    #12 word seeds only
-    vectors = filter(lambda x: len(x[1].split())==12, vectors)
-    check_bip39_case(vectors)
+
+@pytest.mark.usefixtures("setup_wallets")
+class AsyncioTestCase(IsolatedAsyncioTestCase):
+
+    async def test_query_utxo_set(self):
+        load_test_config()
+        jm_single().bc_interface.tick_forward_chain_interval = 1
+        wallet_service = await create_wallet_for_sync([2, 3, 0, 0, 0],
+                                        ["wallet4utxo.json", "4utxo", [2, 3]])
+        await wallet_service.sync_wallet(fast=True)
+        txid = await do_tx(wallet_service, 90000000)
+        txid2 = await do_tx(wallet_service, 20000000)
+        print("Got txs: ", txid, txid2)
+        res1 = jm_single().bc_interface.query_utxo_set(
+            (txid, 0), include_mempool=True)
+        res2 = jm_single().bc_interface.query_utxo_set(
+            [(txid, 0), (txid2, 1)],
+            includeconfs=True, include_mempool=True)
+        assert len(res1) == 1
+        assert len(res2) == 2
+        assert all([x in res1[0] for x in ['script', 'value']])
+        assert not 'confirms' in res1[0]
+        assert 'confirms' in res2[0]
+        assert 'confirms' in res2[1]
+        res3 = jm_single().bc_interface.query_utxo_set((b"\xee" * 32, 25))
+        assert res3 == [None]
+
+    """Purely blockchaininterface related error condition tests"""
+    async def test_wrong_network_bci(self):
+        rpc = jm_single().bc_interface.jsonRpc
+        with pytest.raises(Exception) as e_info:
+            x = BitcoinCoreInterface(rpc, 'mainnet')
+
+    async def test_pushtx_errors(self):
+        """Ensure pushtx fails return False
+        """
+        badtx = b"\xaa\xaa"
+        assert not jm_single().bc_interface.pushtx(badtx)
+        #Break the authenticated jsonrpc and try again
+        jm_single().bc_interface.jsonRpc.port = 18333
+        assert not jm_single().bc_interface.pushtx(hextobin(t_raw_signed_tx))
+        #rebuild a valid jsonrpc inside the bci
+        load_test_config()
+
+    """Tests mainly for wallet.py"""
+    async def test_absurd_fee(self):
+        jm_single().config.set("POLICY", "absurd_fee_per_kb", "1000")
+        with pytest.raises(ValueError) as e_info:
+            estimate_tx_fee(10, 2)
+        load_test_config()
+
+    """
+    Sanity check of basic bip39 functionality for 12 words seed, copied from
+    https://github.com/trezor/python-mnemonic/blob/master/tests/test_mnemonic.py
+    """
+    async def test_bip39_vectors(self):
+        with open(os.path.join(testdir, 'bip39vectors.json'), 'r') as f:
+            vectors_full = json.load(f)
+        data = vectors_full['english']
+        #default from-file cases use passphrase 'TREZOR'; TODO add other
+        #extensions, but note there is coverage of that in the below test
+        for d in data:
+            d.append("TREZOR")
+        vectors = []
+        for d in data:
+            vectors.append(tuple(d))
+        #12 word seeds only
+        vectors = filter(lambda x: len(x[1].split())==12, vectors)
+        check_bip39_case(vectors)
 
 
 @pytest.fixture(scope="module")

test/jmclient/test_walletservice.py

@@ -2,6 +2,9 @@
 
 import os
 import pytest
+
+from unittest import IsolatedAsyncioTestCase
+
 from jmbase import get_log
 from jmclient import load_test_config, jm_single, \
      WalletService
@@ -16,57 +19,62 @@ log = get_log()
 def set_freeze_reuse_config(x):
     jm_single().config.set('POLICY', 'max_sats_freeze_reuse', str(x))
 
-def try_address_reuse(wallet_service, idx, funding_amt, config_threshold,
-                      expected_final_balance):
+async def try_address_reuse(wallet_service, idx, funding_amt, config_threshold,
+                            expected_final_balance):
     set_freeze_reuse_config(config_threshold)
     # check that below the threshold on the same address is not allowed:
-    fund_wallet_addr(wallet_service.wallet, wallet_service.get_addr(0, 1, idx),
+    fund_wallet_addr(wallet_service.wallet,
+                     await wallet_service.get_addr(0, 1, idx),
                      value_btc=funding_amt)
-    wallet_service.transaction_monitor()
+    await wallet_service.transaction_monitor()
     balances = wallet_service.get_balance_by_mixdepth()
     assert balances[0] == expected_final_balance
 
-def test_address_reuse_freezing(setup_walletservice):
-    """ Creates a WalletService on a pre-populated wallet,
-    and sets different values of the config var
-    'max_sats_freeze_reuse' then adds utxos to different
-    already used addresses to check that they are frozen or
-    not as appropriate.
-    Note that to avoid a twisted main loop the WalletService is
-    not actually started, but the transaction_monitor is triggered
-    manually (which executes the same code).
-    A custom test version of the reuse warning callback is added
-    and to check correct function, we check that this callback is
-    called, and that the balance available in the mixdepth correctly
-    reflects the usage pattern and freeze policy.
-    """
-    context = {'cb_called': 0}
-    def reuse_callback(utxostr):
-        context['cb_called'] += 1
-    # we must fund after initial sync (for imports), hence
-    # "populated" with no coins
-    wallet = get_populated_wallet(num=0)
-    wallet_service = WalletService(wallet)
-    wallet_service.set_autofreeze_warning_cb(reuse_callback)
-    sync_test_wallet(True, wallet_service)
-    for i in range(3):
-        fund_wallet_addr(wallet_service.wallet,
-                         wallet_service.get_addr(0, 1, i))
-    # manually force the wallet service to see the new utxos:
-    wallet_service.transaction_monitor()
 
-    # check that with default status any reuse is blocked:
-    try_address_reuse(wallet_service, 0, 1, -1, 3 * 10**8)
-    assert context['cb_called'] == 1, "Failed to trigger freeze callback"
+@pytest.mark.usefixtures("setup_walletservice")
+class AsyncioTestCase(IsolatedAsyncioTestCase):
 
-    # check that above the threshold is allowed (1 sat less than funding)
-    try_address_reuse(wallet_service, 1, 1, 99999999, 4 * 10**8)
-    assert context['cb_called'] == 1, "Incorrectly triggered freeze callback"
+    async def test_address_reuse_freezing(self):
+        """ Creates a WalletService on a pre-populated wallet,
+        and sets different values of the config var
+        'max_sats_freeze_reuse' then adds utxos to different
+        already used addresses to check that they are frozen or
+        not as appropriate.
+        Note that to avoid a twisted main loop the WalletService is
+        not actually started, but the transaction_monitor is triggered
+        manually (which executes the same code).
+        A custom test version of the reuse warning callback is added
+        and to check correct function, we check that this callback is
+        called, and that the balance available in the mixdepth correctly
+        reflects the usage pattern and freeze policy.
+        """
+        context = {'cb_called': 0}
+        def reuse_callback(utxostr):
+            context['cb_called'] += 1
+        # we must fund after initial sync (for imports), hence
+        # "populated" with no coins
+        wallet = await get_populated_wallet(num=0)
+        wallet_service = WalletService(wallet)
+        wallet_service.set_autofreeze_warning_cb(reuse_callback)
+        await sync_test_wallet(True, wallet_service)
+        for i in range(3):
+            fund_wallet_addr(wallet_service.wallet,
+                             await wallet_service.get_addr(0, 1, i))
+        # manually force the wallet service to see the new utxos:
+        await wallet_service.transaction_monitor()
 
-    # check that below the threshold on the same address is not allowed:
-    try_address_reuse(wallet_service, 1, 0.99999998, 99999999, 4 * 10**8)
-    # note can be more than 1 extra call here, somewhat suboptimal:
-    assert context['cb_called'] > 1, "Failed to trigger freeze callback"
+        # check that with default status any reuse is blocked:
+        await try_address_reuse(wallet_service, 0, 1, -1, 3 * 10**8)
+        assert context['cb_called'] == 1, "Failed to trigger freeze callback"
+
+        # check that above the threshold is allowed (1 sat less than funding)
+        await try_address_reuse(wallet_service, 1, 1, 99999999, 4 * 10**8)
+        assert context['cb_called'] == 1, "Incorrectly triggered freeze callback"
+
+        # check that below the threshold on the same address is not allowed:
+        await try_address_reuse(wallet_service, 1, 0.99999998, 99999999, 4 * 10**8)
+        # note can be more than 1 extra call here, somewhat suboptimal:
+        assert context['cb_called'] > 1, "Failed to trigger freeze callback"
 
 
 @pytest.fixture(scope='module')

test/jmclient/test_walletutils.py

@@ -1,4 +1,8 @@
 import pytest
+from unittest import IsolatedAsyncioTestCase
+
+from unittest_parametrize import parametrize, ParametrizedTestCase
+
 from jmbitcoin import select_chain_params
 from jmclient import (SegwitLegacyWallet, SegwitWallet, get_network,
                       jm_single, VolatileStorage, load_test_config)
@@ -9,103 +13,107 @@ from jmclient.wallet_utils import (bip32pathparse, WalletView,
 pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
 
 
-# The below signatures have all been verified against Electrum 4.0.9:
-@pytest.mark.parametrize('seed, hdpath, walletcls, message, sig, addr', [
-    [b"\x01"*16, "m/84'/0'/0'/0/0", SegwitWallet, "hello",
-     "IOLk6ct/8aKtvTNnEAc+xojIWKv5FOwnzHGcnHkTJJwRBAyhrZ2ZyB0Re+dKS4SEav3qgjQeqMYRm+7mHi4sFKA=",
-     "bc1qq53d9372u8d50jfd5agq9zv7m7zdnzwducuqgz"],
-    [b"\x01"*16, "m/49'/0'/0'/0/0", SegwitLegacyWallet, "hello",
-     "HxVaQuXyBpl1UKutiusJjeLfKHwJYBzUiWuu6hEbmNFeSZGt/mbXKJ071ANR1gvdICbS/AnEa2RKDq9xMd/nU8s=",
-     "3AdTcqdoLHFGNq6znkahJDT41u65HAwiRv"],
-    [b"\x02"*16, "m/84'/0'/2'/1/0", SegwitWallet, "sign me",
-     "IA/V5DG7u108aNzCnpNPHqfrJAL8pF4GQ0sSqpf4Vlg5UWizauXzh2KskoD6Usl13hzqXBi4XDXl7Xxo5z6M298=",
-     "bc1q8mm69xs740sr0l2umrhmpl4ewhxfudxg2zvjw5"],
-    [b"\x02"*16, "m/49'/0'/2'/1/0", SegwitLegacyWallet, "sign me",
-     "H4cAtoE+zL+Mr+U8jm9DiYxZlym5xeZM3mcgymLz+TF4YYr4lgnM8qTZhFwlK4izcPaLuF27LFEoGJ/ltleIHUI=",
-     "3Qan1D4Vcy1yMGHfR9j7szDuC8QxSFVScA"],
-])
-def test_signmessage(seed, hdpath, walletcls, message, sig, addr):
-    load_test_config()
-    jm_single().config.set('BLOCKCHAIN', 'network', 'mainnet')
-    select_chain_params("bitcoin/mainnet")
-    storage = VolatileStorage()
-    walletcls.initialize(
-        storage, get_network(), entropy=seed, max_mixdepth=3)
-    wallet = walletcls(storage)
-    s, m, a = wallet_signmessage(wallet, hdpath, message,
-                                        out_str=False)
-    assert (s, m, a) == (sig, message, addr)
-    jm_single().config.set("BLOCKCHAIN", "network", "testnet")
-    select_chain_params("bitcoin/regtest")
+class AsyncioTestCase(IsolatedAsyncioTestCase, ParametrizedTestCase):
 
-def test_bip32_pathparse():
-    assert bip32pathparse("m/2/1/0017")
-    assert not bip32pathparse("n/1/1/1/1")
-    assert bip32pathparse("m/0/1'/100'/3'/2/2/21/004/005")
-    assert not bip32pathparse("m/0/0/00k")
+    # The below signatures have all been verified against Electrum 4.0.9:
+    @parametrize(
+        'seed, hdpath, walletcls, message, sig, addr',
+        [
+            (b"\x01"*16, "m/84'/0'/0'/0/0", SegwitWallet, "hello",
+             "IOLk6ct/8aKtvTNnEAc+xojIWKv5FOwnzHGcnHkTJJwRBAyhrZ2ZyB0Re+dKS4SEav3qgjQeqMYRm+7mHi4sFKA=",
+             "bc1qq53d9372u8d50jfd5agq9zv7m7zdnzwducuqgz"),
+            (b"\x01"*16, "m/49'/0'/0'/0/0", SegwitLegacyWallet, "hello",
+             "HxVaQuXyBpl1UKutiusJjeLfKHwJYBzUiWuu6hEbmNFeSZGt/mbXKJ071ANR1gvdICbS/AnEa2RKDq9xMd/nU8s=",
+             "3AdTcqdoLHFGNq6znkahJDT41u65HAwiRv"),
+            (b"\x02"*16, "m/84'/0'/2'/1/0", SegwitWallet, "sign me",
+             "IA/V5DG7u108aNzCnpNPHqfrJAL8pF4GQ0sSqpf4Vlg5UWizauXzh2KskoD6Usl13hzqXBi4XDXl7Xxo5z6M298=",
+             "bc1q8mm69xs740sr0l2umrhmpl4ewhxfudxg2zvjw5"),
+            (b"\x02"*16, "m/49'/0'/2'/1/0", SegwitLegacyWallet, "sign me",
+             "H4cAtoE+zL+Mr+U8jm9DiYxZlym5xeZM3mcgymLz+TF4YYr4lgnM8qTZhFwlK4izcPaLuF27LFEoGJ/ltleIHUI=",
+             "3Qan1D4Vcy1yMGHfR9j7szDuC8QxSFVScA"),
+        ])
+    async def test_signmessage(self, seed, hdpath, walletcls, message, sig, addr):
+        load_test_config()
+        jm_single().config.set('BLOCKCHAIN', 'network', 'mainnet')
+        select_chain_params("bitcoin/mainnet")
+        storage = VolatileStorage()
+        walletcls.initialize(
+            storage, get_network(), entropy=seed, max_mixdepth=3)
+        wallet = walletcls(storage)
+        await wallet.async_init(storage)
+        s, m, a = await wallet_signmessage(
+            wallet, hdpath, message, out_str=False)
+        assert (s, m, a) == (sig, message, addr)
+        jm_single().config.set("BLOCKCHAIN", "network", "testnet")
+        select_chain_params("bitcoin/regtest")
 
+    async def test_bip32_pathparse(self):
+        assert bip32pathparse("m/2/1/0017")
+        assert not bip32pathparse("n/1/1/1/1")
+        assert bip32pathparse("m/0/1'/100'/3'/2/2/21/004/005")
+        assert not bip32pathparse("m/0/0/00k")
 
-def test_walletview():
-    rootpath = "m/0"
-    walletbranch = 0
-    accounts = range(3)
-    acctlist = []
-    for a in accounts:
-        branches = []
-        for address_type in range(2):
-            entries = []
-            for i in range(4):
-                entries.append(WalletViewEntry(rootpath, a, address_type,
-                    i, "DUMMYADDRESS" + str(i+a), [i*10000000, i*10000000]))
-            branches.append(WalletViewBranch(rootpath, a, address_type,
-                branchentries=entries,
-                xpub="xpubDUMMYXPUB" + str(a + address_type)))
-        acctlist.append(WalletViewAccount(rootpath, a, branches=branches))
-    wallet = WalletView(rootpath + "/" + str(walletbranch),
-        accounts=acctlist)
-    assert(wallet.serialize() == (
-        'JM wallet\n'
-        'mixdepth\t0\n'
-        'external addresses\tm/0\txpubDUMMYXPUB0\n'
-        'm/0                 \tDUMMYADDRESS0\t0.00000000\tnew\t\t\n'
-        'm/0                 \tDUMMYADDRESS1\t0.10000000\tnew\t\t\n'
-        'm/0                 \tDUMMYADDRESS2\t0.20000000\tnew\t\t\n'
-        'm/0                 \tDUMMYADDRESS3\t0.30000000\tnew\t\t\n'
-        'Balance:\t0.60000000\n'
-        'internal addresses\tm/0\txpubDUMMYXPUB1\n'
-        'm/0                 \tDUMMYADDRESS0\t0.00000000\tnew\t\t\n'
-        'm/0                 \tDUMMYADDRESS1\t0.10000000\tnew\t\t\n'
-        'm/0                 \tDUMMYADDRESS2\t0.20000000\tnew\t\t\n'
-        'm/0                 \tDUMMYADDRESS3\t0.30000000\tnew\t\t\n'
-        'Balance:\t0.60000000\n'
-        'Balance for mixdepth 0:\t1.20000000\n'
-        'mixdepth\t1\n'
-        'external addresses\tm/0\txpubDUMMYXPUB1\n'
-        'm/0                 \tDUMMYADDRESS1\t0.00000000\tnew\t\t\n'
-        'm/0                 \tDUMMYADDRESS2\t0.10000000\tnew\t\t\n'
-        'm/0                 \tDUMMYADDRESS3\t0.20000000\tnew\t\t\n'
-        'm/0                 \tDUMMYADDRESS4\t0.30000000\tnew\t\t\n'
-        'Balance:\t0.60000000\n'
-        'internal addresses\tm/0\txpubDUMMYXPUB2\n'
-        'm/0                 \tDUMMYADDRESS1\t0.00000000\tnew\t\t\n'
-        'm/0                 \tDUMMYADDRESS2\t0.10000000\tnew\t\t\n'
-        'm/0                 \tDUMMYADDRESS3\t0.20000000\tnew\t\t\n'
-        'm/0                 \tDUMMYADDRESS4\t0.30000000\tnew\t\t\n'
-        'Balance:\t0.60000000\n'
-        'Balance for mixdepth 1:\t1.20000000\n'
-        'mixdepth\t2\n'
-        'external addresses\tm/0\txpubDUMMYXPUB2\n'
-        'm/0                 \tDUMMYADDRESS2\t0.00000000\tnew\t\t\n'
-        'm/0                 \tDUMMYADDRESS3\t0.10000000\tnew\t\t\n'
-        'm/0                 \tDUMMYADDRESS4\t0.20000000\tnew\t\t\n'
-        'm/0                 \tDUMMYADDRESS5\t0.30000000\tnew\t\t\n'
-        'Balance:\t0.60000000\n'
-        'internal addresses\tm/0\txpubDUMMYXPUB3\n'
-        'm/0                 \tDUMMYADDRESS2\t0.00000000\tnew\t\t\n'
-        'm/0                 \tDUMMYADDRESS3\t0.10000000\tnew\t\t\n'
-        'm/0                 \tDUMMYADDRESS4\t0.20000000\tnew\t\t\n'
-        'm/0                 \tDUMMYADDRESS5\t0.30000000\tnew\t\t\n'
-        'Balance:\t0.60000000\n'
-        'Balance for mixdepth 2:\t1.20000000\n'
-        'Total balance:\t3.60000000'))
+    async def test_walletview(self):
+        rootpath = "m/0"
+        walletbranch = 0
+        accounts = range(3)
+        acctlist = []
+        for a in accounts:
+            branches = []
+            for address_type in range(2):
+                entries = []
+                for i in range(4):
+                    entries.append(WalletViewEntry(rootpath, a, address_type,
+                        i, "DUMMYADDRESS" + str(i+a), [i*10000000, i*10000000]))
+                branches.append(WalletViewBranch(rootpath, a, address_type,
+                    branchentries=entries,
+                    xpub="xpubDUMMYXPUB" + str(a + address_type)))
+            acctlist.append(WalletViewAccount(rootpath, a, branches=branches))
+        wallet = WalletView(rootpath + "/" + str(walletbranch),
+            accounts=acctlist)
+        assert(wallet.serialize() == (
+            'JM wallet\n'
+            'mixdepth\t0\n'
+            'external addresses\tm/0\txpubDUMMYXPUB0\n'
+            'm/0                 \tDUMMYADDRESS0\t0.00000000\tnew\t\t\n'
+            'm/0                 \tDUMMYADDRESS1\t0.10000000\tnew\t\t\n'
+            'm/0                 \tDUMMYADDRESS2\t0.20000000\tnew\t\t\n'
+            'm/0                 \tDUMMYADDRESS3\t0.30000000\tnew\t\t\n'
+            'Balance:\t0.60000000\n'
+            'internal addresses\tm/0\txpubDUMMYXPUB1\n'
+            'm/0                 \tDUMMYADDRESS0\t0.00000000\tnew\t\t\n'
+            'm/0                 \tDUMMYADDRESS1\t0.10000000\tnew\t\t\n'
+            'm/0                 \tDUMMYADDRESS2\t0.20000000\tnew\t\t\n'
+            'm/0                 \tDUMMYADDRESS3\t0.30000000\tnew\t\t\n'
+            'Balance:\t0.60000000\n'
+            'Balance for mixdepth 0:\t1.20000000\n'
+            'mixdepth\t1\n'
+            'external addresses\tm/0\txpubDUMMYXPUB1\n'
+            'm/0                 \tDUMMYADDRESS1\t0.00000000\tnew\t\t\n'
+            'm/0                 \tDUMMYADDRESS2\t0.10000000\tnew\t\t\n'
+            'm/0                 \tDUMMYADDRESS3\t0.20000000\tnew\t\t\n'
+            'm/0                 \tDUMMYADDRESS4\t0.30000000\tnew\t\t\n'
+            'Balance:\t0.60000000\n'
+            'internal addresses\tm/0\txpubDUMMYXPUB2\n'
+            'm/0                 \tDUMMYADDRESS1\t0.00000000\tnew\t\t\n'
+            'm/0                 \tDUMMYADDRESS2\t0.10000000\tnew\t\t\n'
+            'm/0                 \tDUMMYADDRESS3\t0.20000000\tnew\t\t\n'
+            'm/0                 \tDUMMYADDRESS4\t0.30000000\tnew\t\t\n'
+            'Balance:\t0.60000000\n'
+            'Balance for mixdepth 1:\t1.20000000\n'
+            'mixdepth\t2\n'
+            'external addresses\tm/0\txpubDUMMYXPUB2\n'
+            'm/0                 \tDUMMYADDRESS2\t0.00000000\tnew\t\t\n'
+            'm/0                 \tDUMMYADDRESS3\t0.10000000\tnew\t\t\n'
+            'm/0                 \tDUMMYADDRESS4\t0.20000000\tnew\t\t\n'
+            'm/0                 \tDUMMYADDRESS5\t0.30000000\tnew\t\t\n'
+            'Balance:\t0.60000000\n'
+            'internal addresses\tm/0\txpubDUMMYXPUB3\n'
+            'm/0                 \tDUMMYADDRESS2\t0.00000000\tnew\t\t\n'
+            'm/0                 \tDUMMYADDRESS3\t0.10000000\tnew\t\t\n'
+            'm/0                 \tDUMMYADDRESS4\t0.20000000\tnew\t\t\n'
+            'm/0                 \tDUMMYADDRESS5\t0.30000000\tnew\t\t\n'
+            'Balance:\t0.60000000\n'
+            'Balance for mixdepth 2:\t1.20000000\n'
+            'Total balance:\t3.60000000'))
 

test/jmclient/test_websocket.py

@@ -99,8 +99,10 @@ class WebsocketTestBase(object):
                                 test_tx_hex_txid)
 
     def tearDown(self):
+        reactor.disconnectAll()
         for dc in reactor.getDelayedCalls():
-            dc.cancel()
+            if not dc.cancelled:
+                dc.cancel()
         self.client_connector.disconnect()
         return self.stopListening()
 

test/jmclient/test_yieldgenerator.py

@@ -6,6 +6,7 @@ from jmbitcoin import CMutableTxOut, CMutableTransaction
 from jmclient import load_test_config, jm_single,\
     SegwitLegacyWallet, VolatileStorage, YieldGeneratorBasic, \
     get_network, WalletService
+from commontest import TrialAsyncioTestCase
 
 pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
 
@@ -23,15 +24,19 @@ class CustomUtxoWallet(SegwitLegacyWallet):
 
         load_test_config()
 
-        storage = VolatileStorage()
+        self._storage = storage = VolatileStorage()
+        self.balances = balances
         super().initialize(storage, get_network(), max_mixdepth=len(balances)-1)
         super().__init__(storage)
 
-        for m, b in enumerate(balances):
-            self.add_utxo_at_mixdepth(m, b)
+    async def async_init(self, storage, **kwargs):
+        await super().async_init(storage)
+        for m, b in enumerate(self.balances):
+            await self.add_utxo_at_mixdepth(m, b)
 
-    def add_utxo_at_mixdepth(self, mixdepth, balance):
-        txout = CMutableTxOut(balance, self.get_internal_script(mixdepth))
+    async def add_utxo_at_mixdepth(self, mixdepth, balance):
+        txout = CMutableTxOut(
+            balance, await self.get_internal_script(mixdepth))
         tx = CMutableTransaction()
         tx.vout = [txout]
         # (note: earlier requirement that txid be generated uniquely is now
@@ -46,13 +51,14 @@ class CustomUtxoWallet(SegwitLegacyWallet):
             assert self.get_addr_mixdepth(u['address']) == expected
 
 
-def create_yg_basic(balances, txfee_contribution=0, cjfee_a=0, cjfee_r=0,
-                    ordertype='swabsoffer', minsize=0):
+async def create_yg_basic(balances, txfee_contribution=0, cjfee_a=0, cjfee_r=0,
+                          ordertype='swabsoffer', minsize=0):
     """Constructs a YieldGeneratorBasic instance with a fake wallet.  The
     wallet will have the given balances at mixdepths, and the offer params
     will be set as given here."""
 
     wallet = CustomUtxoWallet(balances)
+    await wallet.async_init(wallet._storage)
     offerconfig = (txfee_contribution, cjfee_a, cjfee_r, ordertype, minsize,
         None, None, None)
 
@@ -67,17 +73,18 @@ def create_yg_basic(balances, txfee_contribution=0, cjfee_a=0, cjfee_r=0,
     return yg
 
 
-class CreateMyOrdersTests(unittest.TestCase):
+class CreateMyOrdersTests(TrialAsyncioTestCase):
     """Unit tests for YieldGeneratorBasic.create_my_orders."""
 
-    def test_no_coins(self):
-        yg = create_yg_basic([0] * 3)
+    async def test_no_coins(self):
+        yg = await create_yg_basic([0] * 3)
         self.assertEqual(yg.create_my_orders(), [])
 
-    def test_abs_fee(self):
+    async def test_abs_fee(self):
         jm_single().DUST_THRESHOLD = 10
-        yg = create_yg_basic([0, 2000000, 1000000], txfee_contribution=1000,
-                cjfee_a=10, ordertype='swabsoffer', minsize=100000)
+        yg = await create_yg_basic(
+            [0, 2000000, 1000000], txfee_contribution=1000,
+            cjfee_a=10, ordertype='swabsoffer', minsize=100000)
         self.assertEqual(yg.create_my_orders(), [
           {'oid': 0,
            'ordertype': 'swabsoffer',
@@ -87,10 +94,11 @@ class CreateMyOrdersTests(unittest.TestCase):
            'cjfee': '1010'},
         ])
 
-    def test_rel_fee(self):
+    async def test_rel_fee(self):
         jm_single().DUST_THRESHOLD = 10
-        yg = create_yg_basic([0, 2000000, 1000000], txfee_contribution=1000,
-                cjfee_r=0.1, ordertype='sw0reloffer', minsize=10)
+        yg = await create_yg_basic(
+            [0, 2000000, 1000000], txfee_contribution=1000,
+            cjfee_r=0.1, ordertype='sw0reloffer', minsize=10)
         self.assertEqual(yg.create_my_orders(), [
           {'oid': 0,
            'ordertype': 'sw0reloffer',
@@ -100,10 +108,11 @@ class CreateMyOrdersTests(unittest.TestCase):
            'cjfee': 0.1},
         ])
 
-    def test_dust_threshold(self):
+    async def test_dust_threshold(self):
         jm_single().DUST_THRESHOLD = 1000
-        yg = create_yg_basic([0, 2000000, 1000000], txfee_contribution=10,
-                cjfee_a=10, ordertype='swabsoffer', minsize=100000)
+        yg = await create_yg_basic(
+            [0, 2000000, 1000000], txfee_contribution=10,
+            cjfee_a=10, ordertype='swabsoffer', minsize=100000)
         self.assertEqual(yg.create_my_orders(), [
           {'oid': 0,
            'ordertype': 'swabsoffer',
@@ -113,95 +122,98 @@ class CreateMyOrdersTests(unittest.TestCase):
            'cjfee': '20'},
         ])
 
-    def test_minsize_above_maxsize(self):
+    async def test_minsize_above_maxsize(self):
         jm_single().DUST_THRESHOLD = 10
-        yg = create_yg_basic([0, 20000, 10000], txfee_contribution=1000,
-                cjfee_a=10, ordertype='swabsoffer', minsize=100000)
+        yg = await create_yg_basic(
+            [0, 20000, 10000], txfee_contribution=1000,
+            cjfee_a=10, ordertype='swabsoffer', minsize=100000)
         self.assertEqual(yg.create_my_orders(), [])
 
 
-class OidToOrderTests(unittest.TestCase):
+class OidToOrderTests(TrialAsyncioTestCase):
     """Tests YieldGeneratorBasic.oid_to_order."""
 
-    def call_oid_to_order(self, yg, amount):
+    async def call_oid_to_order(self, yg, amount):
         """Calls oid_to_order on the given yg instance.  It passes the
         txfee and abs fee from yg as offer."""
         offer = {'txfee': yg.txfee_contribution,
                  'cjfee': str(yg.cjfee_a),
                  'ordertype': 'swabsoffer'}
-        return yg.oid_to_order(offer, amount)
+        return await yg.oid_to_order(offer, amount)
 
-    def test_not_enough_balance(self):
-        yg = create_yg_basic([100], txfee_contribution=0, cjfee_a=10)
-        self.assertEqual(self.call_oid_to_order(yg, 1000), (None, None, None))
+    async def test_not_enough_balance(self):
+        yg = await create_yg_basic([100], txfee_contribution=0, cjfee_a=10)
+        self.assertEqual(
+            await self.call_oid_to_order(yg, 1000), (None, None, None))
 
-    def test_chooses_single_utxo(self):
+    async def test_chooses_single_utxo(self):
         jm_single().DUST_THRESHOLD = 10
-        yg = create_yg_basic([10, 1000, 2000])
-        utxos, cj_addr, change_addr = self.call_oid_to_order(yg, 500)
+        yg = await create_yg_basic([10, 1000, 2000])
+        utxos, cj_addr, change_addr = await self.call_oid_to_order(yg, 500)
         self.assertEqual(len(utxos), 1)
         yg.wallet_service.wallet.assert_utxos_from_mixdepth(utxos, 1)
         self.assertEqual(yg.wallet_service.wallet.get_addr_mixdepth(cj_addr), 2)
         self.assertEqual(yg.wallet_service.wallet.get_addr_mixdepth(change_addr), 1)
 
-    def test_not_enough_balance_with_dust_threshold(self):
+    async def test_not_enough_balance_with_dust_threshold(self):
         # 410 is exactly the size of the change output.  So it will be
         # right at the dust threshold.  The wallet won't be able to find
         # any extra inputs, though.
         jm_single().DUST_THRESHOLD = 410
-        yg = create_yg_basic([10, 1000, 10], txfee_contribution=100,
-                cjfee_a=10)
-        self.assertEqual(self.call_oid_to_order(yg, 500), (None, None, None))
+        yg = await create_yg_basic(
+            [10, 1000, 10], txfee_contribution=100, cjfee_a=10)
+        self.assertEqual(
+            await self.call_oid_to_order(yg, 500), (None, None, None))
 
-    def test_extra_with_dust_threshold(self):
+    async def test_extra_with_dust_threshold(self):
         # The output will be right at the dust threshold, so that we will
         # need to include the extra_utxo from the wallet as well to get
         # over the threshold.
         jm_single().DUST_THRESHOLD = 410
-        yg = create_yg_basic([10, 1000, 10], txfee_contribution=100,
-                cjfee_a=10)
-        yg.wallet_service.wallet.add_utxo_at_mixdepth(1, 500)
-        utxos, cj_addr, change_addr = self.call_oid_to_order(yg, 500)
+        yg = await create_yg_basic(
+            [10, 1000, 10], txfee_contribution=100, cjfee_a=10)
+        await yg.wallet_service.wallet.add_utxo_at_mixdepth(1, 500)
+        utxos, cj_addr, change_addr = await self.call_oid_to_order(yg, 500)
         self.assertEqual(len(utxos), 2)
         yg.wallet_service.wallet.assert_utxos_from_mixdepth(utxos, 1)
         self.assertEqual(yg.wallet_service.wallet.get_addr_mixdepth(cj_addr), 2)
         self.assertEqual(yg.wallet_service.wallet.get_addr_mixdepth(change_addr), 1)
 
 
-class OfferReannouncementTests(unittest.TestCase):
+class OfferReannouncementTests(TrialAsyncioTestCase):
     """Tests offer reannouncement logic from on_tx_unconfirmed."""
 
     def call_on_tx_unconfirmed(self, yg):
         """Calls yg.on_tx_unconfirmed with fake arguments."""
         return yg.on_tx_unconfirmed({'cjaddr': 'addr'}, 'txid')
 
-    def create_yg_and_offer(self, maxsize):
+    async def create_yg_and_offer(self, maxsize):
         """Constructs a fake yg instance that has an offer with the given
         maxsize.  Returns it together with the offer."""
         jm_single().DUST_THRESHOLD = 10
-        yg = create_yg_basic([100 + maxsize], txfee_contribution=100,
-                ordertype='swabsoffer')
+        yg = await create_yg_basic(
+            [100 + maxsize], txfee_contribution=100, ordertype='swabsoffer')
         offers = yg.create_my_orders()
         self.assertEqual(len(offers), 1)
         self.assertEqual(offers[0]['maxsize'], maxsize)
         return yg, offers[0]
 
-    def test_no_new_offers(self):
-        yg = create_yg_basic([0] * 3)
+    async def test_no_new_offers(self):
+        yg = await create_yg_basic([0] * 3)
         yg.offerlist = [{'oid': 0}]
         self.assertEqual(self.call_on_tx_unconfirmed(yg), ([0], []))
 
-    def test_no_old_offers(self):
-        yg, offer = self.create_yg_and_offer(100)
+    async def test_no_old_offers(self):
+        yg, offer = await self.create_yg_and_offer(100)
         yg.offerlist = []
         self.assertEqual(self.call_on_tx_unconfirmed(yg), ([], [offer]))
 
-    def test_offer_unchanged(self):
-        yg, offer = self.create_yg_and_offer(100)
+    async def test_offer_unchanged(self):
+        yg, offer = await self.create_yg_and_offer(100)
         yg.offerlist = [offer]
         self.assertEqual(self.call_on_tx_unconfirmed(yg), ([], []))
 
-    def test_offer_changed(self):
-        yg, offer = self.create_yg_and_offer(100)
+    async def test_offer_changed(self):
+        yg, offer = await self.create_yg_and_offer(100)
         yg.offerlist = [{'oid': 0, 'maxsize': 10}]
         self.assertEqual(self.call_on_tx_unconfirmed(yg), ([], [offer]))