Merge JoinMarket-Org/joinmarket-clientserver#1432: tests: make setup fixtures optional.

c4d9b9264a tests: make setup fixtures optional. (PulpCattel)

Pull request description:

  Currently, in tests, we always setup `bitcoind` and `miniircd` (this happens because of `autouse=True` in fixtures).
  A lot of test modules do not require these slow and expensive setups. (`miniircd` in particular is AFAICT only used in 1 test module `jmdaemon/test/test_irc_messaging.py`).

  After #1427, we now have indipendent fixtures for each component, so we can tell `Pytest` specifically what we need.
  This PR uses [pytest.mark.usefixtures()](https://docs.pytest.org/en/7.2.x/how-to/fixtures.html#use-fixtures-in-classes-and-modules-with-usefixtures) to specify fixtures usage at the test module level.
  Useful so that fixtures like `setup_regtest_bitcoind` can run *only* if we are planning to invoke a test that requires them.

  For example, it's now possible to run:

  ```
  pytest jmbitcoin/
  ```

  This will skip all setup initialization because no test module in `/jmbitcoin` asks for it.

  When running the entire test suite, there should be no difference in behaviour.

  To see "behind the scene” of how `Pytest` schedules a test run, without actually running it, there's the very useful option `--setup-plan`.

ACKs for top commit:
  AdamISZ:
    Nice change, thanks. tACK c4d9b9264a

Tree-SHA512: 307d8a60a13cbbd9648efc489cb6da2ecf8e3d42772df09ec99a14fa822ad86ca53212364fd437c79ab718a8effe0de6b6810f6a8ee1df3ae1ad44181beec7a3

conftest.py

@@ -82,6 +82,29 @@ def pytest_addoption(parser: Any) -> None:
 
 
 @pytest.fixture(scope="session", autouse=True)
+def setup_early_if_needed(request) -> None:
+    """
+    Make sure fixtures requested by test *modules* are executed first.
+    I.e., like a dynamically set `autouse=True`.
+    (By default, without `autouse=True`, they would run later at request time)
+    Useful so that fixtures like `setup_regtest_bitcoind` can run *only* if
+    we are planning to invoke a test that requires it, but still at startup time.
+    """
+    modules = set()
+    # Loop through the collected tests
+    for item in request.node.items:
+        module = item.getparent(pytest.Module)
+        if module in modules:
+            continue
+        modules.add(module)
+        # Loop through each test module marker
+        for marker in module.iter_markers('usefixtures'):
+            # We know we are gonna need these fixtures, so we invoke them early
+            for fixture in marker.args:
+                request.getfixturevalue(fixture)
+
+
+@pytest.fixture(scope="session")
 def setup_miniircd(pytestconfig):
     """
     Setup miniircd and handle its clean up.
@@ -102,7 +125,7 @@ def setup_miniircd(pytestconfig):
         m.kill()
 
 
-@pytest.fixture(scope="session", autouse=True)
+@pytest.fixture(scope="session")
 def setup_regtest_bitcoind(pytestconfig):
     """
     Setup regtest bitcoind and handle its clean up.

jmclient/test/test_blockchaininterface.py

@@ -10,6 +10,8 @@ from jmclient import load_test_config, jm_single, BaseWallet
 
 log = get_log()
 
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 
 def sync_test_wallet(fast, wallet_service):
     sync_count = 0

jmclient/test/test_bond_calc.py

@@ -4,6 +4,8 @@ import pytest
 from jmclient import jm_single, load_test_config, FidelityBondMixin
 from jmclient.bond_calc import get_next_locktime, get_bond_values, get_percentiles
 
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 
 @pytest.mark.parametrize(('date', 'next_locktime'),
                          ((datetime(2022, 1, 1, 1, 1), datetime(2022, 2, 1)),

jmclient/test/test_client_protocol.py

@@ -21,6 +21,11 @@ import json
 import jmbitcoin as bitcoin
 import twisted
 import base64
+
+import pytest
+
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 twisted.internet.base.DelayedCall.debug = True
 
 test_completed = False

jmclient/test/test_coinjoin.py

@@ -18,6 +18,8 @@ from commontest import make_wallets, default_max_cj_fee
 from test_taker import dummy_filter_orderbook
 import jmbitcoin as btc
 
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 testdir = os.path.dirname(os.path.realpath(__file__))
 log = get_log()
 

jmclient/test/test_commitment_utils.py

@@ -6,6 +6,9 @@ from jmclient import (load_test_config, jm_single, BTC_P2WPKH)
 from jmclient.commitment_utils import get_utxo_info, validate_utxo_data
 from jmbitcoin import select_chain_params
 
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
+
 def test_get_utxo_info():
     load_test_config()
     # this test tests mainnet keys, so temporarily switch network

jmclient/test/test_configure.py

@@ -4,6 +4,8 @@ import pytest
 from jmclient import load_test_config, jm_single
 from jmclient.configure import get_blockchain_interface_instance
 
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 
 def test_attribute_dict():
     from jmclient.configure import AttributeDict

jmclient/test/test_core_nohistory_sync.py

@@ -11,6 +11,8 @@ from jmclient import (load_test_config, SegwitLegacyWallet,
                       SegwitWallet, jm_single, BaseWallet)
 from jmbitcoin import select_chain_params
 
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 log = get_log()
 
 def test_fast_sync_unavailable(setup_sync):

jmclient/test/test_payjoin.py

@@ -4,6 +4,8 @@ Test doing payjoins over tcp client/server
 """
 
 import os
+
+import pytest
 from twisted.internet import reactor
 from twisted.web.server import Site
 from twisted.web.client import readBody
@@ -26,6 +28,8 @@ from jmclient.payjoin import process_payjoin_proposal_from_server
 from commontest import make_wallets
 from test_coinjoin import make_wallets_to_list, sync_wallets
 
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 testdir = os.path.dirname(os.path.realpath(__file__))
 log = get_log()
 

jmclient/test/test_podle.py

@@ -12,6 +12,9 @@ from jmclient import load_test_config, jm_single, generate_podle,\
     get_podle_commitments, add_external_commitments, update_commitments
 from jmclient.podle import verify_all_NUMS, verify_podle, PoDLEError
 from commontest import make_wallets
+
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 log = get_log()
 
 def test_commitments_empty(setup_podle):

jmclient/test/test_privkeys.py

@@ -7,6 +7,9 @@ from jmclient import BTCEngine, jm_single, load_test_config
 import json
 import pytest
 import os
+
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 testdir = os.path.dirname(os.path.realpath(__file__))
 
 def test_read_raw_privkeys(setup_keys):

jmclient/test/test_psbt_wallet.py

@@ -17,6 +17,9 @@ from jmclient import (load_test_config, jm_single, direct_send,
                       SegwitLegacyWallet, SegwitWallet, LegacyWallet,
                       VolatileStorage, get_network)
 from jmclient.wallet import PSBTWalletMixin
+
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 log = get_log()
 
 def create_volatile_wallet(seedphrase, wallet_cls=SegwitWallet):

jmclient/test/test_schedule.py

@@ -6,6 +6,8 @@ from jmclient import (get_schedule, get_tumble_schedule,
                       tweak_tumble_schedule, load_test_config)
 import os
 
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 valids = """#sample for testing
 1, 110000000, 3, INTERNAL, 0, 16, 1
 0, 20000000, 2, mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw, 9.88, 16, 0

jmclient/test/test_snicker.py

@@ -10,6 +10,8 @@ from jmbase import get_log, bintohex
 from jmclient import (load_test_config, estimate_tx_fee, SNICKERReceiver,
                       direct_send, BaseWallet)
 
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 log = get_log()
 
 @pytest.mark.parametrize(

jmclient/test/test_taker.py

@@ -19,6 +19,9 @@ from taker_test_data import t_utxos_by_mixdepth, t_orderbook,\
     t_maker_response, t_chosen_orders, t_dummy_ext
 from commontest import default_max_cj_fee
 
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
+
 def convert_utxos(utxodict):
     return_dict = {}
     for uk, val in utxodict.items():

jmclient/test/test_tx_creation.py

@@ -13,6 +13,8 @@ import pytest
 from jmbase import get_log
 from jmclient import load_test_config, jm_single, direct_send, estimate_tx_fee, compute_tx_locktime
 
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 log = get_log()
 #just a random selection of pubkeys for receiving multisigs;
 #if we ever need the privkeys, they are in a json file somewhere

jmclient/test/test_valid_addresses.py

@@ -7,6 +7,9 @@ from bitcointx import ChainParams
 import json
 import pytest
 import os
+
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 testdir = os.path.dirname(os.path.realpath(__file__))
 
 def address_valid_somewhere(addr):

jmclient/test/test_wallet.py

@@ -19,6 +19,7 @@ from test_blockchaininterface import sync_test_wallet
 from freezegun import freeze_time
 from bitcointx.wallet import CCoinAddressError
 
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
 
 testdir = os.path.dirname(os.path.realpath(__file__))
 

jmclient/test/test_wallet_rpc.py

@@ -1,4 +1,6 @@
 import os, json
+
+import pytest
 from twisted.internet import reactor, defer, task
 
 from twisted.web.client import readBody, Headers
@@ -19,6 +21,8 @@ from test_coinjoin import make_wallets_to_list, sync_wallets
 from test_websocket import (ClientTProtocol, test_tx_hex_1,
                             test_tx_hex_txid, encoded_token)
 
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 testdir = os.path.dirname(os.path.realpath(__file__))
 
 testfilename = "testwrpc"

jmclient/test/test_wallets.py

@@ -13,6 +13,9 @@ from jmclient import (
     load_test_config, jm_single,
     estimate_tx_fee, BitcoinCoreInterface, Mnemonic)
 from taker_test_data import t_raw_signed_tx
+
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 testdir = os.path.dirname(os.path.realpath(__file__))
 log = get_log()
 

jmclient/test/test_walletservice.py

@@ -7,6 +7,9 @@ from jmclient import load_test_config, jm_single, \
      WalletService
 from test_blockchaininterface import sync_test_wallet
 from test_wallet import fund_wallet_addr, get_populated_wallet
+
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 testdir = os.path.dirname(os.path.realpath(__file__))
 log = get_log()
 

jmclient/test/test_walletutils.py

@@ -6,6 +6,9 @@ from jmclient.wallet_utils import (bip32pathparse, WalletView,
                                    WalletViewAccount, WalletViewBranch,
                                    WalletViewEntry, wallet_signmessage)
 
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
+
 # The below signatures have all been verified against Electrum 4.0.9:
 @pytest.mark.parametrize('seed, hdpath, walletcls, message, sig, addr', [
     [b"\x01"*16, "m/84'/0'/0'/0/0", SegwitWallet, "hello",

jmclient/test/test_yieldgenerator.py

@@ -1,10 +1,14 @@
 
 import unittest
+
+import pytest
 from jmbitcoin import CMutableTxOut, CMutableTransaction
 from jmclient import load_test_config, jm_single,\
     SegwitLegacyWallet, VolatileStorage, YieldGeneratorBasic, \
     get_network, WalletService
 
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 
 class CustomUtxoWallet(SegwitLegacyWallet):
     """A wallet instance that makes it easy to do the tasks we need for

jmdaemon/test/test_daemon_protocol.py

@@ -22,6 +22,11 @@ from msgdata import *
 import base64
 import sys
 from dummy_mc import DummyMessageChannel
+
+import pytest
+
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 test_completed = False
 end_early = False
 jlog = get_log()

jmdaemon/test/test_irc_messaging.py

@@ -8,6 +8,10 @@ from jmdaemon import IRCMessageChannel, MessageChannelCollection
 #needed for test framework
 from jmclient import (load_test_config, get_mchannels, jm_single)
 
+import pytest
+
+pytestmark = pytest.mark.usefixtures("setup_miniircd", "setup_regtest_bitcoind")
+
 si = 1
 class DummyDaemon(object):
     def request_signature_verify(self, a, b, c, d, e,

jmdaemon/test/test_orderbookwatch.py

@@ -7,6 +7,8 @@ from jmdaemon.protocol import JM_VERSION, ORDER_KEYS
 from jmbase.support import hextobin
 from jmclient.fidelity_bond import FidelityBondProof
 
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 class DummyDaemon(object):
     def request_signature_verify(self, a, b, c, d, e,
             f, g, h):

test/test_e2e_coinjoin.py

@@ -27,6 +27,8 @@ from jmclient import (YieldGeneratorBasic, load_test_config, jm_single,
 import jmclient
 from jmclient.wallet_rpc import api_version_string
 
+pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
+
 log = get_log()
 
 wallet_name = "test-onion-yg-runner.jmdat"