From 9112b51bb94c0298de297a0d24a4febdfae2b05e Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Thu, 12 Jan 2023 15:48:55 +0100
Subject: [PATCH 01/37] Support both pytest and legacy system test runner

The legacy system test framework uses pytest to execute some tests.
Since it'd be quite difficult to convince pytest to decide whether to
include conftest.py (or which ones to include when launching from
subdir), it makes more sense to have a shared conftest.py which is used
by both the legacy test runner invocations of pytest and the new pytest
system test runner. It is ugly, but once we drop support for the legacy
runner, we'll get rid of it.

Properly scope the *port fixtures in order to ensure they'll work as
expected with the new pytest runner. Instead of using "session" (which
means the fixture is only evaluated once for the entire execution of
pytest), use "module" scope, which is evaluated separately for each
module. The legacy runner invoked pytest for each system test
separately, while the new pytest runner is invoked once for all system
tests -- therefore it requires the more fine-grained "module" scope to
for the fixtures to work properly.

Remove python shebang, as conftest.py isn't supposed to be an executable
script.

(cherry picked from commit 30cb9b7e28316208749f4587d5bc880c7beca410)
---
 bin/tests/system/conftest.py | 30 +++++++++++++++++++++++-------
 bin/tests/system/run.sh.in   |  2 +-
 2 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 96de101c3c..009d989ceb 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -1,5 +1,3 @@
-#!/usr/bin/python3
-
 # Copyright (C) Internet Systems Consortium, Inc. ("ISC")
 #
 # SPDX-License-Identifier: MPL-2.0
@@ -12,25 +10,43 @@
 # information regarding copyright ownership.
 
 import os
-
 import pytest
 
 
-@pytest.fixture(scope="session")
+# ======================= LEGACY=COMPATIBLE FIXTURES =========================
+# The following fixtures are designed to work with both pytest system test
+# runner and the legacy system test framework.
+
+
+@pytest.fixture(scope="module")
 def named_port():
     return int(os.environ.get("PORT", default=5300))
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def named_tlsport():
     return int(os.environ.get("TLSPORT", default=8853))
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def named_httpsport():
     return int(os.environ.get("HTTPSPORT", default=4443))
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def control_port():
     return int(os.environ.get("CONTROLPORT", default=9953))
+
+
+# ======================= PYTEST SYSTEM TEST RUNNER ==========================
+# From this point onward, any setting, fixtures or functions only apply to the
+# new pytest runner. Ideally, these would be in a separate file. However, due
+# to how pytest works and how it's used by the legacy runner, the best approach
+# is to have everything in this file to avoid duplication and set the
+# LEGACY_TEST_RUNNER if pytest is executed from the legacy framework.
+#
+# FUTURE: Once legacy runner is no longer supported, remove the env var and
+# don't branch the code.
+
+if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
+    pass  # will be implemented in followup commits
diff --git a/bin/tests/system/run.sh.in b/bin/tests/system/run.sh.in
index 550574c61f..8ba6224536 100644
--- a/bin/tests/system/run.sh.in
+++ b/bin/tests/system/run.sh.in
@@ -230,7 +230,7 @@ if [ $status -eq 0 ]; then
             if start_servers; then
                 run=$((run+1))
                 test_status=0
-                (cd "$systest" && "$PYTEST" -rsxX -v "$test" "$@" || echo "$?" > "$test.status") | SYSTESTDIR="$systest" cat_d
+                (cd "$systest" && LEGACY_TEST_RUNNER=1 "$PYTEST" -rsxX -v "$test" "$@" || echo "$?" > "$test.status") | SYSTESTDIR="$systest" cat_d
                 if [ -f "$systest/$test.status" ]; then
                     if [ "$(cat "$systest/$test.status")" = "5" ]; then
                         echowarn "R:$systest:SKIPPED"

From 495216f81c7ae4e263f8935a7a10931d716aaff7 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Thu, 12 Jan 2023 16:26:25 +0100
Subject: [PATCH 02/37] Collect existing tests in pytest runner

Ensure pytest picks up our python test modules, since we're using
tests_*.py convention, which is different from the default.

Configure pytest logging to display the output from all tests (even the
ones that passed). This ensures we have sufficient amount of information
to debug test post-mortem just from the artifacts.

(cherry picked from commit 08c4e35bc01760cb179cf833e05c6f455dab1eae)
---
 bin/tests/system/pytest.ini | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 bin/tests/system/pytest.ini

diff --git a/bin/tests/system/pytest.ini b/bin/tests/system/pytest.ini
new file mode 100644
index 0000000000..e1cb863d47
--- /dev/null
+++ b/bin/tests/system/pytest.ini
@@ -0,0 +1,18 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+[pytest]
+addopts = --tb=short -rA
+log_format = %(asctime)s %(levelname)s:%(name)s %(message)s
+log_date_format = %Y-%m-%d %H:%M:%S
+log_cli = 1
+log_level = INFO
+python_files = tests_*.py

From 6000883f8d3ecc249066cc2fb7a7f1ec93580b9f Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Thu, 12 Jan 2023 16:41:41 +0100
Subject: [PATCH 03/37] Obtain env vars from conf.sh in pytest runner

The commands executed by pytest during a system test need to have the
same environment variables set as if they were executed by the run.sh
shell script.

It was decided that for the moment, legacy way of executing system tests
with run.sh should be kept, which complicates things a bit. In order to
avoid duplicating the required variables in both conf.sh and pytest, it
was decided to use the existing conf.sh as the only authoritative
place for the variables.

It is necessary to process the environment variables from conf.sh right
when conftest.py is loaded, since they might be needed right away (e.g.
to test for feature support during test collection).

This solution is a bit hacky and is only meant to be used during the
transitory phase when both pytest and the legacy run.sh are both
supported. In the future, a superior pytest-only solution should be
used.

For discussion of other options, refer to
https://gitlab.isc.org/isc-projects/bind9/-/merge_requests/6809#note_318889

(cherry picked from commit 2f7af791a1291f06887437623bd224a3447cec50)
---
 bin/tests/system/conftest.py | 42 +++++++++++++++++++++++++++++++++++-
 1 file changed, 41 insertions(+), 1 deletion(-)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 009d989ceb..f77a1b1097 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -49,4 +49,44 @@ def control_port():
 # don't branch the code.
 
 if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
-    pass  # will be implemented in followup commits
+    import logging
+    from pathlib import Path
+    import re
+    import subprocess
+
+    # ----------------------- Globals definition -----------------------------
+
+    FILE_DIR = os.path.abspath(Path(__file__).parent)
+    ENV_RE = re.compile("([^=]+)=(.*)")
+
+    # ---------------------- Module initialization ---------------------------
+
+    def parse_env(env_text):
+        """Parse the POSIX env format into Python dictionary."""
+        out = {}
+        for line in env_text.splitlines():
+            match = ENV_RE.match(line)
+            if match:
+                out[match.groups()[0]] = match.groups()[1]
+        return out
+
+    def get_env(cmd):
+        try:
+            proc = subprocess.run(
+                [cmd],
+                shell=True,
+                check=True,
+                cwd=FILE_DIR,
+                stdout=subprocess.PIPE,
+            )
+        except subprocess.CalledProcessError as exc:
+            logging.error("failed to get shell env: %s", exc)
+            raise exc
+        env_text = proc.stdout.decode("utf-8")
+        return parse_env(env_text)
+
+    # Read common environment variables for running tests from conf.sh.
+    # FUTURE: Remove conf.sh entirely and define all variables in pytest only.
+    CONF_ENV = get_env(". ./conf.sh && env")
+    os.environ.update(CONF_ENV)
+    logging.debug("conf.sh env: %s", CONF_ENV)

From f23ce6a37f3c13a874a1c146854a4f1f14b801cd Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Thu, 12 Jan 2023 16:47:55 +0100
Subject: [PATCH 04/37] Ensure system test programs and deps are compiled

Some system tests require extra programs and/or dependencies to be
compiled first. This is done via `make check` with the automake
framework when using check_* variables such as check_PROGRAMS.

To avoid running any tests via the automake framework, set the TESTS
env variable to empty string and utilize `make -e check` to override
default Makefile variables with environment ones. This ensures automake
will only compile the needed dependencies without running any tests.

Additional consideration needs to be taken for xdist. The compilation
command should be called just once before any tests are executed. To
achieve that, use the pytest_configure() hook and check that the
PYTEST_XDIST_WORKER env variable isn't set -- if it is, it indicates
we're in the spawned xdist worker and the compilation was already done
by the main pytest process that spawned the workers.

This is mostly done to have on-par functionality with legacy test
framework. In the future, we should get rid of the need to run "empty"
make -e check and perhaps compile test-stuff by default.

(cherry picked from commit ee260668971a61c59f8d82018a082d83f413a5db)
---
 bin/tests/system/conftest.py | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index f77a1b1097..69d40837fd 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -56,6 +56,7 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
 
     # ----------------------- Globals definition -----------------------------
 
+    XDIST_WORKER = os.environ.get("PYTEST_XDIST_WORKER", "")
     FILE_DIR = os.path.abspath(Path(__file__).parent)
     ENV_RE = re.compile("([^=]+)=(.*)")
 
@@ -90,3 +91,31 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
     CONF_ENV = get_env(". ./conf.sh && env")
     os.environ.update(CONF_ENV)
     logging.debug("conf.sh env: %s", CONF_ENV)
+
+    # --------------------------- pytest hooks -------------------------------
+
+    def pytest_configure():
+        # Ensure this hook only runs on the main pytest instance if xdist is
+        # used to spawn other workers.
+        if not XDIST_WORKER:
+            logging.debug("compiling required files")
+            env = os.environ.copy()
+            env["TESTS"] = ""  # disable automake test framework - compile-only
+            try:
+                # FUTURE: Remove the need to run this compilation command
+                # before executing tests. Currently it's only here to have
+                # on-par functionality with the legacy test framework.
+                proc = subprocess.run(
+                    "make -e check",
+                    shell=True,
+                    check=True,
+                    cwd=FILE_DIR,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                    env=env,
+                )
+            except subprocess.CalledProcessError as exc:
+                logging.debug(exc.stdout)
+                logging.error("failed to compile test files: %s", exc)
+                raise exc
+            logging.debug(proc.stdout)

From aa199e2726c8baf4cbfbe891e0c3182e2d47bdb2 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Thu, 12 Jan 2023 16:52:49 +0100
Subject: [PATCH 05/37] Assign unique ports to pytest modules

This is basically a pytest re-implementation of the get_ports.sh script.
The main difference is that ports are assigned on a module basis, rather
than a directory basis. Module is the new atomic unit for parallel
execution, therefore it needs to have unique ports to avoid collisions.

Each module gets its ports through the env fixture which is updated with
ports and other module-specific variables.

(cherry picked from commit 006175815640efafa1add511242b90af8c42547d)
---
 bin/tests/system/conftest.py | 84 ++++++++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 69d40837fd..4d603c0cfe 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -16,6 +16,8 @@ import pytest
 # ======================= LEGACY=COMPATIBLE FIXTURES =========================
 # The following fixtures are designed to work with both pytest system test
 # runner and the legacy system test framework.
+#
+# FUTURE: Rewrite the individual port fixtures to re-use the `ports` fixture.
 
 
 @pytest.fixture(scope="module")
@@ -53,12 +55,19 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
     from pathlib import Path
     import re
     import subprocess
+    import time
+
+    # Silence warnings caused by passing a pytest fixture to another fixture.
+    # pylint: disable=redefined-outer-name
 
     # ----------------------- Globals definition -----------------------------
 
     XDIST_WORKER = os.environ.get("PYTEST_XDIST_WORKER", "")
     FILE_DIR = os.path.abspath(Path(__file__).parent)
     ENV_RE = re.compile("([^=]+)=(.*)")
+    PORT_MIN = 5001
+    PORT_MAX = 32767
+    PORTS_PER_TEST = 20
 
     # ---------------------- Module initialization ---------------------------
 
@@ -119,3 +128,78 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
                 logging.error("failed to compile test files: %s", exc)
                 raise exc
             logging.debug(proc.stdout)
+
+    # --------------------------- Fixtures -----------------------------------
+
+    @pytest.fixture(scope="session")
+    def modules():
+        """Sorted list of all modules. Used to determine port distribution."""
+        mods = []
+        for dirpath, _dirs, files in os.walk(os.getcwd()):
+            for file in files:
+                if file.startswith("tests_") and file.endswith(".py"):
+                    mod = f"{dirpath}/{file}"
+                    mods.append(mod)
+        return sorted(mods)
+
+    @pytest.fixture(scope="session")
+    def module_base_ports(modules):
+        """
+        Dictionary containing assigned base port for every module.
+
+        Note that this is a session-wide fixture. The port numbers are
+        deterministically assigned before any testing starts. This fixture MUST
+        return the same value when called again during the same test session.
+        When running tests in parallel, this is exactly what happens - every
+        worker thread will call this fixture to determine test ports.
+        """
+        port_min = PORT_MIN
+        port_max = PORT_MAX - len(modules) * PORTS_PER_TEST
+        if port_max < port_min:
+            raise RuntimeError(
+                "not enough ports to assign unique port set to each module"
+            )
+
+        # Rotate the base port value over time to detect possible test issues
+        # with using random ports. This introduces a very slight race condition
+        # risk. If this value changes between pytest invocation and spawning
+        # worker threads, multiple tests may have same port values assigned. If
+        # these tests are then executed simultaneously, the test results will
+        # be misleading.
+        base_port = int(time.time() // 3600) % (port_max - port_min)
+
+        return {mod: base_port + i * PORTS_PER_TEST for i, mod in enumerate(modules)}
+
+    @pytest.fixture(scope="module")
+    def base_port(request, module_base_ports):
+        """Start of the port range assigned to a particular test module."""
+        port = module_base_ports[request.fspath]
+        return port
+
+    @pytest.fixture(scope="module")
+    def ports(base_port):
+        """Dictionary containing port names and their assigned values."""
+        return {
+            "PORT": str(base_port),
+            "TLSPORT": str(base_port + 1),
+            "HTTPPORT": str(base_port + 2),
+            "HTTPSPORT": str(base_port + 3),
+            "EXTRAPORT1": str(base_port + 4),
+            "EXTRAPORT2": str(base_port + 5),
+            "EXTRAPORT3": str(base_port + 6),
+            "EXTRAPORT4": str(base_port + 7),
+            "EXTRAPORT5": str(base_port + 8),
+            "EXTRAPORT6": str(base_port + 9),
+            "EXTRAPORT7": str(base_port + 10),
+            "EXTRAPORT8": str(base_port + 11),
+            "CONTROLPORT": str(base_port + 12),
+        }
+
+    @pytest.fixture(scope="module")
+    def env(ports):
+        """Dictionary containing environment variables for the test."""
+        env = CONF_ENV.copy()
+        env.update(ports)
+        env["builddir"] = f"{env['TOP_BUILDDIR']}/bin/tests/system"
+        env["srcdir"] = f"{env['TOP_SRCDIR']}/bin/tests/system"
+        return env

From 23cecd2190e05a0a2f09e0a009434076d5f1fdc2 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Thu, 12 Jan 2023 17:01:36 +0100
Subject: [PATCH 06/37] Utility fixtures for pytest runner

Add fixtures for deriving system test name from the directory name and
for a module-specific logger.

Add fixtures to execute shell and perl scripts. Similar to how run.sh
calls commands, this functionality is also needed in pytest. The
fixtures take care of switching to a proper directory, logging
everything and handling errors.

Note that the fixture system_test_dir is not defined yet. It is omitted
now for review readability and it's added in follow-up commits.

(cherry picked from commit 6886cc1826648deb9f9b71dd418bd20c74978cd6)
---
 bin/tests/system/conftest.py | 68 ++++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 4d603c0cfe..2eae1acc44 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -51,11 +51,13 @@ def control_port():
 # don't branch the code.
 
 if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
+    from functools import partial
     import logging
     from pathlib import Path
     import re
     import subprocess
     import time
+    from typing import List, Optional
 
     # Silence warnings caused by passing a pytest fixture to another fixture.
     # pylint: disable=redefined-outer-name
@@ -203,3 +205,69 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
         env["builddir"] = f"{env['TOP_BUILDDIR']}/bin/tests/system"
         env["srcdir"] = f"{env['TOP_SRCDIR']}/bin/tests/system"
         return env
+
+    @pytest.fixture(scope="module")
+    def system_test_name(request):
+        """Name of the system test directory."""
+        path = Path(request.fspath)
+        return path.parent.name
+
+    @pytest.fixture(scope="module")
+    def logger(system_test_name):
+        """Logging facility specific to this test."""
+        return logging.getLogger(system_test_name)
+
+    def _run_script(  # pylint: disable=too-many-arguments
+        env,
+        logger,
+        system_test_dir: Path,
+        interpreter: str,
+        script: str,
+        args: Optional[List[str]] = None,
+    ):
+        """Helper function for the shell / perl script invocations (through fixtures below)."""
+        if args is None:
+            args = []
+        path = Path(script)
+        if not path.is_absolute():
+            # make sure relative paths are always relative to system_dir
+            path = system_test_dir.parent / path
+        script = str(path)
+        cwd = os.getcwd()
+        if not path.exists():
+            raise FileNotFoundError(f"script {script} not found in {cwd}")
+        logger.debug("running script: %s %s %s", interpreter, script, " ".join(args))
+        logger.debug("  workdir: %s", cwd)
+        stdout = b""
+        returncode = 1
+        try:
+            proc = subprocess.run(
+                [interpreter, script] + args,
+                env=env,
+                check=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+            )
+        except subprocess.CalledProcessError as exc:
+            stdout = exc.stdout
+            returncode = exc.returncode
+            raise exc
+        else:
+            stdout = proc.stdout
+            returncode = proc.returncode
+        finally:
+            if stdout:
+                for line in stdout.decode().splitlines():
+                    logger.debug("    %s", line)
+            logger.debug("  exited with %d", returncode)
+        return proc
+
+    @pytest.fixture(scope="module")
+    def shell(env, system_test_dir, logger):
+        """Function to call a shell script with arguments."""
+        return partial(_run_script, env, logger, system_test_dir, env["SHELL"])
+
+    @pytest.fixture(scope="module")
+    def perl(env, system_test_dir, logger):
+        """Function to call a perl script with arguments."""
+        return partial(_run_script, env, logger, system_test_dir, env["PERL"])

From 0ed639010c1773fe8eff7b7fe1e36ed07991e4a6 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Thu, 12 Jan 2023 17:27:07 +0100
Subject: [PATCH 07/37] Execute the system test workflow in pytest runner

This is basically the pytest re-implementation of the run.sh script.

The fixture is applied to every module and ensures complete test
setup/teardown, such as starting/stopping servers, detecting coredumps
etc.

Note that the fixture system_test_dir is not defined yet. It is omitted
now for review readability and it's added in follow-up commits.

(cherry picked from commit 2b64618624eee66590262c37e1c02ef06c96d958)
---
 bin/tests/system/conftest.py | 97 +++++++++++++++++++++++++++++++++++-
 1 file changed, 96 insertions(+), 1 deletion(-)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 2eae1acc44..47691361bc 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -57,7 +57,7 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
     import re
     import subprocess
     import time
-    from typing import List, Optional
+    from typing import Any, Dict, List, Optional
 
     # Silence warnings caused by passing a pytest fixture to another fixture.
     # pylint: disable=redefined-outer-name
@@ -271,3 +271,98 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
     def perl(env, system_test_dir, logger):
         """Function to call a perl script with arguments."""
         return partial(_run_script, env, logger, system_test_dir, env["PERL"])
+
+    @pytest.fixture(scope="module", autouse=True)
+    def system_test(  # pylint: disable=too-many-arguments,too-many-statements
+        request,
+        env: Dict[str, str],
+        logger,
+        system_test_dir,
+        shell,
+        perl,
+    ):
+        """
+        Driver of the test setup/teardown process. Used automatically for every test module.
+
+        This is the most important one-fixture-to-rule-them-all. Note the
+        autouse=True which causes this fixture to be loaded by every test
+        module without the need to explicitly specify it.
+
+        When this fixture is used, it utilizes other fixtures, such as
+        system_test_dir, which handles the creation of the temporary test
+        directory.
+
+        Afterwards, it checks the test environment and takes care of starting
+        the servers. When everything is ready, that's when the actual tests are
+        executed. Once that is done, this fixture stops the servers and checks
+        for any artifacts indicating an issue (e.g. coredumps).
+
+        Finally, when this fixture reaches an end (or encounters an exception,
+        which may be caused by fail/skip invocations), any fixtures which is
+        used by this one are finalized - e.g. system_test_dir performs final
+        checks and cleans up the temporary test directory.
+        """
+
+        def check_net_interfaces():
+            try:
+                perl("testsock.pl", ["-p", env["PORT"]])
+            except subprocess.CalledProcessError as exc:
+                logger.error("testsock.pl: exited with code %d", exc.returncode)
+                pytest.skip("Network interface aliases not set up.")
+
+        def check_prerequisites():
+            try:
+                shell(f"{system_test_dir}/prereq.sh")
+            except FileNotFoundError:
+                pass  # prereq.sh is optional
+            except subprocess.CalledProcessError:
+                pytest.skip("Prerequisites missing.")
+
+        def setup_test():
+            try:
+                shell(f"{system_test_dir}/setup.sh")
+            except FileNotFoundError:
+                pass  # setup.sh is optional
+            except subprocess.CalledProcessError as exc:
+                logger.error("Failed to run test setup")
+                pytest.fail(f"setup.sh exited with {exc.returncode}")
+
+        def start_servers():
+            try:
+                perl("start.pl", ["--port", env["PORT"], system_test_dir.name])
+            except subprocess.CalledProcessError as exc:
+                logger.error("Failed to start servers")
+                pytest.fail(f"start.pl exited with {exc.returncode}")
+
+        def stop_servers():
+            try:
+                perl("stop.pl", [system_test_dir.name])
+            except subprocess.CalledProcessError as exc:
+                logger.error("Failed to stop servers")
+                pytest.fail(f"stop.pl exited with {exc.returncode}")
+
+        def get_core_dumps():
+            try:
+                shell("get_core_dumps.sh", [system_test_dir.name])
+            except subprocess.CalledProcessError as exc:
+                logger.error("Found core dumps")
+                pytest.fail(f"get_core_dumps.sh exited with {exc.returncode}")
+
+        os.environ.update(env)  # Ensure pytests have the same env vars as shell tests.
+        logger.info(f"test started: {request.node.name}")
+        port = int(env["PORT"])
+        logger.info("using port range: <%d, %d>", port, port + PORTS_PER_TEST - 1)
+
+        # Perform checks which may skip this test.
+        check_net_interfaces()
+        check_prerequisites()
+
+        setup_test()
+        try:
+            start_servers()
+            logger.debug("executing test(s)")
+            yield
+        finally:
+            logger.debug("test(s) finished")
+            stop_servers()
+            get_core_dumps()

From d1e6400d637d14cda83de3b5d9f3838bd5d23e06 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Thu, 12 Jan 2023 17:30:22 +0100
Subject: [PATCH 08/37] Run system tests inside temporary directories

For every pytest module, create a copy of its system test directory and
run the test from that directory. This makes it easier to clean up
afterwards and makes it less error-prone when re-running (failed) tests.

Configure the logger to capture the module's log output into a separate
file available from the temporary directory. This is quite convenient
for exploring failures.

Cases where temporary directory should be kept are handled in a
follow-up commits.

(cherry picked from commit c571af8c8fd0edc32b283819a4d9b2aa8c7458f9)
---
 bin/tests/system/conftest.py | 45 ++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 47691361bc..306d29701a 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -55,7 +55,9 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
     import logging
     from pathlib import Path
     import re
+    import shutil
     import subprocess
+    import tempfile
     import time
     from typing import Any, Dict, List, Optional
 
@@ -64,6 +66,7 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
 
     # ----------------------- Globals definition -----------------------------
 
+    LOG_FORMAT = "%(asctime)s %(levelname)7s:%(name)s  %(message)s"
     XDIST_WORKER = os.environ.get("PYTEST_XDIST_WORKER", "")
     FILE_DIR = os.path.abspath(Path(__file__).parent)
     ENV_RE = re.compile("([^=]+)=(.*)")
@@ -217,6 +220,48 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
         """Logging facility specific to this test."""
         return logging.getLogger(system_test_name)
 
+    @pytest.fixture(scope="module")
+    def system_test_dir(request, env, system_test_name, logger):
+        """
+        Temporary directory for executing the test.
+
+        This fixture is responsible for creating (and potentially removing) a
+        copy of the system test directory which is used as a temporary
+        directory for the test execution.
+
+        FUTURE: This removes the need to have clean.sh scripts.
+        """
+
+        # Create a temporary directory with a copy of the original system test dir contents
+        system_test_root = Path(f"{env['TOP_BUILDDIR']}/bin/tests/system")
+        testdir = Path(
+            tempfile.mkdtemp(prefix=f"{system_test_name}_tmp_", dir=system_test_root)
+        )
+        shutil.rmtree(testdir)
+        shutil.copytree(system_test_root / system_test_name, testdir)
+
+        # Configure logger to write to a file inside the temporary test directory
+        logger.handlers.clear()
+        logger.setLevel(logging.DEBUG)
+        handler = logging.FileHandler(testdir / "pytest.log.txt", mode="w")
+        formatter = logging.Formatter(LOG_FORMAT)
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+
+        # System tests are meant to be executed from their directory - switch to it.
+        old_cwd = os.getcwd()
+        os.chdir(testdir)
+        logger.info("switching to tmpdir: %s", testdir)
+        try:
+            yield testdir  # other fixtures / tests will execute here
+        finally:
+            os.chdir(old_cwd)
+            logger.debug("changed workdir to: %s", old_cwd)
+
+            # cases when tempdir should be kept are handled in follow-up commits
+            logger.debug("deleting temporary directory")
+            shutil.rmtree(testdir)
+
     def _run_script(  # pylint: disable=too-many-arguments
         env,
         logger,

From ba922ba7749a993ab2cbcead8410e1911f275cf7 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Thu, 12 Jan 2023 17:38:06 +0100
Subject: [PATCH 09/37] Add --noclean option to pytest runner

Support the --noclean option to allow the user to keep the artifacts
from any test run.

(cherry picked from commit 8b1a906b39dfd9a7e20e6b6746101f095787608c)
---
 bin/tests/system/conftest.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 306d29701a..30a89395e1 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -108,6 +108,14 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
 
     # --------------------------- pytest hooks -------------------------------
 
+    def pytest_addoption(parser):
+        parser.addoption(
+            "--noclean",
+            action="store_true",
+            default=False,
+            help="don't remove the temporary test directories with artifacts",
+        )
+
     def pytest_configure():
         # Ensure this hook only runs on the main pytest instance if xdist is
         # used to spawn other workers.
@@ -258,9 +266,12 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
             os.chdir(old_cwd)
             logger.debug("changed workdir to: %s", old_cwd)
 
-            # cases when tempdir should be kept are handled in follow-up commits
-            logger.debug("deleting temporary directory")
-            shutil.rmtree(testdir)
+            # Clean temporary dir unless it should be kept
+            if request.config.getoption("--noclean"):
+                logger.debug("--noclean requested, keeping temporary directory")
+            else:
+                logger.debug("deleting temporary directory")
+                shutil.rmtree(testdir)
 
     def _run_script(  # pylint: disable=too-many-arguments
         env,

From cdbdbd9eafe8eba8542169ea9a8a6e7205a880bf Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Thu, 12 Jan 2023 17:41:35 +0100
Subject: [PATCH 10/37] Ensure tempdir is kept for failed system tests

The temporary directory contains artifacts for the pytest module. That
module may contain multiple individual tests which were executed
sequentially. The artifacts should be kept if even one of these tests
failed.

Since pytest doesn't have any facility to expose test results to
fixtures, customize the pytest_runtest_makereport() hook to enable that.
It stores the test results into a session scope variable which is
available in all fixtures.

When deciding whether to remove the temporary directory, find the
relevant test results for this module and don't remove the tmpdir if any
one the tests failed.

(cherry picked from commit 2d93ed8546bb23d39b47ee168ee47b11474670a7)
---
 bin/tests/system/conftest.py | 44 ++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 30a89395e1..eb69ce4eef 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -142,6 +142,27 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
                 raise exc
             logging.debug(proc.stdout)
 
+    @pytest.hookimpl(tryfirst=True, hookwrapper=True)
+    def pytest_runtest_makereport(item):
+        """Hook that is used to expose test results to session (for use in fixtures)."""
+        # execute all other hooks to obtain the report object
+        outcome = yield
+        report = outcome.get_result()
+
+        # Set the test outcome in session, so we can access it from module-level
+        # fixture using nodeid. Note that this hook is called three times: for
+        # setup, call and teardown. We only care about the overall result so we
+        # merge the results together and preserve the information whether a test
+        # passed.
+        test_results = {}
+        try:
+            test_results = getattr(item.session, "test_results")
+        except AttributeError:
+            setattr(item.session, "test_results", test_results)
+        node_result = test_results.get(item.nodeid)
+        if node_result is None or report.outcome != "passed":
+            test_results[item.nodeid] = report.outcome
+
     # --------------------------- Fixtures -----------------------------------
 
     @pytest.fixture(scope="session")
@@ -240,6 +261,25 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
         FUTURE: This removes the need to have clean.sh scripts.
         """
 
+        def get_test_result():
+            """Aggregate test results from all individual tests from this module
+            into a single result: failed > skipped > passed."""
+            test_results = {
+                node.nodeid: request.session.test_results[node.nodeid]
+                for node in request.node.collect()
+                if node.nodeid in request.session.test_results
+            }
+            logger.debug(test_results)
+            assert len(test_results)
+            failed = any(res == "failed" for res in test_results.values())
+            skipped = any(res == "skipped" for res in test_results.values())
+            if failed:
+                return "failed"
+            if skipped:
+                return "skipped"
+            assert all(res == "passed" for res in test_results.values())
+            return "passed"
+
         # Create a temporary directory with a copy of the original system test dir contents
         system_test_root = Path(f"{env['TOP_BUILDDIR']}/bin/tests/system")
         testdir = Path(
@@ -266,9 +306,13 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
             os.chdir(old_cwd)
             logger.debug("changed workdir to: %s", old_cwd)
 
+            result = get_test_result()
+
             # Clean temporary dir unless it should be kept
             if request.config.getoption("--noclean"):
                 logger.debug("--noclean requested, keeping temporary directory")
+            elif result == "failed":
+                logger.debug("test failure detected, keeping temporary directory")
             else:
                 logger.debug("deleting temporary directory")
                 shutil.rmtree(testdir)

From eaf64e93aa231cb61ab1f774b9e9935031a51f62 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Thu, 12 Jan 2023 17:59:51 +0100
Subject: [PATCH 11/37] Keep the tempdir in case test setup/teardown fails

When an issue occurs inside a fixture (e.g. servers fail to start/stop),
the test result won't be detected as failed, but rather an error will be
thrown.

To ensure the tempdir is kept even if the test itself passes but the
system_test() fixture throws an error, a different mechanism is needed.
At the start of the critical test setup section, note that the fixture
hasn't finished yet. When this is detected in the system_test_dir()
fixture, it is recognized as error in test setup/teardown and the temp
directory is kept.

This may seem cumbersome, because it is. It's basically a workaround for
the way pytest handles fixtures and test errors in general.

(cherry picked from commit 247e90c382447470a7f2b26487c265370cc04c7c)
---
 bin/tests/system/conftest.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index eb69ce4eef..84a160f8bd 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -66,6 +66,7 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
 
     # ----------------------- Globals definition -----------------------------
 
+    FIXTURE_OK = pytest.StashKey[bool]()  # pylint: disable=no-member
     LOG_FORMAT = "%(asctime)s %(levelname)7s:%(name)s  %(message)s"
     XDIST_WORKER = os.environ.get("PYTEST_XDIST_WORKER", "")
     FILE_DIR = os.path.abspath(Path(__file__).parent)
@@ -313,6 +314,10 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
                 logger.debug("--noclean requested, keeping temporary directory")
             elif result == "failed":
                 logger.debug("test failure detected, keeping temporary directory")
+            elif not request.node.stash[FIXTURE_OK]:
+                logger.debug(
+                    "test setup/teardown issue detected, keeping temporary directory"
+                )
             else:
                 logger.debug("deleting temporary directory")
                 shutil.rmtree(testdir)
@@ -453,10 +458,17 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
         port = int(env["PORT"])
         logger.info("using port range: <%d, %d>", port, port + PORTS_PER_TEST - 1)
 
+        request.node.stash[FIXTURE_OK] = True
+
         # Perform checks which may skip this test.
         check_net_interfaces()
         check_prerequisites()
 
+        # Store the fact that this fixture hasn't successfully finished yet.
+        # This is checked before temporary directory teardown to decide whether
+        # it's okay to remove the directory.
+        request.node.stash[FIXTURE_OK] = False
+
         setup_test()
         try:
             start_servers()
@@ -466,3 +478,4 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
             logger.debug("test(s) finished")
             stop_servers()
             get_core_dumps()
+            request.node.stash[FIXTURE_OK] = True

From b8d6434d5d2882ed9fbd6735623749783935bbb0 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Thu, 12 Jan 2023 18:01:02 +0100
Subject: [PATCH 12/37] Ensure compatiblity with older pytest

Special care needs to be taken to support older pytest / xdist versions.
The target versions are what is available in EL8, since that seems to
have the oldest versions that can be reasonably supported.

(cherry picked from commit 527ac6ad269105a989c726871d8ec503ef0d69f9)
---
 bin/tests/system/conftest.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 84a160f8bd..4b9dccdc8e 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -64,9 +64,22 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
     # Silence warnings caused by passing a pytest fixture to another fixture.
     # pylint: disable=redefined-outer-name
 
+    # ----------------- Older pytest / xdist compatibility -------------------
+    # As of 2023-01-11, the minimal supported pytest / xdist versions are
+    # determined by what is available in EL8/EPEL8:
+    # - pytest 3.4.2
+    # - pytest-xdist 1.24.1
+    _pytest_ver = pytest.__version__.split(".")
+    _pytest_major_ver = int(_pytest_ver[0])
+    if _pytest_major_ver < 7:
+        # pytest.Stash/pytest.StashKey mechanism has been added in 7.0.0
+        # for older versions, use regular dictionary with string keys instead
+        FIXTURE_OK = "fixture_ok"  # type: Any
+    else:
+        FIXTURE_OK = pytest.StashKey[bool]()  # pylint: disable=no-member
+
     # ----------------------- Globals definition -----------------------------
 
-    FIXTURE_OK = pytest.StashKey[bool]()  # pylint: disable=no-member
     LOG_FORMAT = "%(asctime)s %(levelname)7s:%(name)s  %(message)s"
     XDIST_WORKER = os.environ.get("PYTEST_XDIST_WORKER", "")
     FILE_DIR = os.path.abspath(Path(__file__).parent)
@@ -458,6 +471,8 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
         port = int(env["PORT"])
         logger.info("using port range: <%d, %d>", port, port + PORTS_PER_TEST - 1)
 
+        if not hasattr(request.node, "stash"):  # compatibility with pytest<7.0.0
+            request.node.stash = {}  # use regular dict instead of pytest.Stash
         request.node.stash[FIXTURE_OK] = True
 
         # Perform checks which may skip this test.

From 0df2ea8fab348cf513681c908de862c7f80c0f39 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Thu, 12 Jan 2023 18:02:39 +0100
Subject: [PATCH 13/37] Ignore tempdirs during pytest collection phase

(cherry picked from commit 8a406b73c976ef06a942091c317eb9a82898690f)
---
 bin/tests/system/.gitignore  | 1 +
 bin/tests/system/conftest.py | 8 ++++++++
 2 files changed, 9 insertions(+)

diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore
index 70e4df15e2..588c6dc0e2 100644
--- a/bin/tests/system/.gitignore
+++ b/bin/tests/system/.gitignore
@@ -22,3 +22,4 @@ parallel.mk
 /start.sh
 /stop.sh
 /ifconfig.sh
+/*_tmp_*
diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 4b9dccdc8e..f2b5be828b 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -156,6 +156,14 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
                 raise exc
             logging.debug(proc.stdout)
 
+    def pytest_ignore_collect(path):
+        # System tests are executed in temporary directories inside
+        # bin/tests/system. These temporary directories contain all files
+        # needed for the system tests - including tests_*.py files. Make sure to
+        # ignore these during test collection phase. Otherwise, test artifacts
+        # from previous runs could mess with the runner.
+        return "_tmp_" in str(path)
+
     @pytest.hookimpl(tryfirst=True, hookwrapper=True)
     def pytest_runtest_makereport(item):
         """Hook that is used to expose test results to session (for use in fixtures)."""

From 3b6ca76f46bf0e0767d923c5960509b659a22b74 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Tue, 29 Nov 2022 15:07:02 +0100
Subject: [PATCH 14/37] Add pytest functions for shell system tests

In order to run the shell system tests, the pytest runner has to pick
them up somehow. Adding an extra python file with a single function
for the shell tests for each system test proved to be the most
compatible way of running the shell tests across older pytest/xdist
versions.

Modify the legacy run.sh script to ignore these pytest-runner specific
glue files when executing tests written in pytest.

(cherry picked from commit 2f5bf6d971f7cdaeb488403377363d3e7c601541)

(manually added host/tests_sh_host.py, tkey/tests_sh_tkey.py on top)
---
 bin/tests/system/acl/tests_sh_acl.py               | 14 ++++++++++++++
 bin/tests/system/additional/tests_sh_additional.py | 14 ++++++++++++++
 bin/tests/system/addzone/tests_sh_addzone.py       | 14 ++++++++++++++
 .../system/allow-query/tests_sh_allowquery.py      | 14 ++++++++++++++
 bin/tests/system/auth/tests_sh_auth.py             | 14 ++++++++++++++
 bin/tests/system/autosign/tests_sh_autosign.py     | 14 ++++++++++++++
 bin/tests/system/builtin/tests_sh_builtin.py       | 14 ++++++++++++++
 bin/tests/system/cacheclean/tests_sh_cacheclean.py | 14 ++++++++++++++
 bin/tests/system/case/tests_sh_case.py             | 14 ++++++++++++++
 bin/tests/system/catz/tests_sh_catz.py             | 14 ++++++++++++++
 bin/tests/system/cds/tests_sh_cds.py               | 14 ++++++++++++++
 bin/tests/system/chain/tests_sh_chain.py           | 14 ++++++++++++++
 bin/tests/system/checkconf/tests_sh_checkconf.py   | 14 ++++++++++++++
 bin/tests/system/checknames/tests_sh_checknames.py | 14 ++++++++++++++
 bin/tests/system/checkzone/tests_sh_checkzone.py   | 14 ++++++++++++++
 bin/tests/system/conftest.py                       |  9 +++++++++
 bin/tests/system/cookie/tests_sh_cookie.py         | 14 ++++++++++++++
 bin/tests/system/database/tests_sh_database.py     | 14 ++++++++++++++
 bin/tests/system/dialup/tests_sh_dialup.py         | 14 ++++++++++++++
 bin/tests/system/digdelv/tests_sh_digdelv.py       | 14 ++++++++++++++
 .../system/dlzexternal/tests_sh_dlzexternal.py     | 14 ++++++++++++++
 bin/tests/system/dns64/tests_sh_dns64.py           | 14 ++++++++++++++
 bin/tests/system/dnssec/tests_sh_dnssec.py         | 14 ++++++++++++++
 bin/tests/system/dnstap/tests_sh_dnstap.py         | 14 ++++++++++++++
 bin/tests/system/doth/tests_sh_doth.py             | 14 ++++++++++++++
 bin/tests/system/dsdigest/tests_sh_dsdigest.py     | 14 ++++++++++++++
 bin/tests/system/dupsigs/tests_sh_dupsigs.py       | 14 ++++++++++++++
 bin/tests/system/dyndb/tests_sh_dyndb.py           | 14 ++++++++++++++
 bin/tests/system/ecdsa/tests_sh_ecdsa.py           | 14 ++++++++++++++
 bin/tests/system/eddsa/tests_sh_eddsa.py           | 14 ++++++++++++++
 .../ednscompliance/tests_sh_ednscompliance.py      | 14 ++++++++++++++
 bin/tests/system/emptyzones/tests_sh_emptyzones.py | 14 ++++++++++++++
 .../system/enginepkcs11/tests_sh_enginepkcs11.py   | 14 ++++++++++++++
 bin/tests/system/fetchlimit/tests_sh_fetchlimit.py | 14 ++++++++++++++
 .../system/filter-aaaa/tests_sh_filter_aaaa.py     | 14 ++++++++++++++
 bin/tests/system/formerr/tests_sh_formerr.py       | 14 ++++++++++++++
 bin/tests/system/forward/tests_sh_forward.py       | 14 ++++++++++++++
 bin/tests/system/geoip2/tests_sh_geoip2.py         | 14 ++++++++++++++
 bin/tests/system/glue/tests_sh_glue.py             | 14 ++++++++++++++
 bin/tests/system/host/tests_sh_host.py             | 14 ++++++++++++++
 bin/tests/system/idna/tests_sh_idna.py             | 14 ++++++++++++++
 .../tests_sh_include_multiplecfg.py                | 14 ++++++++++++++
 bin/tests/system/inline/tests_sh_inline.py         | 14 ++++++++++++++
 bin/tests/system/integrity/tests_sh_integrity.py   | 14 ++++++++++++++
 bin/tests/system/ixfr/tests_sh_ixfr.py             | 14 ++++++++++++++
 bin/tests/system/journal/tests_sh_journal.py       | 14 ++++++++++++++
 bin/tests/system/kasp/tests_sh_kasp.py             | 14 ++++++++++++++
 bin/tests/system/keepalive/tests_sh_keepalive.py   | 14 ++++++++++++++
 .../system/keyfromlabel/tests_sh_keyfromlabel.py   | 14 ++++++++++++++
 .../system/keymgr2kasp/tests_sh_keymgr2kasp.py     | 14 ++++++++++++++
 bin/tests/system/legacy/tests_sh_legacy.py         | 14 ++++++++++++++
 bin/tests/system/limits/tests_sh_limits.py         | 14 ++++++++++++++
 .../system/logfileconfig/tests_sh_logfileconfig.py | 14 ++++++++++++++
 bin/tests/system/masterfile/tests_sh_masterfile.py | 14 ++++++++++++++
 .../system/masterformat/tests_sh_masterformat.py   | 14 ++++++++++++++
 bin/tests/system/metadata/tests_sh_metadata.py     | 14 ++++++++++++++
 bin/tests/system/mirror/tests_sh_mirror.py         | 14 ++++++++++++++
 bin/tests/system/mkeys/tests_sh_mkeys.py           | 14 ++++++++++++++
 bin/tests/system/names/tests_sh_names.py           | 14 ++++++++++++++
 bin/tests/system/notify/tests_sh_notify.py         | 14 ++++++++++++++
 bin/tests/system/nsec3/tests_sh_nsec3.py           | 14 ++++++++++++++
 bin/tests/system/nslookup/tests_sh_nslookup.py     | 14 ++++++++++++++
 bin/tests/system/nsupdate/tests_sh_nsupdate.py     | 14 ++++++++++++++
 bin/tests/system/nzd2nzf/tests_sh_nzd2nzf.py       | 14 ++++++++++++++
 bin/tests/system/padding/tests_sh_padding.py       | 14 ++++++++++++++
 bin/tests/system/pending/tests_sh_pending.py       | 14 ++++++++++++++
 bin/tests/system/pipelined/tests_sh_pipelined.py   | 14 ++++++++++++++
 bin/tests/system/qmin/tests_sh_qmin.py             | 14 ++++++++++++++
 bin/tests/system/reclimit/tests_sh_reclimit.py     | 14 ++++++++++++++
 bin/tests/system/redirect/tests_sh_redirect.py     | 14 ++++++++++++++
 bin/tests/system/resolver/tests_sh_resolver.py     | 14 ++++++++++++++
 bin/tests/system/rndc/tests_sh_rndc.py             | 14 ++++++++++++++
 .../rootkeysentinel/tests_sh_rootkeysentinel.py    | 14 ++++++++++++++
 bin/tests/system/rpz/tests_sh_rpz.py               | 14 ++++++++++++++
 bin/tests/system/rpzrecurse/tests_sh_rpzrecurse.py | 14 ++++++++++++++
 bin/tests/system/rrchecker/tests_sh_rrchecker.py   | 14 ++++++++++++++
 bin/tests/system/rrl/tests_sh_rrl.py               | 14 ++++++++++++++
 bin/tests/system/rrsetorder/tests_sh_rrsetorder.py | 14 ++++++++++++++
 .../rsabigexponent/tests_sh_rsabigexponent.py      | 14 ++++++++++++++
 bin/tests/system/run.sh.in                         |  2 +-
 bin/tests/system/runtime/tests_sh_runtime.py       | 14 ++++++++++++++
 .../system/serve-stale/tests_sh_serve_stale.py     | 14 ++++++++++++++
 bin/tests/system/sfcache/tests_sh_sfcache.py       | 14 ++++++++++++++
 bin/tests/system/smartsign/tests_sh_smartsign.py   | 14 ++++++++++++++
 bin/tests/system/sortlist/tests_sh_sortlist.py     | 14 ++++++++++++++
 bin/tests/system/spf/tests_sh_spf.py               | 14 ++++++++++++++
 bin/tests/system/staticstub/tests_sh_staticstub.py | 14 ++++++++++++++
 bin/tests/system/statistics/tests_sh_statistics.py | 14 ++++++++++++++
 .../system/statschannel/tests_sh_statschannel.py   | 14 ++++++++++++++
 bin/tests/system/stress/tests_sh_stress.py         | 14 ++++++++++++++
 bin/tests/system/stub/tests_sh_stub.py             | 14 ++++++++++++++
 .../synthfromdnssec/tests_sh_synthfromdnssec.py    | 14 ++++++++++++++
 bin/tests/system/tcp/tests_sh_tcp.py               | 14 ++++++++++++++
 bin/tests/system/tkey/tests_sh_tkey.py             | 14 ++++++++++++++
 bin/tests/system/tools/tests_sh_tools.py           | 14 ++++++++++++++
 .../system/transport-acl/tests_sh_transport_acl.py | 14 ++++++++++++++
 bin/tests/system/tsig/tests_sh_tsig.py             | 14 ++++++++++++++
 bin/tests/system/tsiggss/tests_sh_tsiggss.py       | 14 ++++++++++++++
 bin/tests/system/unknown/tests_sh_unknown.py       | 14 ++++++++++++++
 bin/tests/system/upforwd/tests_sh_upforwd.py       | 14 ++++++++++++++
 bin/tests/system/verify/tests_sh_verify.py         | 14 ++++++++++++++
 bin/tests/system/views/tests_sh_views.py           | 14 ++++++++++++++
 bin/tests/system/wildcard/tests_sh_wildcard.py     | 14 ++++++++++++++
 bin/tests/system/xfer/tests_sh_xfer.py             | 14 ++++++++++++++
 bin/tests/system/xferquota/tests_sh_xferquota.py   | 14 ++++++++++++++
 bin/tests/system/zero/tests_sh_zero.py             | 14 ++++++++++++++
 bin/tests/system/zonechecks/tests_sh_zonechecks.py | 14 ++++++++++++++
 107 files changed, 1480 insertions(+), 1 deletion(-)
 create mode 100644 bin/tests/system/acl/tests_sh_acl.py
 create mode 100644 bin/tests/system/additional/tests_sh_additional.py
 create mode 100644 bin/tests/system/addzone/tests_sh_addzone.py
 create mode 100644 bin/tests/system/allow-query/tests_sh_allowquery.py
 create mode 100644 bin/tests/system/auth/tests_sh_auth.py
 create mode 100644 bin/tests/system/autosign/tests_sh_autosign.py
 create mode 100644 bin/tests/system/builtin/tests_sh_builtin.py
 create mode 100644 bin/tests/system/cacheclean/tests_sh_cacheclean.py
 create mode 100644 bin/tests/system/case/tests_sh_case.py
 create mode 100644 bin/tests/system/catz/tests_sh_catz.py
 create mode 100644 bin/tests/system/cds/tests_sh_cds.py
 create mode 100644 bin/tests/system/chain/tests_sh_chain.py
 create mode 100644 bin/tests/system/checkconf/tests_sh_checkconf.py
 create mode 100644 bin/tests/system/checknames/tests_sh_checknames.py
 create mode 100644 bin/tests/system/checkzone/tests_sh_checkzone.py
 create mode 100644 bin/tests/system/cookie/tests_sh_cookie.py
 create mode 100644 bin/tests/system/database/tests_sh_database.py
 create mode 100644 bin/tests/system/dialup/tests_sh_dialup.py
 create mode 100644 bin/tests/system/digdelv/tests_sh_digdelv.py
 create mode 100644 bin/tests/system/dlzexternal/tests_sh_dlzexternal.py
 create mode 100644 bin/tests/system/dns64/tests_sh_dns64.py
 create mode 100644 bin/tests/system/dnssec/tests_sh_dnssec.py
 create mode 100644 bin/tests/system/dnstap/tests_sh_dnstap.py
 create mode 100644 bin/tests/system/doth/tests_sh_doth.py
 create mode 100644 bin/tests/system/dsdigest/tests_sh_dsdigest.py
 create mode 100644 bin/tests/system/dupsigs/tests_sh_dupsigs.py
 create mode 100644 bin/tests/system/dyndb/tests_sh_dyndb.py
 create mode 100644 bin/tests/system/ecdsa/tests_sh_ecdsa.py
 create mode 100644 bin/tests/system/eddsa/tests_sh_eddsa.py
 create mode 100644 bin/tests/system/ednscompliance/tests_sh_ednscompliance.py
 create mode 100644 bin/tests/system/emptyzones/tests_sh_emptyzones.py
 create mode 100644 bin/tests/system/enginepkcs11/tests_sh_enginepkcs11.py
 create mode 100644 bin/tests/system/fetchlimit/tests_sh_fetchlimit.py
 create mode 100644 bin/tests/system/filter-aaaa/tests_sh_filter_aaaa.py
 create mode 100644 bin/tests/system/formerr/tests_sh_formerr.py
 create mode 100644 bin/tests/system/forward/tests_sh_forward.py
 create mode 100644 bin/tests/system/geoip2/tests_sh_geoip2.py
 create mode 100644 bin/tests/system/glue/tests_sh_glue.py
 create mode 100644 bin/tests/system/host/tests_sh_host.py
 create mode 100644 bin/tests/system/idna/tests_sh_idna.py
 create mode 100644 bin/tests/system/include-multiplecfg/tests_sh_include_multiplecfg.py
 create mode 100644 bin/tests/system/inline/tests_sh_inline.py
 create mode 100644 bin/tests/system/integrity/tests_sh_integrity.py
 create mode 100644 bin/tests/system/ixfr/tests_sh_ixfr.py
 create mode 100644 bin/tests/system/journal/tests_sh_journal.py
 create mode 100644 bin/tests/system/kasp/tests_sh_kasp.py
 create mode 100644 bin/tests/system/keepalive/tests_sh_keepalive.py
 create mode 100644 bin/tests/system/keyfromlabel/tests_sh_keyfromlabel.py
 create mode 100644 bin/tests/system/keymgr2kasp/tests_sh_keymgr2kasp.py
 create mode 100644 bin/tests/system/legacy/tests_sh_legacy.py
 create mode 100644 bin/tests/system/limits/tests_sh_limits.py
 create mode 100644 bin/tests/system/logfileconfig/tests_sh_logfileconfig.py
 create mode 100644 bin/tests/system/masterfile/tests_sh_masterfile.py
 create mode 100644 bin/tests/system/masterformat/tests_sh_masterformat.py
 create mode 100644 bin/tests/system/metadata/tests_sh_metadata.py
 create mode 100644 bin/tests/system/mirror/tests_sh_mirror.py
 create mode 100644 bin/tests/system/mkeys/tests_sh_mkeys.py
 create mode 100644 bin/tests/system/names/tests_sh_names.py
 create mode 100644 bin/tests/system/notify/tests_sh_notify.py
 create mode 100644 bin/tests/system/nsec3/tests_sh_nsec3.py
 create mode 100644 bin/tests/system/nslookup/tests_sh_nslookup.py
 create mode 100644 bin/tests/system/nsupdate/tests_sh_nsupdate.py
 create mode 100644 bin/tests/system/nzd2nzf/tests_sh_nzd2nzf.py
 create mode 100644 bin/tests/system/padding/tests_sh_padding.py
 create mode 100644 bin/tests/system/pending/tests_sh_pending.py
 create mode 100644 bin/tests/system/pipelined/tests_sh_pipelined.py
 create mode 100644 bin/tests/system/qmin/tests_sh_qmin.py
 create mode 100644 bin/tests/system/reclimit/tests_sh_reclimit.py
 create mode 100644 bin/tests/system/redirect/tests_sh_redirect.py
 create mode 100644 bin/tests/system/resolver/tests_sh_resolver.py
 create mode 100644 bin/tests/system/rndc/tests_sh_rndc.py
 create mode 100644 bin/tests/system/rootkeysentinel/tests_sh_rootkeysentinel.py
 create mode 100644 bin/tests/system/rpz/tests_sh_rpz.py
 create mode 100644 bin/tests/system/rpzrecurse/tests_sh_rpzrecurse.py
 create mode 100644 bin/tests/system/rrchecker/tests_sh_rrchecker.py
 create mode 100644 bin/tests/system/rrl/tests_sh_rrl.py
 create mode 100644 bin/tests/system/rrsetorder/tests_sh_rrsetorder.py
 create mode 100644 bin/tests/system/rsabigexponent/tests_sh_rsabigexponent.py
 create mode 100644 bin/tests/system/runtime/tests_sh_runtime.py
 create mode 100644 bin/tests/system/serve-stale/tests_sh_serve_stale.py
 create mode 100644 bin/tests/system/sfcache/tests_sh_sfcache.py
 create mode 100644 bin/tests/system/smartsign/tests_sh_smartsign.py
 create mode 100644 bin/tests/system/sortlist/tests_sh_sortlist.py
 create mode 100644 bin/tests/system/spf/tests_sh_spf.py
 create mode 100644 bin/tests/system/staticstub/tests_sh_staticstub.py
 create mode 100644 bin/tests/system/statistics/tests_sh_statistics.py
 create mode 100644 bin/tests/system/statschannel/tests_sh_statschannel.py
 create mode 100644 bin/tests/system/stress/tests_sh_stress.py
 create mode 100644 bin/tests/system/stub/tests_sh_stub.py
 create mode 100644 bin/tests/system/synthfromdnssec/tests_sh_synthfromdnssec.py
 create mode 100644 bin/tests/system/tcp/tests_sh_tcp.py
 create mode 100644 bin/tests/system/tkey/tests_sh_tkey.py
 create mode 100644 bin/tests/system/tools/tests_sh_tools.py
 create mode 100644 bin/tests/system/transport-acl/tests_sh_transport_acl.py
 create mode 100644 bin/tests/system/tsig/tests_sh_tsig.py
 create mode 100644 bin/tests/system/tsiggss/tests_sh_tsiggss.py
 create mode 100644 bin/tests/system/unknown/tests_sh_unknown.py
 create mode 100644 bin/tests/system/upforwd/tests_sh_upforwd.py
 create mode 100644 bin/tests/system/verify/tests_sh_verify.py
 create mode 100644 bin/tests/system/views/tests_sh_views.py
 create mode 100644 bin/tests/system/wildcard/tests_sh_wildcard.py
 create mode 100644 bin/tests/system/xfer/tests_sh_xfer.py
 create mode 100644 bin/tests/system/xferquota/tests_sh_xferquota.py
 create mode 100644 bin/tests/system/zero/tests_sh_zero.py
 create mode 100644 bin/tests/system/zonechecks/tests_sh_zonechecks.py

diff --git a/bin/tests/system/acl/tests_sh_acl.py b/bin/tests/system/acl/tests_sh_acl.py
new file mode 100644
index 0000000000..2c98644e01
--- /dev/null
+++ b/bin/tests/system/acl/tests_sh_acl.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_acl(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/additional/tests_sh_additional.py b/bin/tests/system/additional/tests_sh_additional.py
new file mode 100644
index 0000000000..cdc38f4d81
--- /dev/null
+++ b/bin/tests/system/additional/tests_sh_additional.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_additional(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/addzone/tests_sh_addzone.py b/bin/tests/system/addzone/tests_sh_addzone.py
new file mode 100644
index 0000000000..dca8e7415c
--- /dev/null
+++ b/bin/tests/system/addzone/tests_sh_addzone.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_addzone(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/allow-query/tests_sh_allowquery.py b/bin/tests/system/allow-query/tests_sh_allowquery.py
new file mode 100644
index 0000000000..ce20d79a8e
--- /dev/null
+++ b/bin/tests/system/allow-query/tests_sh_allowquery.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_allowquery(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/auth/tests_sh_auth.py b/bin/tests/system/auth/tests_sh_auth.py
new file mode 100644
index 0000000000..97233fa27d
--- /dev/null
+++ b/bin/tests/system/auth/tests_sh_auth.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_auth(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/autosign/tests_sh_autosign.py b/bin/tests/system/autosign/tests_sh_autosign.py
new file mode 100644
index 0000000000..16dfc29caf
--- /dev/null
+++ b/bin/tests/system/autosign/tests_sh_autosign.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_autosign(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/builtin/tests_sh_builtin.py b/bin/tests/system/builtin/tests_sh_builtin.py
new file mode 100644
index 0000000000..2246cb4595
--- /dev/null
+++ b/bin/tests/system/builtin/tests_sh_builtin.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_builtin(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/cacheclean/tests_sh_cacheclean.py b/bin/tests/system/cacheclean/tests_sh_cacheclean.py
new file mode 100644
index 0000000000..e47157eb32
--- /dev/null
+++ b/bin/tests/system/cacheclean/tests_sh_cacheclean.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_cacheclean(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/case/tests_sh_case.py b/bin/tests/system/case/tests_sh_case.py
new file mode 100644
index 0000000000..bbe94d3773
--- /dev/null
+++ b/bin/tests/system/case/tests_sh_case.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_case(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/catz/tests_sh_catz.py b/bin/tests/system/catz/tests_sh_catz.py
new file mode 100644
index 0000000000..eae546fe45
--- /dev/null
+++ b/bin/tests/system/catz/tests_sh_catz.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_catz(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/cds/tests_sh_cds.py b/bin/tests/system/cds/tests_sh_cds.py
new file mode 100644
index 0000000000..d00a8ae51c
--- /dev/null
+++ b/bin/tests/system/cds/tests_sh_cds.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_cds(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/chain/tests_sh_chain.py b/bin/tests/system/chain/tests_sh_chain.py
new file mode 100644
index 0000000000..ca3c05794e
--- /dev/null
+++ b/bin/tests/system/chain/tests_sh_chain.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_chain(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/checkconf/tests_sh_checkconf.py b/bin/tests/system/checkconf/tests_sh_checkconf.py
new file mode 100644
index 0000000000..3a348ba2f8
--- /dev/null
+++ b/bin/tests/system/checkconf/tests_sh_checkconf.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_checkconf(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/checknames/tests_sh_checknames.py b/bin/tests/system/checknames/tests_sh_checknames.py
new file mode 100644
index 0000000000..e0e035b439
--- /dev/null
+++ b/bin/tests/system/checknames/tests_sh_checknames.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_checknames(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/checkzone/tests_sh_checkzone.py b/bin/tests/system/checkzone/tests_sh_checkzone.py
new file mode 100644
index 0000000000..87613cbb26
--- /dev/null
+++ b/bin/tests/system/checkzone/tests_sh_checkzone.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_checkzone(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index f2b5be828b..853a49fe4e 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -398,6 +398,15 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
         """Function to call a perl script with arguments."""
         return partial(_run_script, env, logger, system_test_dir, env["PERL"])
 
+    @pytest.fixture(scope="module")
+    def run_tests_sh(system_test_dir, shell):
+        """Utility function to execute tests.sh as a python test."""
+
+        def run_tests():
+            shell(f"{system_test_dir}/tests.sh")
+
+        return run_tests
+
     @pytest.fixture(scope="module", autouse=True)
     def system_test(  # pylint: disable=too-many-arguments,too-many-statements
         request,
diff --git a/bin/tests/system/cookie/tests_sh_cookie.py b/bin/tests/system/cookie/tests_sh_cookie.py
new file mode 100644
index 0000000000..2f1d029925
--- /dev/null
+++ b/bin/tests/system/cookie/tests_sh_cookie.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_cookie(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/database/tests_sh_database.py b/bin/tests/system/database/tests_sh_database.py
new file mode 100644
index 0000000000..b48469e3fb
--- /dev/null
+++ b/bin/tests/system/database/tests_sh_database.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_database(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/dialup/tests_sh_dialup.py b/bin/tests/system/dialup/tests_sh_dialup.py
new file mode 100644
index 0000000000..fa28e86dcf
--- /dev/null
+++ b/bin/tests/system/dialup/tests_sh_dialup.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_dialup(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/digdelv/tests_sh_digdelv.py b/bin/tests/system/digdelv/tests_sh_digdelv.py
new file mode 100644
index 0000000000..2973c26a9f
--- /dev/null
+++ b/bin/tests/system/digdelv/tests_sh_digdelv.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_digdelv(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/dlzexternal/tests_sh_dlzexternal.py b/bin/tests/system/dlzexternal/tests_sh_dlzexternal.py
new file mode 100644
index 0000000000..1c0f7e5a29
--- /dev/null
+++ b/bin/tests/system/dlzexternal/tests_sh_dlzexternal.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_dlzexternal(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/dns64/tests_sh_dns64.py b/bin/tests/system/dns64/tests_sh_dns64.py
new file mode 100644
index 0000000000..7eb152c7f4
--- /dev/null
+++ b/bin/tests/system/dns64/tests_sh_dns64.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_dns64(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/dnssec/tests_sh_dnssec.py b/bin/tests/system/dnssec/tests_sh_dnssec.py
new file mode 100644
index 0000000000..65c3d4341f
--- /dev/null
+++ b/bin/tests/system/dnssec/tests_sh_dnssec.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_dnssec(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/dnstap/tests_sh_dnstap.py b/bin/tests/system/dnstap/tests_sh_dnstap.py
new file mode 100644
index 0000000000..8094f0d6c8
--- /dev/null
+++ b/bin/tests/system/dnstap/tests_sh_dnstap.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_dnstap(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/doth/tests_sh_doth.py b/bin/tests/system/doth/tests_sh_doth.py
new file mode 100644
index 0000000000..ef87a06acd
--- /dev/null
+++ b/bin/tests/system/doth/tests_sh_doth.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_doth(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/dsdigest/tests_sh_dsdigest.py b/bin/tests/system/dsdigest/tests_sh_dsdigest.py
new file mode 100644
index 0000000000..348d704739
--- /dev/null
+++ b/bin/tests/system/dsdigest/tests_sh_dsdigest.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_dsdigest(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/dupsigs/tests_sh_dupsigs.py b/bin/tests/system/dupsigs/tests_sh_dupsigs.py
new file mode 100644
index 0000000000..1e065db3da
--- /dev/null
+++ b/bin/tests/system/dupsigs/tests_sh_dupsigs.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_dupsigs(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/dyndb/tests_sh_dyndb.py b/bin/tests/system/dyndb/tests_sh_dyndb.py
new file mode 100644
index 0000000000..ca8f7a1e5b
--- /dev/null
+++ b/bin/tests/system/dyndb/tests_sh_dyndb.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_dyndb(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/ecdsa/tests_sh_ecdsa.py b/bin/tests/system/ecdsa/tests_sh_ecdsa.py
new file mode 100644
index 0000000000..98b9eeec05
--- /dev/null
+++ b/bin/tests/system/ecdsa/tests_sh_ecdsa.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_ecdsa(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/eddsa/tests_sh_eddsa.py b/bin/tests/system/eddsa/tests_sh_eddsa.py
new file mode 100644
index 0000000000..6646e36791
--- /dev/null
+++ b/bin/tests/system/eddsa/tests_sh_eddsa.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_eddsa(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/ednscompliance/tests_sh_ednscompliance.py b/bin/tests/system/ednscompliance/tests_sh_ednscompliance.py
new file mode 100644
index 0000000000..8eea611c18
--- /dev/null
+++ b/bin/tests/system/ednscompliance/tests_sh_ednscompliance.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_ednscompliance(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/emptyzones/tests_sh_emptyzones.py b/bin/tests/system/emptyzones/tests_sh_emptyzones.py
new file mode 100644
index 0000000000..66dc9dc75d
--- /dev/null
+++ b/bin/tests/system/emptyzones/tests_sh_emptyzones.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_emptyzones(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/enginepkcs11/tests_sh_enginepkcs11.py b/bin/tests/system/enginepkcs11/tests_sh_enginepkcs11.py
new file mode 100644
index 0000000000..334142dde2
--- /dev/null
+++ b/bin/tests/system/enginepkcs11/tests_sh_enginepkcs11.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_enginepkcs11(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/fetchlimit/tests_sh_fetchlimit.py b/bin/tests/system/fetchlimit/tests_sh_fetchlimit.py
new file mode 100644
index 0000000000..6df4441b91
--- /dev/null
+++ b/bin/tests/system/fetchlimit/tests_sh_fetchlimit.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_fetchlimit(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/filter-aaaa/tests_sh_filter_aaaa.py b/bin/tests/system/filter-aaaa/tests_sh_filter_aaaa.py
new file mode 100644
index 0000000000..bc6a90df73
--- /dev/null
+++ b/bin/tests/system/filter-aaaa/tests_sh_filter_aaaa.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_filter_aaaa(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/formerr/tests_sh_formerr.py b/bin/tests/system/formerr/tests_sh_formerr.py
new file mode 100644
index 0000000000..73638561fb
--- /dev/null
+++ b/bin/tests/system/formerr/tests_sh_formerr.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_formerr(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/forward/tests_sh_forward.py b/bin/tests/system/forward/tests_sh_forward.py
new file mode 100644
index 0000000000..4380a498bf
--- /dev/null
+++ b/bin/tests/system/forward/tests_sh_forward.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_forward(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/geoip2/tests_sh_geoip2.py b/bin/tests/system/geoip2/tests_sh_geoip2.py
new file mode 100644
index 0000000000..7cac6a8af8
--- /dev/null
+++ b/bin/tests/system/geoip2/tests_sh_geoip2.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_geoip2(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/glue/tests_sh_glue.py b/bin/tests/system/glue/tests_sh_glue.py
new file mode 100644
index 0000000000..4f3ff04afc
--- /dev/null
+++ b/bin/tests/system/glue/tests_sh_glue.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_glue(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/host/tests_sh_host.py b/bin/tests/system/host/tests_sh_host.py
new file mode 100644
index 0000000000..56da3b1f21
--- /dev/null
+++ b/bin/tests/system/host/tests_sh_host.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_host(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/idna/tests_sh_idna.py b/bin/tests/system/idna/tests_sh_idna.py
new file mode 100644
index 0000000000..82a640ceae
--- /dev/null
+++ b/bin/tests/system/idna/tests_sh_idna.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_idna(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/include-multiplecfg/tests_sh_include_multiplecfg.py b/bin/tests/system/include-multiplecfg/tests_sh_include_multiplecfg.py
new file mode 100644
index 0000000000..e767c1d698
--- /dev/null
+++ b/bin/tests/system/include-multiplecfg/tests_sh_include_multiplecfg.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_include_multiplecfg(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/inline/tests_sh_inline.py b/bin/tests/system/inline/tests_sh_inline.py
new file mode 100644
index 0000000000..f7520b8792
--- /dev/null
+++ b/bin/tests/system/inline/tests_sh_inline.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_inline(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/integrity/tests_sh_integrity.py b/bin/tests/system/integrity/tests_sh_integrity.py
new file mode 100644
index 0000000000..7316af3c14
--- /dev/null
+++ b/bin/tests/system/integrity/tests_sh_integrity.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_integrity(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/ixfr/tests_sh_ixfr.py b/bin/tests/system/ixfr/tests_sh_ixfr.py
new file mode 100644
index 0000000000..2c7fc28792
--- /dev/null
+++ b/bin/tests/system/ixfr/tests_sh_ixfr.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_ixfr(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/journal/tests_sh_journal.py b/bin/tests/system/journal/tests_sh_journal.py
new file mode 100644
index 0000000000..e492e12b52
--- /dev/null
+++ b/bin/tests/system/journal/tests_sh_journal.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_journal(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/kasp/tests_sh_kasp.py b/bin/tests/system/kasp/tests_sh_kasp.py
new file mode 100644
index 0000000000..d01125d0c1
--- /dev/null
+++ b/bin/tests/system/kasp/tests_sh_kasp.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_kasp(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/keepalive/tests_sh_keepalive.py b/bin/tests/system/keepalive/tests_sh_keepalive.py
new file mode 100644
index 0000000000..a2433c4029
--- /dev/null
+++ b/bin/tests/system/keepalive/tests_sh_keepalive.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_keepalive(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/keyfromlabel/tests_sh_keyfromlabel.py b/bin/tests/system/keyfromlabel/tests_sh_keyfromlabel.py
new file mode 100644
index 0000000000..9cbf56e125
--- /dev/null
+++ b/bin/tests/system/keyfromlabel/tests_sh_keyfromlabel.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_keyfromlabel(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/keymgr2kasp/tests_sh_keymgr2kasp.py b/bin/tests/system/keymgr2kasp/tests_sh_keymgr2kasp.py
new file mode 100644
index 0000000000..ba9b667fde
--- /dev/null
+++ b/bin/tests/system/keymgr2kasp/tests_sh_keymgr2kasp.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_keymgr2kasp(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/legacy/tests_sh_legacy.py b/bin/tests/system/legacy/tests_sh_legacy.py
new file mode 100644
index 0000000000..ded36f049b
--- /dev/null
+++ b/bin/tests/system/legacy/tests_sh_legacy.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_legacy(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/limits/tests_sh_limits.py b/bin/tests/system/limits/tests_sh_limits.py
new file mode 100644
index 0000000000..c7fb5ba779
--- /dev/null
+++ b/bin/tests/system/limits/tests_sh_limits.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_limits(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/logfileconfig/tests_sh_logfileconfig.py b/bin/tests/system/logfileconfig/tests_sh_logfileconfig.py
new file mode 100644
index 0000000000..655a8eeb5a
--- /dev/null
+++ b/bin/tests/system/logfileconfig/tests_sh_logfileconfig.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_logfileconfig(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/masterfile/tests_sh_masterfile.py b/bin/tests/system/masterfile/tests_sh_masterfile.py
new file mode 100644
index 0000000000..1556892af3
--- /dev/null
+++ b/bin/tests/system/masterfile/tests_sh_masterfile.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_masterfile(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/masterformat/tests_sh_masterformat.py b/bin/tests/system/masterformat/tests_sh_masterformat.py
new file mode 100644
index 0000000000..b70f036d08
--- /dev/null
+++ b/bin/tests/system/masterformat/tests_sh_masterformat.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_masterformat(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/metadata/tests_sh_metadata.py b/bin/tests/system/metadata/tests_sh_metadata.py
new file mode 100644
index 0000000000..484328868e
--- /dev/null
+++ b/bin/tests/system/metadata/tests_sh_metadata.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_metadata(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/mirror/tests_sh_mirror.py b/bin/tests/system/mirror/tests_sh_mirror.py
new file mode 100644
index 0000000000..f8d6b9d247
--- /dev/null
+++ b/bin/tests/system/mirror/tests_sh_mirror.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_mirror(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/mkeys/tests_sh_mkeys.py b/bin/tests/system/mkeys/tests_sh_mkeys.py
new file mode 100644
index 0000000000..141ee838aa
--- /dev/null
+++ b/bin/tests/system/mkeys/tests_sh_mkeys.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_mkeys(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/names/tests_sh_names.py b/bin/tests/system/names/tests_sh_names.py
new file mode 100644
index 0000000000..ebeed2ff56
--- /dev/null
+++ b/bin/tests/system/names/tests_sh_names.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_names(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/notify/tests_sh_notify.py b/bin/tests/system/notify/tests_sh_notify.py
new file mode 100644
index 0000000000..105b2d8238
--- /dev/null
+++ b/bin/tests/system/notify/tests_sh_notify.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_notify(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/nsec3/tests_sh_nsec3.py b/bin/tests/system/nsec3/tests_sh_nsec3.py
new file mode 100644
index 0000000000..16df54112d
--- /dev/null
+++ b/bin/tests/system/nsec3/tests_sh_nsec3.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_nsec3(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/nslookup/tests_sh_nslookup.py b/bin/tests/system/nslookup/tests_sh_nslookup.py
new file mode 100644
index 0000000000..9553a82aee
--- /dev/null
+++ b/bin/tests/system/nslookup/tests_sh_nslookup.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_nslookup(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/nsupdate/tests_sh_nsupdate.py b/bin/tests/system/nsupdate/tests_sh_nsupdate.py
new file mode 100644
index 0000000000..897eb08316
--- /dev/null
+++ b/bin/tests/system/nsupdate/tests_sh_nsupdate.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_nsupdate(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/nzd2nzf/tests_sh_nzd2nzf.py b/bin/tests/system/nzd2nzf/tests_sh_nzd2nzf.py
new file mode 100644
index 0000000000..b0304516d0
--- /dev/null
+++ b/bin/tests/system/nzd2nzf/tests_sh_nzd2nzf.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_nzd2nzf(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/padding/tests_sh_padding.py b/bin/tests/system/padding/tests_sh_padding.py
new file mode 100644
index 0000000000..a2eec7cd60
--- /dev/null
+++ b/bin/tests/system/padding/tests_sh_padding.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_padding(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/pending/tests_sh_pending.py b/bin/tests/system/pending/tests_sh_pending.py
new file mode 100644
index 0000000000..54b5afb271
--- /dev/null
+++ b/bin/tests/system/pending/tests_sh_pending.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_pending(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/pipelined/tests_sh_pipelined.py b/bin/tests/system/pipelined/tests_sh_pipelined.py
new file mode 100644
index 0000000000..2d7fb928ee
--- /dev/null
+++ b/bin/tests/system/pipelined/tests_sh_pipelined.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_pipelined(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/qmin/tests_sh_qmin.py b/bin/tests/system/qmin/tests_sh_qmin.py
new file mode 100644
index 0000000000..2566c78e0f
--- /dev/null
+++ b/bin/tests/system/qmin/tests_sh_qmin.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_qmin(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/reclimit/tests_sh_reclimit.py b/bin/tests/system/reclimit/tests_sh_reclimit.py
new file mode 100644
index 0000000000..38cfbc6e7a
--- /dev/null
+++ b/bin/tests/system/reclimit/tests_sh_reclimit.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_reclimit(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/redirect/tests_sh_redirect.py b/bin/tests/system/redirect/tests_sh_redirect.py
new file mode 100644
index 0000000000..9009391236
--- /dev/null
+++ b/bin/tests/system/redirect/tests_sh_redirect.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_redirect(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/resolver/tests_sh_resolver.py b/bin/tests/system/resolver/tests_sh_resolver.py
new file mode 100644
index 0000000000..d8de300aa0
--- /dev/null
+++ b/bin/tests/system/resolver/tests_sh_resolver.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_resolver(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/rndc/tests_sh_rndc.py b/bin/tests/system/rndc/tests_sh_rndc.py
new file mode 100644
index 0000000000..ac6a4dc930
--- /dev/null
+++ b/bin/tests/system/rndc/tests_sh_rndc.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_rndc(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/rootkeysentinel/tests_sh_rootkeysentinel.py b/bin/tests/system/rootkeysentinel/tests_sh_rootkeysentinel.py
new file mode 100644
index 0000000000..d1090d0972
--- /dev/null
+++ b/bin/tests/system/rootkeysentinel/tests_sh_rootkeysentinel.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_rootkeysentinel(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/rpz/tests_sh_rpz.py b/bin/tests/system/rpz/tests_sh_rpz.py
new file mode 100644
index 0000000000..1d551b53a0
--- /dev/null
+++ b/bin/tests/system/rpz/tests_sh_rpz.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_rpz(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/rpzrecurse/tests_sh_rpzrecurse.py b/bin/tests/system/rpzrecurse/tests_sh_rpzrecurse.py
new file mode 100644
index 0000000000..903ac49820
--- /dev/null
+++ b/bin/tests/system/rpzrecurse/tests_sh_rpzrecurse.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_rpzrecurse(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/rrchecker/tests_sh_rrchecker.py b/bin/tests/system/rrchecker/tests_sh_rrchecker.py
new file mode 100644
index 0000000000..16368e7c82
--- /dev/null
+++ b/bin/tests/system/rrchecker/tests_sh_rrchecker.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_rrchecker(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/rrl/tests_sh_rrl.py b/bin/tests/system/rrl/tests_sh_rrl.py
new file mode 100644
index 0000000000..6c8edc55c2
--- /dev/null
+++ b/bin/tests/system/rrl/tests_sh_rrl.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_rrl(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/rrsetorder/tests_sh_rrsetorder.py b/bin/tests/system/rrsetorder/tests_sh_rrsetorder.py
new file mode 100644
index 0000000000..e413f97967
--- /dev/null
+++ b/bin/tests/system/rrsetorder/tests_sh_rrsetorder.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_rrsetorder(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/rsabigexponent/tests_sh_rsabigexponent.py b/bin/tests/system/rsabigexponent/tests_sh_rsabigexponent.py
new file mode 100644
index 0000000000..38ab3810d4
--- /dev/null
+++ b/bin/tests/system/rsabigexponent/tests_sh_rsabigexponent.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_rsabigexponent(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/run.sh.in b/bin/tests/system/run.sh.in
index 8ba6224536..b523637cdb 100644
--- a/bin/tests/system/run.sh.in
+++ b/bin/tests/system/run.sh.in
@@ -225,7 +225,7 @@ fi
 
 if [ $status -eq 0 ]; then
     if [ -n "$PYTEST" ]; then
-        for test in $(cd "${systest}" && find . -name "tests*.py"); do
+        for test in $(cd "${systest}" && find . -name "tests*.py" ! -name "tests_sh_*.py"); do
             rm -f "$systest/$test.status"
             if start_servers; then
                 run=$((run+1))
diff --git a/bin/tests/system/runtime/tests_sh_runtime.py b/bin/tests/system/runtime/tests_sh_runtime.py
new file mode 100644
index 0000000000..089690e256
--- /dev/null
+++ b/bin/tests/system/runtime/tests_sh_runtime.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_runtime(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/serve-stale/tests_sh_serve_stale.py b/bin/tests/system/serve-stale/tests_sh_serve_stale.py
new file mode 100644
index 0000000000..3e6d3a1981
--- /dev/null
+++ b/bin/tests/system/serve-stale/tests_sh_serve_stale.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_serve_stale(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/sfcache/tests_sh_sfcache.py b/bin/tests/system/sfcache/tests_sh_sfcache.py
new file mode 100644
index 0000000000..d323b395c4
--- /dev/null
+++ b/bin/tests/system/sfcache/tests_sh_sfcache.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_sfcache(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/smartsign/tests_sh_smartsign.py b/bin/tests/system/smartsign/tests_sh_smartsign.py
new file mode 100644
index 0000000000..e291b64eec
--- /dev/null
+++ b/bin/tests/system/smartsign/tests_sh_smartsign.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_smartsign(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/sortlist/tests_sh_sortlist.py b/bin/tests/system/sortlist/tests_sh_sortlist.py
new file mode 100644
index 0000000000..5a66d7dde9
--- /dev/null
+++ b/bin/tests/system/sortlist/tests_sh_sortlist.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_sortlist(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/spf/tests_sh_spf.py b/bin/tests/system/spf/tests_sh_spf.py
new file mode 100644
index 0000000000..53c623d594
--- /dev/null
+++ b/bin/tests/system/spf/tests_sh_spf.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_spf(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/staticstub/tests_sh_staticstub.py b/bin/tests/system/staticstub/tests_sh_staticstub.py
new file mode 100644
index 0000000000..52c661c3e8
--- /dev/null
+++ b/bin/tests/system/staticstub/tests_sh_staticstub.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_staticstub(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/statistics/tests_sh_statistics.py b/bin/tests/system/statistics/tests_sh_statistics.py
new file mode 100644
index 0000000000..d87688f0dc
--- /dev/null
+++ b/bin/tests/system/statistics/tests_sh_statistics.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_statistics(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/statschannel/tests_sh_statschannel.py b/bin/tests/system/statschannel/tests_sh_statschannel.py
new file mode 100644
index 0000000000..85d5d14acf
--- /dev/null
+++ b/bin/tests/system/statschannel/tests_sh_statschannel.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_statschannel(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/stress/tests_sh_stress.py b/bin/tests/system/stress/tests_sh_stress.py
new file mode 100644
index 0000000000..283e2617b6
--- /dev/null
+++ b/bin/tests/system/stress/tests_sh_stress.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_stress(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/stub/tests_sh_stub.py b/bin/tests/system/stub/tests_sh_stub.py
new file mode 100644
index 0000000000..311e450687
--- /dev/null
+++ b/bin/tests/system/stub/tests_sh_stub.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_stub(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/synthfromdnssec/tests_sh_synthfromdnssec.py b/bin/tests/system/synthfromdnssec/tests_sh_synthfromdnssec.py
new file mode 100644
index 0000000000..5cbf8488ae
--- /dev/null
+++ b/bin/tests/system/synthfromdnssec/tests_sh_synthfromdnssec.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_synthfromdnssec(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/tcp/tests_sh_tcp.py b/bin/tests/system/tcp/tests_sh_tcp.py
new file mode 100644
index 0000000000..b1d797c321
--- /dev/null
+++ b/bin/tests/system/tcp/tests_sh_tcp.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_tcp(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/tkey/tests_sh_tkey.py b/bin/tests/system/tkey/tests_sh_tkey.py
new file mode 100644
index 0000000000..9a29490085
--- /dev/null
+++ b/bin/tests/system/tkey/tests_sh_tkey.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_tkey(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/tools/tests_sh_tools.py b/bin/tests/system/tools/tests_sh_tools.py
new file mode 100644
index 0000000000..319a0fe63a
--- /dev/null
+++ b/bin/tests/system/tools/tests_sh_tools.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_tools(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/transport-acl/tests_sh_transport_acl.py b/bin/tests/system/transport-acl/tests_sh_transport_acl.py
new file mode 100644
index 0000000000..400c85996e
--- /dev/null
+++ b/bin/tests/system/transport-acl/tests_sh_transport_acl.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_transport_acl(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/tsig/tests_sh_tsig.py b/bin/tests/system/tsig/tests_sh_tsig.py
new file mode 100644
index 0000000000..b421852c84
--- /dev/null
+++ b/bin/tests/system/tsig/tests_sh_tsig.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_tsig(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/tsiggss/tests_sh_tsiggss.py b/bin/tests/system/tsiggss/tests_sh_tsiggss.py
new file mode 100644
index 0000000000..926a9a9e6c
--- /dev/null
+++ b/bin/tests/system/tsiggss/tests_sh_tsiggss.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_tsiggss(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/unknown/tests_sh_unknown.py b/bin/tests/system/unknown/tests_sh_unknown.py
new file mode 100644
index 0000000000..fe1f4512b9
--- /dev/null
+++ b/bin/tests/system/unknown/tests_sh_unknown.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_unknown(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/upforwd/tests_sh_upforwd.py b/bin/tests/system/upforwd/tests_sh_upforwd.py
new file mode 100644
index 0000000000..35bcc64f34
--- /dev/null
+++ b/bin/tests/system/upforwd/tests_sh_upforwd.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_upforwd(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/verify/tests_sh_verify.py b/bin/tests/system/verify/tests_sh_verify.py
new file mode 100644
index 0000000000..2fd0c5c203
--- /dev/null
+++ b/bin/tests/system/verify/tests_sh_verify.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_verify(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/views/tests_sh_views.py b/bin/tests/system/views/tests_sh_views.py
new file mode 100644
index 0000000000..bfaacf4897
--- /dev/null
+++ b/bin/tests/system/views/tests_sh_views.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_views(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/wildcard/tests_sh_wildcard.py b/bin/tests/system/wildcard/tests_sh_wildcard.py
new file mode 100644
index 0000000000..89a9fef605
--- /dev/null
+++ b/bin/tests/system/wildcard/tests_sh_wildcard.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_wildcard(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/xfer/tests_sh_xfer.py b/bin/tests/system/xfer/tests_sh_xfer.py
new file mode 100644
index 0000000000..c83d6e9692
--- /dev/null
+++ b/bin/tests/system/xfer/tests_sh_xfer.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_xfer(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/xferquota/tests_sh_xferquota.py b/bin/tests/system/xferquota/tests_sh_xferquota.py
new file mode 100644
index 0000000000..762e182585
--- /dev/null
+++ b/bin/tests/system/xferquota/tests_sh_xferquota.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_xferquota(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/zero/tests_sh_zero.py b/bin/tests/system/zero/tests_sh_zero.py
new file mode 100644
index 0000000000..0e8ceb2626
--- /dev/null
+++ b/bin/tests/system/zero/tests_sh_zero.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_zero(run_tests_sh):
+    run_tests_sh()
diff --git a/bin/tests/system/zonechecks/tests_sh_zonechecks.py b/bin/tests/system/zonechecks/tests_sh_zonechecks.py
new file mode 100644
index 0000000000..28a7bb1c95
--- /dev/null
+++ b/bin/tests/system/zonechecks/tests_sh_zonechecks.py
@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+
+def test_zonechecks(run_tests_sh):
+    run_tests_sh()

From e9c15a5736ef486e171d88767207fbeacc20f7c5 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Wed, 11 Jan 2023 16:37:45 +0100
Subject: [PATCH 15/37] Update user docs with pytest system test runner

(cherry picked from commit dc84121004709824d5b5038b546e6073badc5b64)
---
 bin/tests/system/README | 47 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 46 insertions(+), 1 deletion(-)

diff --git a/bin/tests/system/README b/bin/tests/system/README
index 5db29de277..88685bf42e 100644
--- a/bin/tests/system/README
+++ b/bin/tests/system/README
@@ -67,9 +67,54 @@ then run
 ... as root.
 
 
-Running the System Tests
+Running the System Tests with pytest
 ===
 
+The pytest system test runner is currently in development, but it is the
+recommended way to run tests. Please report issues to QA.
+
+Running an Individual Test
+---
+
+pytest -k <test-name>
+
+Note that in comparison to the legacy test runner, some additional tests might
+be picked up when specifying just the system test directory name. To check
+which tests will be executed, you can use the `--collect-only` option. You
+might also be able to find a more specific test name to provide to ensure only
+your desired test is executed. See help for `-k` option in `pytest --help` for
+more info.
+
+It is also possible to run a single individual pytest test case. For example,
+you can use the name test_sslyze_dot to execute just the test_sslyze_dot()
+function from doth/tests_sslyze.py. The entire needed setup and teardown will
+be handled by the framework.
+
+Running All the System Tests
+---
+
+Issuing plain `pytest` command without any argument will execute all tests
+sequenatially. To execute them in parallel, ensure you have pytest-xdist
+installed and run:
+
+pytest --dist loadscope -n <number-of-workers>
+
+It is vital to provide the `--dist loadscope` option when running the tests in
+parallel to ensure tests from a single module are executed from the same
+thread. Otherwise, there's a risk of port contention and inefficient use of
+resources.
+
+
+Running the System Tests Using the Legacy Runner
+===
+
+!!! WARNING !!!
+---
+The legacy way to run system tests is currently being reworked into a pytest
+system test runner described in the previous section. The contents of this
+section might be out of date and no longer applicable. Please try and use the
+pytest runner if possible and report issues and missing features.
+
 Running an Individual Test
 ---
 The tests can be run individually using the following command:

From 2c167bb35f6654c675cc6e8e4b391574980f2de0 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Fri, 13 Jan 2023 16:06:01 +0100
Subject: [PATCH 16/37] Add developer docs for pytest system test runner

(cherry picked from commit d1ef51f589b3cf83d1eac3581159f615048d31e5)
---
 bin/tests/system/README | 75 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 74 insertions(+), 1 deletion(-)

diff --git a/bin/tests/system/README b/bin/tests/system/README
index 88685bf42e..dea63308c1 100644
--- a/bin/tests/system/README
+++ b/bin/tests/system/README
@@ -708,8 +708,81 @@ completed.  To enable this, set the USE_VALGRIND environment variable to
 "helgrind" to run the Helgrind tool, or any other value to run the Memcheck
 tool.  To use "helgrind" effectively, build BIND with --disable-atomic.
 
+Developer Notes for pytest runner
+===
 
-Maintenance Notes
+Test discovery and collection
+---
+There are two distinct types of system tests. The first is a shell script
+tests.sh containing individual test cases executed sequentially and the
+success/failure is determined by return code. The second type is a regular
+pytest file which contains test functions.
+
+Dealing with the regular pytest files doesn't require any special consideration
+as long as the naming conventions are met. Discovering the tests.sh tests is
+more complicated.
+
+The chosen solution is to add a bit of glue for each system test. For every
+tests.sh, there is an accompanying tests_sh_*.py file that contains a test
+function which utilizes a custom run_tests_sh fixture to call the tests.sh
+script. Other solutions were tried and eventually rejected. While this
+introduces a bit of extra glue, it is the most portable, compatible and least
+complex solution.
+
+Module scope
+---
+Pytest fixtures can have a scope. The "module" scope is the most important for
+our use. A module is a python file which contains test functions. Every system
+test directory may contain multiple modules (i.e. tests_*.py files)!
+
+The server setup/teardown is done for a module. Bundling test cases together
+inside a single module may save some resources. However, test cases inside a
+single module can't be executed in parallel.
+
+It is possible to execute different modules defined within a single system test
+directory in parallel. This is possible thanks to executing the tests inside a
+temporary directory and proper port assignment to ensure there won't be any
+conflicts.
+
+Parallel execution
+---
+As mentioned in the previous section, test cases inside a single module can't
+be executed in parallel. To put it differently, all tests cases inside the same
+module must be performed by the same worker/thread. Otherwise, server
+setup/teardown fixtures won't be shared and runtime issues due to port
+collisions are likely to occur.
+
+Pytest-xdist is used for executing pytest test cases in parallel using the `-n
+N_WORKERS` option. By default, xdist will distribute any test case to any
+worker, which would lead to the issue described above. Therefore, it is vital
+to use the `--dist loadscope` option which ensures that test cases within the
+same (module) scope will be handled by the same worker.
+
+$ pytest -n auto --dist loadscope
+
+Test selection
+---
+It is possible to run just a single pytest test case from any module. Use
+standard pytest facility to select the desired test case(s), i.e. pass a
+sufficiently unique identifier for `-k` parameter. You can also check which
+tests will be executed by using the `--collect-only` flag to debug your `-k`
+expression.
+
+Compatibility with older pytest version
+---
+Keep in mind that the pytest runner must work with ancient versions of pytest.
+When implementing new features, it is advisable to check feature support in
+pytest and pytest-xdist in older distributions first.
+
+As a general rule, any changes to the pytest runner need to keep working on all
+platforms in CI that use the pytest runner. As of 2023-01-13, the oldest
+supported version is whatever is available in EL8.
+
+We may need to add more compat code eventually to handle breaking upstream
+changes. For example, using request.fspath attribute is already deprecatred in
+latest pytest.
+
+Maintenance Notes for legacy runner
 ===
 This section is aimed at developers maintaining BIND's system test framework.
 

From 75a31555bd9bc00b561038256a4776b00e823132 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Tue, 22 Nov 2022 10:47:15 +0100
Subject: [PATCH 17/37] Use pytest system test runner in CI

Replace the legacy system test runner by the pytest system test runner.
Since EL7 and OpenBSD have only ancient versions of pytest / xdist, keep
using the legacy test runner there for now.

Out of tree tests aren't supported by the pytest runner yet. Use the
legacy test runner for that purpose as well.

Use awk to display failures and errors at the end of the log for
convenience, since pytest displays them first, which makes them
difficult to find.

(cherry picked from commit 4bc2b3be48e23152cdced9517f7b1d2839fde8f6)
---
 .gitlab-ci.yml | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0ebe858d2a..9ccc7c7a40 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -249,6 +249,10 @@ stages:
   - PYTHON="$(source bin/tests/system/conf.sh; echo $PYTHON)"
   - test -x "$PYTHON"
 
+.find_pytest: &find_pytest
+  - PYTEST="$(source bin/tests/system/conf.sh; echo $PYTEST)"
+  - test -x "$PYTEST"
+
 .parse_tsan: &parse_tsan
     - find -name 'tsan.*' -exec "$PYTHON" util/parse_tsan.py {} \;
 
@@ -313,6 +317,17 @@ stages:
   before_script:
     - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}"
     - *setup_interfaces
+  script:
+    - *find_pytest
+    - cd bin/tests/system
+    - >
+      "$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" --dist loadscope | tee pytest.out.txt
+    - '( ! grep -F "grep: warning:" pytest.out.txt )'
+  after_script:
+    - awk '/^=+ FAILURES =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
+    - awk '/^=+ ERRORS =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
+
+.system_test_legacy: &system_test_legacy
   script:
     - cd bin/tests/system
     - make -j${TEST_PARALLEL_JOBS:-1} -k check V=1
@@ -332,6 +347,8 @@ stages:
   <<: *system_test_common
   artifacts:
     untracked: true
+    exclude:
+      - "**/__pycache__/**/*"
     when: always
     reports:
       junit: junit.xml
@@ -340,12 +357,15 @@ stages:
   <<: *system_test_common
   artifacts:
     untracked: true
+    exclude:
+      - "**/__pycache__/**/*"
     when: always
 
 .system_test_tsan: &system_test_tsan_job
   <<: *system_test_common
   after_script:
-    - cat bin/tests/system/test-suite.log
+    - awk '/^=+ FAILURES =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
+    - awk '/^=+ ERRORS =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
     - find bin/tests/system -name "*dig.*" | xargs grep "error" || true
     - *find_python
     - *parse_tsan
@@ -353,6 +373,8 @@ stages:
       "$PYTHON" bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml
   artifacts:
     untracked: true
+    exclude:
+      - "**/__pycache__/**/*"
     when: always
     reports:
       junit: junit.xml
@@ -604,6 +626,7 @@ gcc:oraclelinux7:amd64:
 system:gcc:oraclelinux7:amd64:
   <<: *oraclelinux_7_amd64_image
   <<: *system_test_job
+  <<: *system_test_legacy
   needs:
     - job: gcc:oraclelinux7:amd64
       artifacts: true
@@ -830,6 +853,7 @@ system:gcc:out-of-tree:
       artifacts: true
   <<: *base_image
   <<: *system_test_job
+  <<: *system_test_legacy
   <<: *api_schedules_tags_triggers_web_triggering_rules
 
 unit:gcc:out-of-tree:
@@ -1191,6 +1215,7 @@ clang:openbsd:amd64:
 system:clang:openbsd:amd64:
   <<: *openbsd_amd64_image
   <<: *system_test_job
+  <<: *system_test_legacy
   <<: *api_schedules_triggers_web_triggering_rules
   variables:
     USER: gitlab-runner

From ac1e7eb40d188f0cf22cf8a3d1ec5cc6c6010274 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Fri, 13 Jan 2023 16:32:39 +0100
Subject: [PATCH 18/37] Invoke pytest runner from run.sh

Utilize developers' muscle memory to incentivize using the pytest runner
instead of the legacy one. The script also serves as basic examples of
how to run the pyest command to achieve the same results as the legacy
runner.

Invoking pytest directly should be the end goal, since it offers many
potentially useful options (refer to pytest --help).

(cherry picked from commit 4dbe8e53479f92bd467e02127b0f72bb6002c4ad)

(also manually added a additional run.sh -> legacy.run.sh renames)
---
 bin/tests/system/.gitignore                   |  2 +-
 bin/tests/system/Makefile.am                  |  4 +-
 bin/tests/system/README                       | 50 +++++------
 bin/tests/system/conf.sh.common               |  2 +-
 bin/tests/system/get_algorithms.py            |  2 +-
 .../system/{run.sh.in => legacy.run.sh.in}    |  0
 bin/tests/system/parallel.sh                  |  2 +-
 bin/tests/system/run.sh                       | 87 +++++++++++++++++++
 bin/tests/system/runall.sh                    |  2 +-
 bin/tests/system/runsequential.sh             |  2 +-
 configure.ac                                  |  4 +-
 11 files changed, 122 insertions(+), 35 deletions(-)
 rename bin/tests/system/{run.sh.in => legacy.run.sh.in} (100%)
 create mode 100755 bin/tests/system/run.sh

diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore
index 588c6dc0e2..b9cda1a53a 100644
--- a/bin/tests/system/.gitignore
+++ b/bin/tests/system/.gitignore
@@ -17,7 +17,7 @@ parallel.mk
 /*.log
 /*.trs
 /resolve
-/run.sh
+/legacy.run.sh
 /run.log
 /start.sh
 /stop.sh
diff --git a/bin/tests/system/Makefile.am b/bin/tests/system/Makefile.am
index 937c50a382..f98be0b416 100644
--- a/bin/tests/system/Makefile.am
+++ b/bin/tests/system/Makefile.am
@@ -247,9 +247,9 @@ LOG_DRIVER_V_1 = --verbose yes
 LOG_DRIVER = $(srcdir)/custom-test-driver
 AM_LOG_DRIVER_FLAGS = $(LOG_DRIVER_V)
 
-LOG_COMPILER = $(builddir)/run.sh
+LOG_COMPILER = $(builddir)/legacy.run.sh
 AM_LOG_FLAGS = -r
 
-$(TESTS): run.sh
+$(TESTS): legacy.run.sh
 
 test-local: check
diff --git a/bin/tests/system/README b/bin/tests/system/README
index dea63308c1..6a84940966 100644
--- a/bin/tests/system/README
+++ b/bin/tests/system/README
@@ -119,11 +119,11 @@ Running an Individual Test
 ---
 The tests can be run individually using the following command:
 
-    sh run.sh [flags] <test-name> [<test-arguments>]
+    sh legacy.run.sh [flags] <test-name> [<test-arguments>]
 
 e.g.
 
-    sh run.sh [flags] notify
+    sh legacy.run.sh [flags] notify
 
 Optional flags are:
 
@@ -165,10 +165,10 @@ To run all the system tests, enter the command:
     sh runall.sh [-c] [-n] [numproc]
 
 The optional flag "-c" forces colored output (by default system test output is
-not printed in color due to run.sh being piped through "tee").
+not printed in color due to legacy.run.sh being piped through "tee").
 
-The optional flag "-n" has the same effect as it does for "run.sh" - it causes
-the retention of all output files from all tests.
+The optional flag "-n" has the same effect as it does for "legacy.run.sh" - it
+causes the retention of all output files from all tests.
 
 The optional "numproc" argument specifies the maximum number of tests that can
 run in parallel.  The default is 1, which means that all of the tests run
@@ -288,9 +288,9 @@ Re-Running the Tests
 ---
 If there is a requirement to re-run a test (or the entire test suite), the
 files produced by the tests should be deleted first.  Normally, these files are
-deleted if the test succeeds but are retained on error.  The run.sh script
-automatically calls a given test's clean.sh script before invoking its setup.sh
-script.
+deleted if the test succeeds but are retained on error.  The legacy.run.sh
+script automatically calls a given test's clean.sh script before invoking its
+setup.sh script.
 
 Deletion of the files produced by the set of tests (e.g. after the execution
 of "runall.sh") can be carried out using the command:
@@ -330,8 +330,8 @@ tests.sh    Runs the actual tests.  This file is mandatory.
 
 clean.sh    Run at the end to clean up temporary files, but only if the test
             was completed successfully and its running was not inhibited by the
-	    "-n" switch being passed to "run.sh".  Otherwise the temporary
-	    files are left in place for inspection.
+            "-n" switch being passed to "legacy.run.sh".  Otherwise the
+            temporary files are left in place for inspection.
 
 ns<N>       These subdirectories contain test name servers that can be queried
 	    or can interact with each other.  The value of N indicates the
@@ -350,8 +350,8 @@ ans<N>      Like ns[X], but these are simple mock name servers implemented in
 Port Usage
 ---
 In order for the tests to run in parallel, each test requires a unique set of
-ports.  These are specified by the "-p" option passed to "run.sh", which sets
-environment variables that the scripts listed above can reference.
+ports.  These are specified by the "-p" option passed to "legacy.run.sh", which
+sets environment variables that the scripts listed above can reference.
 
 The convention used in the system tests is that the number passed is the start
 of a range of 100 ports.  The test is free to use the ports as required,
@@ -403,10 +403,10 @@ General
 directory.
 
 2. Arguments can be only passed to the script if the test is being run as a
-one-off with "run.sh".  In this case, everything on the command line after the
-name of the test is passed to each script.  For example, the command:
+one-off with "legacy.run.sh".  In this case, everything on the command line
+after the name of the test is passed to each script.  For example, the command:
 
-    sh run.sh -p 12300 mytest -D xyz
+    sh legacy.run.sh -p 12300 mytest -D xyz
 
 ... will run "mytest" with a port range of 12300 to 12399.  Each of the
 framework scripts provided by the test will be invoked using the remaining
@@ -577,8 +577,8 @@ Ideally, the directory numbers should start at 1 and work upwards.
 When running a test, the servers are started using "start.sh" (which is nothing
 more than a wrapper for start.pl).  The options for "start.pl" are documented
 in the header for that file, so will not be repeated here.  In summary, when
-invoked by "run.sh", start.pl looks for directories named "nsN" or "ansN" in
-the test directory and starts the servers it finds there.
+invoked by "legacy.run.sh", start.pl looks for directories named "nsN" or
+"ansN" in the test directory and starts the servers it finds there.
 
 
 "named" Command-Line Options
@@ -675,8 +675,8 @@ the options available are listed in the file's header and will not be repeated
 here.
 
 In summary though, the nameservers for a given test, if left running by
-specifying the "-k" flag to "run.sh" when the test is started, can be stopped
-by the command:
+specifying the "-k" flag to "legacy.run.sh" when the test is started, can be
+stopped by the command:
 
     sh stop.sh <test-name> [server]
 
@@ -788,10 +788,10 @@ This section is aimed at developers maintaining BIND's system test framework.
 
 Notes on Parallel Execution
 ---
-Although execution of an individual test is controlled by "run.sh", which
-executes the above shell scripts (and starts the relevant servers) for each
-test, the running of all tests in the test suite is controlled by the Makefile.
-("runall.sh" does little more than invoke "make" on the Makefile.)
+Although execution of an individual test is controlled by "legacy.run.sh",
+which executes the above shell scripts (and starts the relevant servers) for
+each test, the running of all tests in the test suite is controlled by the
+Makefile. ("runall.sh" does little more than invoke "make" on the Makefile.)
 
 All system tests are capable of being run in parallel.  For this to work, each
 test needs to use a unique set of ports.  To avoid the need to define which
@@ -801,7 +801,7 @@ the ports are assigned when the tests are run.  This is achieved by having the
 when "make check" is run, and contains a target for each test of the form:
 
     <test-name>:
-        @$(SHELL) run.sh -p <baseport> <test-name>
+        @$(SHELL) legacy.run.sh -p <baseport> <test-name>
 
 The <baseport> is unique and the values of <baseport> for each test are
 separated by at least 100 ports.
@@ -825,7 +825,7 @@ If the test fails, all these files are retained.  But if the test succeeds,
 they are cleaned up at different times:
 
 1. Files generated by the test itself are cleaned up by the test's own
-"clean.sh", which is called from "run.sh".
+"clean.sh", which is called from "legacy.run.sh".
 
 2. Files that may not be cleaned up if named exits abnormally can be removed
 using the "cleanall.sh" script.
diff --git a/bin/tests/system/conf.sh.common b/bin/tests/system/conf.sh.common
index 1f3bb68d3e..1499de6f34 100644
--- a/bin/tests/system/conf.sh.common
+++ b/bin/tests/system/conf.sh.common
@@ -702,7 +702,7 @@ get_named_xfer_stats() {
 #
 # Convenience function to copy a configuration file, replacing the tokens
 # QUERYPORT, CONTROLPORT and EXTRAPORT[1-8] with the values of the equivalent
-# environment variables. (These values are set by "run.sh", which calls the
+# environment variables. (These values are set by test runner, which calls the
 # scripts invoking this function.)
 #
 # Usage:
diff --git a/bin/tests/system/get_algorithms.py b/bin/tests/system/get_algorithms.py
index e21e208ce9..b2a060df63 100755
--- a/bin/tests/system/get_algorithms.py
+++ b/bin/tests/system/get_algorithms.py
@@ -230,7 +230,7 @@ def main():
     except Exception:
         # if anything goes wrong, the conf.sh ignores error codes, so make sure
         # we set an environment variable to an error value that can be checked
-        # later by run.sh
+        # later by the test runner and/or tests themselves
         print("export ALGORITHM_SET=error")
         raise
     for name, value in algs_env.items():
diff --git a/bin/tests/system/run.sh.in b/bin/tests/system/legacy.run.sh.in
similarity index 100%
rename from bin/tests/system/run.sh.in
rename to bin/tests/system/legacy.run.sh.in
diff --git a/bin/tests/system/parallel.sh b/bin/tests/system/parallel.sh
index 16b0b49c94..11a424f9b1 100644
--- a/bin/tests/system/parallel.sh
+++ b/bin/tests/system/parallel.sh
@@ -30,5 +30,5 @@ echo "test check: $PARALLELS"
 for directory in $PARALLELDIRS ; do
         echo
         echo "test-$(echo "$directory" | tr _ -): check_interfaces"
-        echo "	@${SHELL} ./run.sh -r $directory 2>&1 | tee test.output.$directory"
+        echo "	@${SHELL} ./legacy.run.sh -r $directory 2>&1 | tee test.output.$directory"
 done
diff --git a/bin/tests/system/run.sh b/bin/tests/system/run.sh
new file mode 100755
index 0000000000..27d122857f
--- /dev/null
+++ b/bin/tests/system/run.sh
@@ -0,0 +1,87 @@
+#!/usr/bin/python3
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+#
+# Run system test using the pytest runner. This is a simple wrapper around
+# pytest for convenience.
+#
+
+import argparse
+import sys
+import time
+
+import pytest
+
+
+def into_pytest_args(in_args):
+    args = []
+    if in_args.expression is None:
+        # running all tests - execute in parallel
+        args.extend(["-n", "auto"])
+        args.extend(["--dist", "loadscope"])
+    else:
+        args.extend(["-k", in_args.expression])
+    if in_args.noclean:
+        args.append("--noclean")
+    if in_args.keep:
+        print(
+            "ERROR -k / --keep option not implemented.\n"
+            "Please contact QA with your use-case and use ./legacy.run.sh in the meantime."
+        )
+        sys.exit(1)
+    return args
+
+
+def main():
+    print(
+        "----- WARNING -----\n"
+        "Using pytest system test runner\n\n"
+        'Please consider invoking "pytest" directly for more control:\n'
+        "  single test:     pytest -k dns64\n"
+        "  parallel tests:  pytest -n auto --dist loadscope\n\n"
+        "Alternately, use ./legacy.run.sh for the legacy system test runner.\n"
+    )
+
+    parser = argparse.ArgumentParser(
+        description="Wrapper script for launching system tests"
+    )
+    parser.add_argument(
+        "--noclean",
+        action="store_true",
+        help="don't clean tmpdir after test run",
+    )
+    parser.add_argument(
+        "-k",
+        "--keep",
+        action="store_true",
+        help="unused - not implemented",
+    )
+    parser.add_argument(
+        "expression",
+        type=str,
+        nargs="?",
+        help="select which test(s) to run",
+    )
+
+    args = into_pytest_args(parser.parse_args())
+    print(f"$ pytest {' '.join(args)}\n" "---------------------------\n")
+
+    time.sleep(2)  # force the user to stare at the warning message
+
+    sys.exit(pytest.main(args))
+
+
+if __name__ == "__main__":
+    main()
+
+# vim: set filetype=python :
diff --git a/bin/tests/system/runall.sh b/bin/tests/system/runall.sh
index a3bd54b26c..bbd072efc9 100755
--- a/bin/tests/system/runall.sh
+++ b/bin/tests/system/runall.sh
@@ -88,7 +88,7 @@ else
     (
 	status=0
         for testdir in $SUBDIRS; do
-            $SHELL run.sh -r "$testdir" || status=1
+            $SHELL legacy.run.sh -r "$testdir" || status=1
         done
 	echo "$status" > systests.status
     ) 2>&1 | tee "systests.output"
diff --git a/bin/tests/system/runsequential.sh b/bin/tests/system/runsequential.sh
index 4b99bd09b9..953bff0fac 100755
--- a/bin/tests/system/runsequential.sh
+++ b/bin/tests/system/runsequential.sh
@@ -22,5 +22,5 @@
 
 for d in $SEQUENTIALDIRS
 do
-    $SHELL run.sh "${@}" "$d" 2>&1 | tee "test.output.$d"
+    $SHELL legacy.run.sh "${@}" "$d" 2>&1 | tee "test.output.$d"
 done
diff --git a/configure.ac b/configure.ac
index 7b6adec282..5aa4e6824c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1591,8 +1591,8 @@ AC_CONFIG_FILES([bin/tests/Makefile
 
 AC_CONFIG_FILES([bin/tests/system/ifconfig.sh],
 		[chmod +x bin/tests/system/ifconfig.sh])
-AC_CONFIG_FILES([bin/tests/system/run.sh],
-		[chmod +x bin/tests/system/run.sh])
+AC_CONFIG_FILES([bin/tests/system/legacy.run.sh],
+		[chmod +x bin/tests/system/legacy.run.sh])
 AC_CONFIG_FILES([bin/tests/system/start.sh],
 		[chmod +x bin/tests/system/start.sh])
 AC_CONFIG_FILES([bin/tests/system/stop.sh],

From a388a689cfb31284e71f59d9fc8502af70e84309 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Tue, 28 Mar 2023 16:52:49 +0200
Subject: [PATCH 19/37] Ensure --dist=loadscope is used when running pytest in
 parallel

The loadscope setting is required for parallel execution of our system
tests using pytest. The option ensure that all tests within a single
(module) scope will be assigned to the same worker.

This is neccessary because the worker sets up the nameservers for all
the tests within a module scope. If tests from the same module would be
assigned to different workers, then the setup could happen multiple
times, causing a race condition. This happens because each module uses
deterministic port numbers for the nameservers.

(cherry picked from commit 8f57bce7afd963062c15bf66f4cb43bec523c0cf)
---
 .gitlab-ci.yml               |  2 +-
 bin/tests/system/README      | 17 +++++++----------
 bin/tests/system/conftest.py | 16 +++++++++++++++-
 bin/tests/system/run.sh      |  3 +--
 4 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9ccc7c7a40..f5aafb2065 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -321,7 +321,7 @@ stages:
     - *find_pytest
     - cd bin/tests/system
     - >
-      "$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" --dist loadscope | tee pytest.out.txt
+      "$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt
     - '( ! grep -F "grep: warning:" pytest.out.txt )'
   after_script:
     - awk '/^=+ FAILURES =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
diff --git a/bin/tests/system/README b/bin/tests/system/README
index 6a84940966..ae7354dfa8 100644
--- a/bin/tests/system/README
+++ b/bin/tests/system/README
@@ -97,12 +97,7 @@ Issuing plain `pytest` command without any argument will execute all tests
 sequenatially. To execute them in parallel, ensure you have pytest-xdist
 installed and run:
 
-pytest --dist loadscope -n <number-of-workers>
-
-It is vital to provide the `--dist loadscope` option when running the tests in
-parallel to ensure tests from a single module are executed from the same
-thread. Otherwise, there's a risk of port contention and inefficient use of
-resources.
+pytest -n <number-of-workers>
 
 
 Running the System Tests Using the Legacy Runner
@@ -754,11 +749,13 @@ collisions are likely to occur.
 
 Pytest-xdist is used for executing pytest test cases in parallel using the `-n
 N_WORKERS` option. By default, xdist will distribute any test case to any
-worker, which would lead to the issue described above. Therefore, it is vital
-to use the `--dist loadscope` option which ensures that test cases within the
-same (module) scope will be handled by the same worker.
+worker, which would lead to the issue described above. Therefore, conftest.py
+enforces equivalent of `--dist loadscope` option which ensures that test cases
+within the same (module) scope will be handled by the same worker. Parallelism
+is automatically disabled when xdist.scheduler.loadscope library is not
+available.
 
-$ pytest -n auto --dist loadscope
+$ pytest -n auto
 
 Test selection
 ---
diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 853a49fe4e..adee5540eb 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -130,7 +130,7 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
             help="don't remove the temporary test directories with artifacts",
         )
 
-    def pytest_configure():
+    def pytest_configure(config):
         # Ensure this hook only runs on the main pytest instance if xdist is
         # used to spawn other workers.
         if not XDIST_WORKER:
@@ -156,6 +156,20 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
                 raise exc
             logging.debug(proc.stdout)
 
+            if config.pluginmanager.has_plugin("xdist") and config.option.numprocesses:
+                # system tests depend on module scope for setup & teardown
+                # enforce use "loadscope" scheduler or disable paralelism
+                try:
+                    import xdist.scheduler.loadscope  # pylint: disable=unused-import
+                except ImportError:
+                    logging.debug(
+                        "xdist is too old and does not have "
+                        "scheduler.loadscope, disabling parallelism"
+                    )
+                    config.option.dist = "no"
+                else:
+                    config.option.dist = "loadscope"
+
     def pytest_ignore_collect(path):
         # System tests are executed in temporary directories inside
         # bin/tests/system. These temporary directories contain all files
diff --git a/bin/tests/system/run.sh b/bin/tests/system/run.sh
index 27d122857f..2f4fcf7dd5 100755
--- a/bin/tests/system/run.sh
+++ b/bin/tests/system/run.sh
@@ -28,7 +28,6 @@ def into_pytest_args(in_args):
     if in_args.expression is None:
         # running all tests - execute in parallel
         args.extend(["-n", "auto"])
-        args.extend(["--dist", "loadscope"])
     else:
         args.extend(["-k", in_args.expression])
     if in_args.noclean:
@@ -48,7 +47,7 @@ def main():
         "Using pytest system test runner\n\n"
         'Please consider invoking "pytest" directly for more control:\n'
         "  single test:     pytest -k dns64\n"
-        "  parallel tests:  pytest -n auto --dist loadscope\n\n"
+        "  parallel tests:  pytest -n auto\n\n"
         "Alternately, use ./legacy.run.sh for the legacy system test runner.\n"
     )
 

From a802e84cd5a2233d6f512e845f1ea4fe9d5f4ed5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Petr=20=C5=A0pa=C4=8Dek?= <pspacek@isc.org>
Date: Thu, 16 Mar 2023 18:40:59 +0100
Subject: [PATCH 20/37] Enable live logging for non-parallel pytest runs

This provides incremental output when test is running _without xdist_,
just like the old runner did.

With xdist the live output is not available, I believe because of
https://github.com/pytest-dev/pytest-xdist/issues/402
https://github.com/pytest-dev/pytest-xdist/pull/883 might help with
that, but I'm not going to hold my breath until it is available on
distros we use.

(cherry picked from commit d0619c7a188e1c9913f6552cf7e79a7f05813f7e)
---
 bin/tests/system/conftest.py | 53 ++++++++++++++++++++++--------------
 1 file changed, 33 insertions(+), 20 deletions(-)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index adee5540eb..eaeb74e4e5 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -90,6 +90,21 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
 
     # ---------------------- Module initialization ---------------------------
 
+    def avoid_duplicated_logs():
+        """
+        Remove direct root logger output to file descriptors.
+        This default is causing duplicates because all our messages go through
+        regular logging as well and are thus displayed twice.
+        """
+        todel = []
+        for handler in logging.root.handlers:
+            if handler.__class__ == logging.StreamHandler:
+                # Beware: As for pytest 7.2.2, LiveLogging and LogCapture
+                # handlers inherit from logging.StreamHandler
+                todel.append(handler)
+        for handler in todel:
+            logging.root.handlers.remove(handler)
+
     def parse_env(env_text):
         """Parse the POSIX env format into Python dictionary."""
         out = {}
@@ -283,6 +298,7 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
     @pytest.fixture(scope="module")
     def logger(system_test_name):
         """Logging facility specific to this test."""
+        avoid_duplicated_logs()
         return logging.getLogger(system_test_name)
 
     @pytest.fixture(scope="module")
@@ -378,29 +394,26 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
             raise FileNotFoundError(f"script {script} not found in {cwd}")
         logger.debug("running script: %s %s %s", interpreter, script, " ".join(args))
         logger.debug("  workdir: %s", cwd)
-        stdout = b""
         returncode = 1
-        try:
-            proc = subprocess.run(
-                [interpreter, script] + args,
-                env=env,
-                check=True,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,
-            )
-        except subprocess.CalledProcessError as exc:
-            stdout = exc.stdout
-            returncode = exc.returncode
-            raise exc
-        else:
-            stdout = proc.stdout
+
+        cmd = [interpreter, script] + args
+        with subprocess.Popen(
+            cmd,
+            env=env,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            bufsize=1,
+            universal_newlines=True,
+            errors="backslashreplace",
+        ) as proc:
+            if proc.stdout:
+                for line in proc.stdout:
+                    logger.info("    %s", line.rstrip("\n"))
+            proc.communicate()
             returncode = proc.returncode
-        finally:
-            if stdout:
-                for line in stdout.decode().splitlines():
-                    logger.debug("    %s", line)
+            if returncode:
+                raise subprocess.CalledProcessError(returncode, cmd)
             logger.debug("  exited with %d", returncode)
-        return proc
 
     @pytest.fixture(scope="module")
     def shell(env, system_test_dir, logger):

From 8627d41d4dddacc4b874b4b7d5d95217cd253f13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Petr=20=C5=A0pa=C4=8Dek?= <pspacek@isc.org>
Date: Fri, 17 Mar 2023 13:20:59 +0100
Subject: [PATCH 21/37] Include logs from failing tests in JUnit output

(cherry picked from commit 8952618262ca62a69fda45ed19fa8c706bb858b0)
---
 bin/tests/system/pytest.ini | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/tests/system/pytest.ini b/bin/tests/system/pytest.ini
index e1cb863d47..1559595852 100644
--- a/bin/tests/system/pytest.ini
+++ b/bin/tests/system/pytest.ini
@@ -16,3 +16,5 @@ log_date_format = %Y-%m-%d %H:%M:%S
 log_cli = 1
 log_level = INFO
 python_files = tests_*.py
+junit_logging = log
+junit_log_passing_tests = 0

From 6103ee988e3f696c66796d6f3e0468451bb8d064 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Tue, 28 Mar 2023 16:42:31 +0200
Subject: [PATCH 22/37] Run system tests sequentially if xdist is not available

(cherry picked from commit 1cc55d01c744aaa8fdbcd9b24c11ab9b7b34d5a4)
---
 bin/tests/system/run.sh | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/bin/tests/system/run.sh b/bin/tests/system/run.sh
index 2f4fcf7dd5..bbbe63ac29 100755
--- a/bin/tests/system/run.sh
+++ b/bin/tests/system/run.sh
@@ -26,8 +26,13 @@ import pytest
 def into_pytest_args(in_args):
     args = []
     if in_args.expression is None:
-        # running all tests - execute in parallel
-        args.extend(["-n", "auto"])
+        try:
+            import xdist
+        except ImportError:
+            pass
+        else:
+            # running all tests - execute in parallel
+            args.extend(["-n", "auto"])
     else:
         args.extend(["-k", in_args.expression])
     if in_args.noclean:

From 445b30b883bae8ff8fbe461decfea7282be1ab11 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Wed, 22 Mar 2023 15:52:05 +0100
Subject: [PATCH 23/37] Handle missing test_results due to pytest runner
 interrupt

If pytest execution is interrupted, the hook that exposes test_results
to the pytest session is never called so the results can't be
interpreted.

(cherry picked from commit 0a063f51d39ee72b46dba0795701de17012e942a)
---
 bin/tests/system/conftest.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index eaeb74e4e5..3d1cfb7c85 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -316,10 +316,17 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
         def get_test_result():
             """Aggregate test results from all individual tests from this module
             into a single result: failed > skipped > passed."""
+            try:
+                all_test_results = request.session.test_results
+            except AttributeError:
+                # This may happen if pytest execution is interrupted and
+                # pytest_runtest_makereport() is never called.
+                logger.debug("can't obtain test results, test run was interrupted")
+                return "error"
             test_results = {
-                node.nodeid: request.session.test_results[node.nodeid]
+                node.nodeid: all_test_results[node.nodeid]
                 for node in request.node.collect()
-                if node.nodeid in request.session.test_results
+                if node.nodeid in all_test_results
             }
             logger.debug(test_results)
             assert len(test_results)

From 23b2777067ae113ee03d56a98bbb9b7f0466bd32 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Thu, 23 Mar 2023 16:47:32 +0100
Subject: [PATCH 24/37] Display pytest failures for system:gcc:tarball

Since the tarball build&tests happen in a subdirectory, ensure the
after_script switches to it, so artifacts can be found properly.

(cherry picked from commit a6559176f1494bcdba2162e42f12a54f35710cb5)
---
 .gitlab-ci.yml | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f5aafb2065..411f22805e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -311,6 +311,10 @@ stages:
         sudo sh -x bin/tests/system/ifconfig.sh up;
       fi
 
+.display_pytest_failures: &display_pytest_failures
+    - awk '/^=+ FAILURES =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
+    - awk '/^=+ ERRORS =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
+
 .system_test_common: &system_test_common
   <<: *default_triggering_rules
   stage: system
@@ -324,8 +328,7 @@ stages:
       "$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt
     - '( ! grep -F "grep: warning:" pytest.out.txt )'
   after_script:
-    - awk '/^=+ FAILURES =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
-    - awk '/^=+ ERRORS =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
+    - *display_pytest_failures
 
 .system_test_legacy: &system_test_legacy
   script:
@@ -364,8 +367,7 @@ stages:
 .system_test_tsan: &system_test_tsan_job
   <<: *system_test_common
   after_script:
-    - awk '/^=+ FAILURES =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
-    - awk '/^=+ ERRORS =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
+    - *display_pytest_failures
     - find bin/tests/system -name "*dig.*" | xargs grep "error" || true
     - *find_python
     - *parse_tsan
@@ -888,6 +890,9 @@ system:gcc:tarball:
   before_script:
     - cd bind-*
     - *setup_interfaces
+  after_script:
+    - cd bind-*
+    - *display_pytest_failures
   needs:
     - job: gcc:tarball
       artifacts: true

From 0277c5098cb3d49cc509c993ccbf6d3f09a58298 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Petr=20=C5=A0pa=C4=8Dek?= <pspacek@isc.org>
Date: Tue, 28 Mar 2023 11:52:38 +0200
Subject: [PATCH 25/37] Use raw byte format of env variables in pytest

In order to avoid issues with decoding/encoding env variables due to
different encodings on different systems, deal with the environment
variables directly as bytes.

(cherry picked from commit 37ed9ad2f82cd4f2dadeb8d4271e533683a79659)
---
 bin/tests/system/conftest.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 3d1cfb7c85..f19b804e0c 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -83,7 +83,7 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
     LOG_FORMAT = "%(asctime)s %(levelname)7s:%(name)s  %(message)s"
     XDIST_WORKER = os.environ.get("PYTEST_XDIST_WORKER", "")
     FILE_DIR = os.path.abspath(Path(__file__).parent)
-    ENV_RE = re.compile("([^=]+)=(.*)")
+    ENV_RE = re.compile(b"([^=]+)=(.*)")
     PORT_MIN = 5001
     PORT_MAX = 32767
     PORTS_PER_TEST = 20
@@ -105,16 +105,16 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
         for handler in todel:
             logging.root.handlers.remove(handler)
 
-    def parse_env(env_text):
+    def parse_env(env_bytes):
         """Parse the POSIX env format into Python dictionary."""
         out = {}
-        for line in env_text.splitlines():
+        for line in env_bytes.splitlines():
             match = ENV_RE.match(line)
             if match:
                 out[match.groups()[0]] = match.groups()[1]
         return out
 
-    def get_env(cmd):
+    def get_env_bytes(cmd):
         try:
             proc = subprocess.run(
                 [cmd],
@@ -126,13 +126,13 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
         except subprocess.CalledProcessError as exc:
             logging.error("failed to get shell env: %s", exc)
             raise exc
-        env_text = proc.stdout.decode("utf-8")
-        return parse_env(env_text)
+        env_bytes = proc.stdout
+        return parse_env(env_bytes)
 
     # Read common environment variables for running tests from conf.sh.
     # FUTURE: Remove conf.sh entirely and define all variables in pytest only.
-    CONF_ENV = get_env(". ./conf.sh && env")
-    os.environ.update(CONF_ENV)
+    CONF_ENV = get_env_bytes(". ./conf.sh && env")
+    os.environb.update(CONF_ENV)
     logging.debug("conf.sh env: %s", CONF_ENV)
 
     # --------------------------- pytest hooks -------------------------------
@@ -283,7 +283,7 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
     @pytest.fixture(scope="module")
     def env(ports):
         """Dictionary containing environment variables for the test."""
-        env = CONF_ENV.copy()
+        env = os.environ.copy()
         env.update(ports)
         env["builddir"] = f"{env['TOP_BUILDDIR']}/bin/tests/system"
         env["srcdir"] = f"{env['TOP_SRCDIR']}/bin/tests/system"

From f0db1c27ba45e978e6981c168c286d848ee46b1a Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Mon, 3 Apr 2023 18:05:29 +0200
Subject: [PATCH 26/37] Ensure assertions and exceptions end up in system test
 log

If a test fails with an assertion failure or exception, its content
along with traceback is displayed in pytest output. This information
should be preserved in the test-specific logger for a given system test
to make it easier to debug test failures.

(cherry picked from commit 3a20e8d9902b79ea22c1ce7eeba74669062cacbe)
---
 bin/tests/system/conftest.py | 31 ++++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index f19b804e0c..6b9b74d288 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -193,6 +193,19 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
         # from previous runs could mess with the runner.
         return "_tmp_" in str(path)
 
+    class NodeResult:
+        def __init__(self, report=None):
+            self.outcome = None
+            self.messages = []
+            if report is not None:
+                self.update(report)
+
+        def update(self, report):
+            if self.outcome is None or report.outcome != "passed":
+                self.outcome = report.outcome
+            if report.longreprtext:
+                self.messages.append(report.longreprtext)
+
     @pytest.hookimpl(tryfirst=True, hookwrapper=True)
     def pytest_runtest_makereport(item):
         """Hook that is used to expose test results to session (for use in fixtures)."""
@@ -210,9 +223,8 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
             test_results = getattr(item.session, "test_results")
         except AttributeError:
             setattr(item.session, "test_results", test_results)
-        node_result = test_results.get(item.nodeid)
-        if node_result is None or report.outcome != "passed":
-            test_results[item.nodeid] = report.outcome
+        node_result = test_results.setdefault(item.nodeid, NodeResult())
+        node_result.update(report)
 
     # --------------------------- Fixtures -----------------------------------
 
@@ -328,15 +340,20 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
                 for node in request.node.collect()
                 if node.nodeid in all_test_results
             }
-            logger.debug(test_results)
             assert len(test_results)
-            failed = any(res == "failed" for res in test_results.values())
-            skipped = any(res == "skipped" for res in test_results.values())
+            messages = []
+            for node, result in test_results.items():
+                logger.debug("%s %s", result.outcome.upper(), node)
+                messages.extend(result.messages)
+            for message in messages:
+                logger.debug("\n" + message)
+            failed = any(res.outcome == "failed" for res in test_results.values())
+            skipped = any(res.outcome == "skipped" for res in test_results.values())
             if failed:
                 return "failed"
             if skipped:
                 return "skipped"
-            assert all(res == "passed" for res in test_results.values())
+            assert all(res.outcome == "passed" for res in test_results.values())
             return "passed"
 
         # Create a temporary directory with a copy of the original system test dir contents

From 862f6e043a69d5e65ecd96ea2cf43000dbc21fcb Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Tue, 4 Apr 2023 10:34:37 +0200
Subject: [PATCH 27/37] Mark selected statschannel tests as xfail

The test_zone_timers_secondary_json() and
test_zone_timers_secondary_xml() tests are affected by issue #3983. Due
to the way tests are run, they are only affected when executing them
with the pytest runner.

Strict mode is set for pytest runner, as it always fails there. The
strict mode ensures we'll catch the change when the it starts passing
once the underlying issue is fixed. It can't be set for the legacy
runner, since the test (incorrectly) passes there.

Related #3983

(cherry picked from commit 087a9b3c973aa503bac3598186714ced0b9ada60)
---
 bin/tests/system/statschannel/tests_json.py | 2 ++
 bin/tests/system/statschannel/tests_xml.py  | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/bin/tests/system/statschannel/tests_json.py b/bin/tests/system/statschannel/tests_json.py
index c4599258ea..6be264f49a 100755
--- a/bin/tests/system/statschannel/tests_json.py
+++ b/bin/tests/system/statschannel/tests_json.py
@@ -12,6 +12,7 @@
 # information regarding copyright ownership.
 
 from datetime import datetime
+import os
 
 import pytest
 
@@ -82,6 +83,7 @@ def test_zone_timers_primary_json(statsport):
     )
 
 
+@pytest.mark.xfail(reason="GL #3983", strict="LEGACY_TEST_RUNNER" not in os.environ)
 def test_zone_timers_secondary_json(statsport):
     generic.test_zone_timers_secondary(
         fetch_zones_json,
diff --git a/bin/tests/system/statschannel/tests_xml.py b/bin/tests/system/statschannel/tests_xml.py
index 7f0b37e846..6375d8a243 100755
--- a/bin/tests/system/statschannel/tests_xml.py
+++ b/bin/tests/system/statschannel/tests_xml.py
@@ -12,6 +12,7 @@
 # information regarding copyright ownership.
 
 from datetime import datetime
+import os
 import xml.etree.ElementTree as ET
 
 import pytest
@@ -112,6 +113,7 @@ def test_zone_timers_primary_xml(statsport):
     )
 
 
+@pytest.mark.xfail(reason="GL #3983", strict="LEGACY_TEST_RUNNER" not in os.environ)
 def test_zone_timers_secondary_xml(statsport):
     generic.test_zone_timers_secondary(
         fetch_zones_xml,

From 1acb8a3f399a5dc4d6d120f4758cebffa7b556cb Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Tue, 4 Apr 2023 17:44:10 +0200
Subject: [PATCH 28/37] Add test specific logger for pytests

The logger fixture is provided as a test-level logging facility which
can be easily passed to tests to enable capturing and/or displaying
messages from tests written in Python.

While this works optimally with the pytest runner, messages on INFO
level or above will also be visible when using the legacy runner.

(cherry picked from commit 952776b61f6129639a582695ea0b5ea2b2f62f15)
---
 bin/tests/system/README      |  11 ++++
 bin/tests/system/conftest.py | 102 +++++++++++++++++++----------------
 2 files changed, 68 insertions(+), 45 deletions(-)

diff --git a/bin/tests/system/README b/bin/tests/system/README
index ae7354dfa8..e4f8778dd2 100644
--- a/bin/tests/system/README
+++ b/bin/tests/system/README
@@ -739,6 +739,17 @@ directory in parallel. This is possible thanks to executing the tests inside a
 temporary directory and proper port assignment to ensure there won't be any
 conflicts.
 
+Test logging
+---
+Each module has a separate log which will be saved as pytest.log.txt in the
+temporary directory in which the test is executed. This log includes messages
+for this module setup/teardown as well as any logging from the tests using the
+`logger` fixture. Logging level DEBUG and above will be present in this log.
+
+In general, any log messages using INFO or above will also be printed out
+during pytest execution. In CI, the pytest output is also saved to
+pytest.out.txt in the bin/tests/system directory.
+
 Parallel execution
 ---
 As mentioned in the previous section, test cases inside a single module can't
diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 6b9b74d288..340ec47288 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -9,6 +9,7 @@
 # See the COPYRIGHT file distributed with this work for additional
 # information regarding copyright ownership.
 
+import logging
 import os
 import pytest
 
@@ -40,19 +41,25 @@ def control_port():
     return int(os.environ.get("CONTROLPORT", default=9953))
 
 
-# ======================= PYTEST SYSTEM TEST RUNNER ==========================
-# From this point onward, any setting, fixtures or functions only apply to the
-# new pytest runner. Ideally, these would be in a separate file. However, due
-# to how pytest works and how it's used by the legacy runner, the best approach
-# is to have everything in this file to avoid duplication and set the
-# LEGACY_TEST_RUNNER if pytest is executed from the legacy framework.
-#
-# FUTURE: Once legacy runner is no longer supported, remove the env var and
-# don't branch the code.
+if os.getenv("LEGACY_TEST_RUNNER", "0") != "0":
+
+    @pytest.fixture
+    def logger(request):
+        """Logging facility specific to a particular test."""
+        return logging.getLogger(request.node.name)
+
+else:
+    # ======================= PYTEST SYSTEM TEST RUNNER ==========================
+    # From this point onward, any setting, fixtures or functions only apply to the
+    # new pytest runner. Ideally, these would be in a separate file. However, due
+    # to how pytest works and how it's used by the legacy runner, the best approach
+    # is to have everything in this file to avoid duplication and set the
+    # LEGACY_TEST_RUNNER if pytest is executed from the legacy framework.
+    #
+    # FUTURE: Once legacy runner is no longer supported, remove the env var and
+    # don't branch the code.
 
-if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
     from functools import partial
-    import logging
     from pathlib import Path
     import re
     import shutil
@@ -308,13 +315,18 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
         return path.parent.name
 
     @pytest.fixture(scope="module")
-    def logger(system_test_name):
-        """Logging facility specific to this test."""
+    def mlogger(system_test_name):
+        """Logging facility specific to this test module."""
         avoid_duplicated_logs()
         return logging.getLogger(system_test_name)
 
+    @pytest.fixture
+    def logger(request, system_test_name):
+        """Logging facility specific to a particular test."""
+        return logging.getLogger(f"{system_test_name}.{request.node.name}")
+
     @pytest.fixture(scope="module")
-    def system_test_dir(request, env, system_test_name, logger):
+    def system_test_dir(request, env, system_test_name, mlogger):
         """
         Temporary directory for executing the test.
 
@@ -333,7 +345,7 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
             except AttributeError:
                 # This may happen if pytest execution is interrupted and
                 # pytest_runtest_makereport() is never called.
-                logger.debug("can't obtain test results, test run was interrupted")
+                mlogger.debug("can't obtain test results, test run was interrupted")
                 return "error"
             test_results = {
                 node.nodeid: all_test_results[node.nodeid]
@@ -343,10 +355,10 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
             assert len(test_results)
             messages = []
             for node, result in test_results.items():
-                logger.debug("%s %s", result.outcome.upper(), node)
+                mlogger.debug("%s %s", result.outcome.upper(), node)
                 messages.extend(result.messages)
             for message in messages:
-                logger.debug("\n" + message)
+                mlogger.debug("\n" + message)
             failed = any(res.outcome == "failed" for res in test_results.values())
             skipped = any(res.outcome == "skipped" for res in test_results.values())
             if failed:
@@ -365,41 +377,41 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
         shutil.copytree(system_test_root / system_test_name, testdir)
 
         # Configure logger to write to a file inside the temporary test directory
-        logger.handlers.clear()
-        logger.setLevel(logging.DEBUG)
+        mlogger.handlers.clear()
+        mlogger.setLevel(logging.DEBUG)
         handler = logging.FileHandler(testdir / "pytest.log.txt", mode="w")
         formatter = logging.Formatter(LOG_FORMAT)
         handler.setFormatter(formatter)
-        logger.addHandler(handler)
+        mlogger.addHandler(handler)
 
         # System tests are meant to be executed from their directory - switch to it.
         old_cwd = os.getcwd()
         os.chdir(testdir)
-        logger.info("switching to tmpdir: %s", testdir)
+        mlogger.info("switching to tmpdir: %s", testdir)
         try:
             yield testdir  # other fixtures / tests will execute here
         finally:
             os.chdir(old_cwd)
-            logger.debug("changed workdir to: %s", old_cwd)
+            mlogger.debug("changed workdir to: %s", old_cwd)
 
             result = get_test_result()
 
             # Clean temporary dir unless it should be kept
             if request.config.getoption("--noclean"):
-                logger.debug("--noclean requested, keeping temporary directory")
+                mlogger.debug("--noclean requested, keeping temporary directory")
             elif result == "failed":
-                logger.debug("test failure detected, keeping temporary directory")
+                mlogger.debug("test failure detected, keeping temporary directory")
             elif not request.node.stash[FIXTURE_OK]:
-                logger.debug(
+                mlogger.debug(
                     "test setup/teardown issue detected, keeping temporary directory"
                 )
             else:
-                logger.debug("deleting temporary directory")
+                mlogger.debug("deleting temporary directory")
                 shutil.rmtree(testdir)
 
     def _run_script(  # pylint: disable=too-many-arguments
         env,
-        logger,
+        mlogger,
         system_test_dir: Path,
         interpreter: str,
         script: str,
@@ -416,8 +428,8 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
         cwd = os.getcwd()
         if not path.exists():
             raise FileNotFoundError(f"script {script} not found in {cwd}")
-        logger.debug("running script: %s %s %s", interpreter, script, " ".join(args))
-        logger.debug("  workdir: %s", cwd)
+        mlogger.debug("running script: %s %s %s", interpreter, script, " ".join(args))
+        mlogger.debug("  workdir: %s", cwd)
         returncode = 1
 
         cmd = [interpreter, script] + args
@@ -432,22 +444,22 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
         ) as proc:
             if proc.stdout:
                 for line in proc.stdout:
-                    logger.info("    %s", line.rstrip("\n"))
+                    mlogger.info("    %s", line.rstrip("\n"))
             proc.communicate()
             returncode = proc.returncode
             if returncode:
                 raise subprocess.CalledProcessError(returncode, cmd)
-            logger.debug("  exited with %d", returncode)
+            mlogger.debug("  exited with %d", returncode)
 
     @pytest.fixture(scope="module")
-    def shell(env, system_test_dir, logger):
+    def shell(env, system_test_dir, mlogger):
         """Function to call a shell script with arguments."""
-        return partial(_run_script, env, logger, system_test_dir, env["SHELL"])
+        return partial(_run_script, env, mlogger, system_test_dir, env["SHELL"])
 
     @pytest.fixture(scope="module")
-    def perl(env, system_test_dir, logger):
+    def perl(env, system_test_dir, mlogger):
         """Function to call a perl script with arguments."""
-        return partial(_run_script, env, logger, system_test_dir, env["PERL"])
+        return partial(_run_script, env, mlogger, system_test_dir, env["PERL"])
 
     @pytest.fixture(scope="module")
     def run_tests_sh(system_test_dir, shell):
@@ -462,7 +474,7 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
     def system_test(  # pylint: disable=too-many-arguments,too-many-statements
         request,
         env: Dict[str, str],
-        logger,
+        mlogger,
         system_test_dir,
         shell,
         perl,
@@ -493,7 +505,7 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
             try:
                 perl("testsock.pl", ["-p", env["PORT"]])
             except subprocess.CalledProcessError as exc:
-                logger.error("testsock.pl: exited with code %d", exc.returncode)
+                mlogger.error("testsock.pl: exited with code %d", exc.returncode)
                 pytest.skip("Network interface aliases not set up.")
 
         def check_prerequisites():
@@ -510,34 +522,34 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
             except FileNotFoundError:
                 pass  # setup.sh is optional
             except subprocess.CalledProcessError as exc:
-                logger.error("Failed to run test setup")
+                mlogger.error("Failed to run test setup")
                 pytest.fail(f"setup.sh exited with {exc.returncode}")
 
         def start_servers():
             try:
                 perl("start.pl", ["--port", env["PORT"], system_test_dir.name])
             except subprocess.CalledProcessError as exc:
-                logger.error("Failed to start servers")
+                mlogger.error("Failed to start servers")
                 pytest.fail(f"start.pl exited with {exc.returncode}")
 
         def stop_servers():
             try:
                 perl("stop.pl", [system_test_dir.name])
             except subprocess.CalledProcessError as exc:
-                logger.error("Failed to stop servers")
+                mlogger.error("Failed to stop servers")
                 pytest.fail(f"stop.pl exited with {exc.returncode}")
 
         def get_core_dumps():
             try:
                 shell("get_core_dumps.sh", [system_test_dir.name])
             except subprocess.CalledProcessError as exc:
-                logger.error("Found core dumps")
+                mlogger.error("Found core dumps")
                 pytest.fail(f"get_core_dumps.sh exited with {exc.returncode}")
 
         os.environ.update(env)  # Ensure pytests have the same env vars as shell tests.
-        logger.info(f"test started: {request.node.name}")
+        mlogger.info(f"test started: {request.node.name}")
         port = int(env["PORT"])
-        logger.info("using port range: <%d, %d>", port, port + PORTS_PER_TEST - 1)
+        mlogger.info("using port range: <%d, %d>", port, port + PORTS_PER_TEST - 1)
 
         if not hasattr(request.node, "stash"):  # compatibility with pytest<7.0.0
             request.node.stash = {}  # use regular dict instead of pytest.Stash
@@ -555,10 +567,10 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
         setup_test()
         try:
             start_servers()
-            logger.debug("executing test(s)")
+            mlogger.debug("executing test(s)")
             yield
         finally:
-            logger.debug("test(s) finished")
+            mlogger.debug("test(s) finished")
             stop_servers()
             get_core_dumps()
             request.node.stash[FIXTURE_OK] = True

From b9596d2107bb62ca0756de1449fdf14aac9f581d Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Wed, 5 Apr 2023 10:48:57 +0200
Subject: [PATCH 29/37] Disable pylint check for too-few-public-methods

This check is overly aggressive and not really useful, especially for
non-Python codebase, where the primary use of Python is for testing.

(cherry picked from commit 6f134283eb3c16be6e056572cda7164822bb1102)
---
 .pylintrc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.pylintrc b/.pylintrc
index f9b1110547..07d503514d 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -7,3 +7,4 @@ disable=
     C0209, # consider-using-f-string
     C0415, # import-outside-toplevel
     R0801, # duplicate-code
+    R0903, # too-few-public-methods

From 16eeb36432340206253d92b327e344a8f0407417 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Wed, 5 Apr 2023 12:39:56 +0200
Subject: [PATCH 30/37] Execute long running system tests first

In order to take the most advantage of parallel execution of tests,
ensure certain long running tests are scheduled first.

The list of tests considered long-running was created empirically. In
addition to the test run time, its position in the default
(alphabetical) ordering was also taken into account.

(cherry picked from commit 99e2e50c0eddf2c16f9a684f4b90db0918a20b07)
---
 bin/tests/system/conftest.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 340ec47288..17693259d0 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -94,6 +94,16 @@ else:
     PORT_MIN = 5001
     PORT_MAX = 32767
     PORTS_PER_TEST = 20
+    PRIORITY_TESTS = [
+        # Tests that are scheduled first. Speeds up parallel execution.
+        "dupsigs/",
+        "rpz/",
+        "rpzrecurse/",
+        "serve-stale/",
+        "timeouts/",
+        "upforwd/",
+    ]
+    PRIORITY_TESTS_RE = re.compile("|".join(PRIORITY_TESTS))
 
     # ---------------------- Module initialization ---------------------------
 
@@ -200,6 +210,17 @@ else:
         # from previous runs could mess with the runner.
         return "_tmp_" in str(path)
 
+    def pytest_collection_modifyitems(items):
+        """Schedule long-running tests first to get more benefit from parallelism."""
+        priority = []
+        other = []
+        for item in items:
+            if PRIORITY_TESTS_RE.search(item.nodeid):
+                priority.append(item)
+            else:
+                other.append(item)
+        items[:] = priority + other
+
     class NodeResult:
         def __init__(self, report=None):
             self.outcome = None

From 5b5210854dfd63d440bfbedf3856c2be5c360bf7 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Wed, 12 Apr 2023 14:22:27 +0200
Subject: [PATCH 31/37] Capture log output during pytest runner setup

Instantiate a new logger that is used during pytest initialization /
configuration. This logging isn't handled by pytest itself, since it
happens outside of any tests or fixtures.

Root logger can't be reused for this purpose, because that would
duplicate the logs. Instead, create a conftest-specific logger for this
purpose.

Unfortunately, this introduces another log file,
pytest.conftest.log.txt, which contains only the logging from pytest
initialization. However, unless one is debugging the runner /
environment, there should be no need to investigate this file.

(cherry picked from commit 12c724ee073d9f9289fb339e252cf6da652c8e96)
---
 bin/tests/system/conftest.py | 31 ++++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 17693259d0..9bef5a1a4a 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -104,9 +104,24 @@ else:
         "upforwd/",
     ]
     PRIORITY_TESTS_RE = re.compile("|".join(PRIORITY_TESTS))
+    CONFTEST_LOGGER = logging.getLogger("conftest")
 
     # ---------------------- Module initialization ---------------------------
 
+    def init_pytest_conftest_logger(conftest_logger):
+        """
+        This initializes the conftest logger which is used for pytest setup
+        and configuration before tests are executed -- aka any logging in this
+        file that is _not_ module-specific.
+        """
+        conftest_logger.setLevel(logging.DEBUG)
+        file_handler = logging.FileHandler("pytest.conftest.log.txt")
+        file_handler.setLevel(logging.DEBUG)
+        file_handler.setFormatter(logging.Formatter(LOG_FORMAT))
+        conftest_logger.addHandler(file_handler)
+
+    init_pytest_conftest_logger(CONFTEST_LOGGER)
+
     def avoid_duplicated_logs():
         """
         Remove direct root logger output to file descriptors.
@@ -141,7 +156,7 @@ else:
                 stdout=subprocess.PIPE,
             )
         except subprocess.CalledProcessError as exc:
-            logging.error("failed to get shell env: %s", exc)
+            CONFTEST_LOGGER.error("failed to get shell env: %s", exc)
             raise exc
         env_bytes = proc.stdout
         return parse_env(env_bytes)
@@ -150,7 +165,9 @@ else:
     # FUTURE: Remove conf.sh entirely and define all variables in pytest only.
     CONF_ENV = get_env_bytes(". ./conf.sh && env")
     os.environb.update(CONF_ENV)
-    logging.debug("conf.sh env: %s", CONF_ENV)
+    CONFTEST_LOGGER.debug(
+        "variables in env: %s", ", ".join([str(key) for key in CONF_ENV])
+    )
 
     # --------------------------- pytest hooks -------------------------------
 
@@ -166,7 +183,7 @@ else:
         # Ensure this hook only runs on the main pytest instance if xdist is
         # used to spawn other workers.
         if not XDIST_WORKER:
-            logging.debug("compiling required files")
+            CONFTEST_LOGGER.debug("compiling required files")
             env = os.environ.copy()
             env["TESTS"] = ""  # disable automake test framework - compile-only
             try:
@@ -183,10 +200,10 @@ else:
                     env=env,
                 )
             except subprocess.CalledProcessError as exc:
-                logging.debug(exc.stdout)
-                logging.error("failed to compile test files: %s", exc)
+                CONFTEST_LOGGER.debug(exc.stdout)
+                CONFTEST_LOGGER.error("failed to compile test files: %s", exc)
                 raise exc
-            logging.debug(proc.stdout)
+            CONFTEST_LOGGER.debug(proc.stdout)
 
             if config.pluginmanager.has_plugin("xdist") and config.option.numprocesses:
                 # system tests depend on module scope for setup & teardown
@@ -194,7 +211,7 @@ else:
                 try:
                     import xdist.scheduler.loadscope  # pylint: disable=unused-import
                 except ImportError:
-                    logging.debug(
+                    CONFTEST_LOGGER.debug(
                         "xdist is too old and does not have "
                         "scheduler.loadscope, disabling parallelism"
                     )

From e2f7dbcbd1bc840ccf6ca502bc802d49896be386 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Wed, 12 Apr 2023 17:29:26 +0200
Subject: [PATCH 32/37] Remove "which" declaration from env vars in EL8+ tests

EL8+ systems declare "which" function using environment variables in the
/etc/profile.d/which2.sh file. Because of our suboptimal environment
variable detection, which is required in order to support the legacy
runner, these variables are picked up by the pytest runner.

If subprocesses are spawned with these environment variables set, it
will cause the following issue when they spawn yet another subprocess:

/bin/sh: which: line 1: syntax error: unexpected end of file
/bin/sh: error importing function definition for `which'

(cherry picked from commit 68b1c6877b6cf118967728eaf2748f304cb934b3)
---
 bin/tests/system/conftest.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 9bef5a1a4a..83a0130511 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -143,6 +143,10 @@ else:
         for line in env_bytes.splitlines():
             match = ENV_RE.match(line)
             if match:
+                # EL8+ workaround for https://access.redhat.com/solutions/6994985
+                # FUTURE: can be removed when we no longer need to parse env vars
+                if match.groups()[0] in [b"which_declare", b"BASH_FUNC_which%%"]:
+                    continue
                 out[match.groups()[0]] = match.groups()[1]
         return out
 

From de206d643657a1887480e80fe274b4e5b6f96ac1 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Fri, 14 Apr 2023 14:40:51 +0200
Subject: [PATCH 33/37] Add .log.txt to gitignore

It can be useful to append the .txt extension to logs. When this
extension is used, GitLab is able to set the proper content type on such
artifacts in CI. This makes it possible to display those files directly
in the browser rather than having to download them.

(cherry picked from commit 8d40156bb2c876a718fca4398182a3706c640901)
---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 7d4fd095d2..3edbde71e7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,7 @@
 *.la
 *.lo
 *.log
+*.log.txt
 *.o
 *.orig
 *.plist/ # ccc-analyzer store its results in .plist directories

From 6811f9704aac13ed9309d597e855a53c5a877b02 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Tue, 9 May 2023 13:32:13 +0200
Subject: [PATCH 34/37] Rewrite run.sh to invoke pytest in a system test
 directory

Previously, run.sh tried to use pytest's -k option for test selection.
The downside was that this filter expression matched any test case with
the given substring, rather than executing a system test suite with the
given name.

The run.sh has been rewritten to invoke pytest from a system test
directory instead. This behaves more consistently with the run.sh from
legacy system test framework.

run.sh is now also a shell script to avoid confusion regarding its
file extension.

(cherry picked from commit 1aaefc9cf40e391560fe34795e50eaf15c22e26c)
---
 bin/tests/system/run.sh | 84 +++++------------------------------------
 1 file changed, 10 insertions(+), 74 deletions(-)

diff --git a/bin/tests/system/run.sh b/bin/tests/system/run.sh
index bbbe63ac29..8e87c0f795 100755
--- a/bin/tests/system/run.sh
+++ b/bin/tests/system/run.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/bin/sh
 #
 # Copyright (C) Internet Systems Consortium, Inc. ("ISC")
 #
@@ -12,80 +12,16 @@
 # information regarding copyright ownership.
 
 #
-# Run system test using the pytest runner. This is a simple wrapper around
-# pytest for convenience.
+# Run a single system test using the pytest runner. This is a simple wrapper
+# around pytest for convenience.
 #
 
-import argparse
-import sys
-import time
+if [ -z "$1" ] || [ ! -d "$1" ]; then
+    echo "Usage: $0 system_test_dir [pytest_args]"
+    exit 2
+fi
 
-import pytest
+system_test_dir="$1"
+shift
 
-
-def into_pytest_args(in_args):
-    args = []
-    if in_args.expression is None:
-        try:
-            import xdist
-        except ImportError:
-            pass
-        else:
-            # running all tests - execute in parallel
-            args.extend(["-n", "auto"])
-    else:
-        args.extend(["-k", in_args.expression])
-    if in_args.noclean:
-        args.append("--noclean")
-    if in_args.keep:
-        print(
-            "ERROR -k / --keep option not implemented.\n"
-            "Please contact QA with your use-case and use ./legacy.run.sh in the meantime."
-        )
-        sys.exit(1)
-    return args
-
-
-def main():
-    print(
-        "----- WARNING -----\n"
-        "Using pytest system test runner\n\n"
-        'Please consider invoking "pytest" directly for more control:\n'
-        "  single test:     pytest -k dns64\n"
-        "  parallel tests:  pytest -n auto\n\n"
-        "Alternately, use ./legacy.run.sh for the legacy system test runner.\n"
-    )
-
-    parser = argparse.ArgumentParser(
-        description="Wrapper script for launching system tests"
-    )
-    parser.add_argument(
-        "--noclean",
-        action="store_true",
-        help="don't clean tmpdir after test run",
-    )
-    parser.add_argument(
-        "-k",
-        "--keep",
-        action="store_true",
-        help="unused - not implemented",
-    )
-    parser.add_argument(
-        "expression",
-        type=str,
-        nargs="?",
-        help="select which test(s) to run",
-    )
-
-    args = into_pytest_args(parser.parse_args())
-    print(f"$ pytest {' '.join(args)}\n" "---------------------------\n")
-
-    time.sleep(2)  # force the user to stare at the warning message
-
-    sys.exit(pytest.main(args))
-
-
-if __name__ == "__main__":
-    main()
-
-# vim: set filetype=python :
+(cd "$system_test_dir" || exit 2 ; /usr/bin/env python3 -m pytest "$@")

From f3c000525784a0114d593793361a1e45a26acded Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Tue, 9 May 2023 14:20:02 +0200
Subject: [PATCH 35/37] Tear down module logger handler in system tests

The module-level logger has a handler that writes into a temporary
directory. Ensure the logging output is flushed and the handler is
closed before attempting to remove this temporary directory.

(cherry picked from commit 0f8a2b07a4bb235c1dd3aff4ad53ee8a083d67d0)
---
 bin/tests/system/conftest.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 83a0130511..efcc2f51ff 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -449,6 +449,8 @@ else:
                 )
             else:
                 mlogger.debug("deleting temporary directory")
+                handler.flush()
+                handler.close()
                 shutil.rmtree(testdir)
 
     def _run_script(  # pylint: disable=too-many-arguments

From b9b3b53e2c6ee03f6f01cb983ea72169a8c9a6f8 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Thu, 30 Mar 2023 15:32:59 +0200
Subject: [PATCH 36/37] Add CHANGES and release note for [GL #3978]

(cherry picked from commit fd889bf0ad9f863378e52a14891cefcd8bc8e6e8)
---
 CHANGES                     | 3 +++
 doc/notes/notes-current.rst | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/CHANGES b/CHANGES
index 4897fbd84a..d5d3e6a628 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,6 @@
+6176.	[test]		Add support for using pytest & pytest-xdist to
+			execute the system test suite. [GL #3978]
+
 6174.	[bug]		BIND could get stuck on reconfiguration when a
 			'listen' statement for HTTP is removed from the
 			configuration. That has been fixed. [GL #4071]
diff --git a/doc/notes/notes-current.rst b/doc/notes/notes-current.rst
index e57a5a6c8c..f42a9e0019 100644
--- a/doc/notes/notes-current.rst
+++ b/doc/notes/notes-current.rst
@@ -22,6 +22,9 @@ New Features
 
 - None.
 
+- The system test suite can now be executed with pytest (along with
+  pytest-xdist for parallel execution). :gl:`#3978`
+
 Removed Features
 ~~~~~~~~~~~~~~~~
 

From 81b7682e2454f9cb5615003b7f2b1b0293433670 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Mon, 22 May 2023 14:20:29 +0200
Subject: [PATCH 37/37] Reorder dead primary checks in upforwd test

The check which attempts to forward dynamic update to a dead primary may
trigger a timing issue #4080. For some reason, this has manifested under
the pytest runner, while the test still passes with the legacy runner.

Move the dead primary check closer to the end of the test to avoid
hitting this issue before we have a proper fix.

(cherry picked from commit edaa5f5d2a221dade97ef8c7892f0b6b7a50ddec)
---
 bin/tests/system/upforwd/tests.sh | 66 +++++++++++++++----------------
 1 file changed, 33 insertions(+), 33 deletions(-)

diff --git a/bin/tests/system/upforwd/tests.sh b/bin/tests/system/upforwd/tests.sh
index 1d11bdf813..bc95aa42ab 100644
--- a/bin/tests/system/upforwd/tests.sh
+++ b/bin/tests/system/upforwd/tests.sh
@@ -190,39 +190,6 @@ then
 fi
 n=`expr $n + 1`
 
-echo_i "checking update forwarding to dead primary ($n)"
-count=0
-ret=0
-while [ $count -lt 5 -a $ret -eq 0 ]
-do
-(
-$NSUPDATE -- - <<EOF 
-local 10.53.0.1
-server 10.53.0.3 ${PORT}
-zone noprimary
-update add unsigned.noprimary. 600 A 10.10.10.1
-update add unsigned.noprimary. 600 TXT Foo
-send
-EOF
-) > /dev/null 2>&1 &
-	$DIG -p ${PORT} +noadd +notcp +noauth noprimary. @10.53.0.3 soa > dig.out.ns3 || ret=1
-	grep "status: NOERROR" dig.out.ns3 > /dev/null || ret=1
-	count=`expr $count + 1`
-done
-if [ $ret != 0 ] ; then echo_i "failed"; status=`expr $status + $ret`; fi
-n=`expr $n + 1`
-
-if $FEATURETEST --enable-dnstap
-then
-	echo_i "checking DNSTAP logging of UPDATE forwarded update replies ($n)"
-	ret=0
-	capture_dnstap
-	uq_equals_ur && ret=1
-	if [ $ret != 0 ] ; then echo_i "failed"; fi
-	status=`expr $status + $ret`
-	n=`expr $n + 1`
-fi
-
 if test -f keyname
 then
 	echo_i "checking update forwarding to with sig0 ($n)"
@@ -269,6 +236,39 @@ grep REFUSED nsupdate.out.$n > /dev/null || ret=1
 if [ $ret != 0 ] ; then echo_i "failed"; status=`expr $status + $ret`; fi
 n=`expr $n + 1`
 
+echo_i "checking update forwarding to dead primary ($n)"
+count=0
+ret=0
+while [ $count -lt 5 -a $ret -eq 0 ]
+do
+(
+$NSUPDATE -- - <<EOF
+local 10.53.0.1
+server 10.53.0.3 ${PORT}
+zone noprimary
+update add unsigned.noprimary. 600 A 10.10.10.1
+update add unsigned.noprimary. 600 TXT Foo
+send
+EOF
+) > /dev/null 2>&1 &
+	$DIG -p ${PORT} +noadd +notcp +noauth noprimary. @10.53.0.3 soa > dig.out.ns3 || ret=1
+	grep "status: NOERROR" dig.out.ns3 > /dev/null || ret=1
+	count=`expr $count + 1`
+done
+if [ $ret != 0 ] ; then echo_i "failed"; status=`expr $status + $ret`; fi
+n=`expr $n + 1`
+
+if $FEATURETEST --enable-dnstap
+then
+	echo_i "checking DNSTAP logging of UPDATE forwarded update replies ($n)"
+	ret=0
+	capture_dnstap
+	uq_equals_ur && ret=1
+	if [ $ret != 0 ] ; then echo_i "failed"; fi
+	status=`expr $status + $ret`
+	n=`expr $n + 1`
+fi
+
 n=$((n + 1))
 ret=0
 echo_i "attempting updates that should exceed quota ($n)"