From aa199e2726c8baf4cbfbe891e0c3182e2d47bdb2 Mon Sep 17 00:00:00 2001
From: Tom Krizek <tkrizek@isc.org>
Date: Thu, 12 Jan 2023 16:52:49 +0100
Subject: [PATCH] Assign unique ports to pytest modules

This is basically a pytest re-implementation of the get_ports.sh script.
The main difference is that ports are assigned on a module basis, rather
than a directory basis. Module is the new atomic unit for parallel
execution, therefore it needs to have unique ports to avoid collisions.

Each module gets its ports through the env fixture which is updated with
ports and other module-specific variables.

(cherry picked from commit 006175815640efafa1add511242b90af8c42547d)
---
 bin/tests/system/conftest.py | 84 ++++++++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 69d40837fd..4d603c0cfe 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -16,6 +16,8 @@ import pytest
 # ======================= LEGACY=COMPATIBLE FIXTURES =========================
 # The following fixtures are designed to work with both pytest system test
 # runner and the legacy system test framework.
+#
+# FUTURE: Rewrite the individual port fixtures to re-use the `ports` fixture.
 
 
 @pytest.fixture(scope="module")
@@ -53,12 +55,19 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
     from pathlib import Path
     import re
     import subprocess
+    import time
+
+    # Silence warnings caused by passing a pytest fixture to another fixture.
+    # pylint: disable=redefined-outer-name
 
     # ----------------------- Globals definition -----------------------------
 
     XDIST_WORKER = os.environ.get("PYTEST_XDIST_WORKER", "")
     FILE_DIR = os.path.abspath(Path(__file__).parent)
     ENV_RE = re.compile("([^=]+)=(.*)")
+    PORT_MIN = 5001
+    PORT_MAX = 32767
+    PORTS_PER_TEST = 20
 
     # ---------------------- Module initialization ---------------------------
 
@@ -119,3 +128,78 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
                 logging.error("failed to compile test files: %s", exc)
                 raise exc
             logging.debug(proc.stdout)
+
+    # --------------------------- Fixtures -----------------------------------
+
+    @pytest.fixture(scope="session")
+    def modules():
+        """Sorted list of all modules. Used to determine port distribution."""
+        mods = []
+        for dirpath, _dirs, files in os.walk(os.getcwd()):
+            for file in files:
+                if file.startswith("tests_") and file.endswith(".py"):
+                    mod = f"{dirpath}/{file}"
+                    mods.append(mod)
+        return sorted(mods)
+
+    @pytest.fixture(scope="session")
+    def module_base_ports(modules):
+        """
+        Dictionary containing assigned base port for every module.
+
+        Note that this is a session-wide fixture. The port numbers are
+        deterministically assigned before any testing starts. This fixture MUST
+        return the same value when called again during the same test session.
+        When running tests in parallel, this is exactly what happens - every
+        worker thread will call this fixture to determine test ports.
+        """
+        port_min = PORT_MIN
+        port_max = PORT_MAX - len(modules) * PORTS_PER_TEST
+        if port_max < port_min:
+            raise RuntimeError(
+                "not enough ports to assign unique port set to each module"
+            )
+
+        # Rotate the base port value over time to detect possible test issues
+        # with using random ports. This introduces a very slight race condition
+        # risk. If this value changes between pytest invocation and spawning
+        # worker threads, multiple tests may have same port values assigned. If
+        # these tests are then executed simultaneously, the test results will
+        # be misleading.
+        base_port = int(time.time() // 3600) % (port_max - port_min)
+
+        return {mod: base_port + i * PORTS_PER_TEST for i, mod in enumerate(modules)}
+
+    @pytest.fixture(scope="module")
+    def base_port(request, module_base_ports):
+        """Start of the port range assigned to a particular test module."""
+        port = module_base_ports[request.fspath]
+        return port
+
+    @pytest.fixture(scope="module")
+    def ports(base_port):
+        """Dictionary containing port names and their assigned values."""
+        return {
+            "PORT": str(base_port),
+            "TLSPORT": str(base_port + 1),
+            "HTTPPORT": str(base_port + 2),
+            "HTTPSPORT": str(base_port + 3),
+            "EXTRAPORT1": str(base_port + 4),
+            "EXTRAPORT2": str(base_port + 5),
+            "EXTRAPORT3": str(base_port + 6),
+            "EXTRAPORT4": str(base_port + 7),
+            "EXTRAPORT5": str(base_port + 8),
+            "EXTRAPORT6": str(base_port + 9),
+            "EXTRAPORT7": str(base_port + 10),
+            "EXTRAPORT8": str(base_port + 11),
+            "CONTROLPORT": str(base_port + 12),
+        }
+
+    @pytest.fixture(scope="module")
+    def env(ports):
+        """Dictionary containing environment variables for the test."""
+        env = CONF_ENV.copy()
+        env.update(ports)
+        env["builddir"] = f"{env['TOP_BUILDDIR']}/bin/tests/system"
+        env["srcdir"] = f"{env['TOP_SRCDIR']}/bin/tests/system"
+        return env