[9.18] chg: pkg: Update requirements for system test suite

Python 3.10 or newer is now required for running the system test suite. The required python packages and their version requirements are now tracked in `bin/tests/system/requirements.txt`.

Support for pytest 9.0.0 has been added its minimum supported version has been raised to 7.0.0. The minimum supported dnspython version has been raised to 2.3.0.

Closes #5690 

Closes #5614

Backport of MR !11415

Merge branch 'backport-nicki/pytest-9-compat-9.18' into 'bind-9.18'

See merge request isc-projects/bind9!11470
This commit is contained in:
Nicki Křížek 2026-01-21 16:39:24 +01:00
commit 37bd997a39
34 changed files with 70 additions and 204 deletions

View file

@ -506,10 +506,6 @@ stages:
- ( if [ "${CI_DISPOSABLE_ENVIRONMENT}" = "true" ]; then sleep 3000; "$PYTHON" "${CI_PROJECT_DIR}/util/get-running-system-tests.py"; fi ) &
- cd bin/tests/system
- RET=0
# With pytest 9.0, there's the following error in pytest_ignore_collect():
# The (path: py.path.local) argument is deprecated, please use (collection_path: pathlib.Path).
# This should be fixed before pytest 9.1, when it becomes ineffective.
- if pytest --version | grep -F "pytest 9.0" >/dev/null; then echo "filterwarnings = ignore::pytest.PytestRemovedIn9Warning" >> pytest.ini; fi
- >
("$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit_pytest.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt) || RET=1
- *git_clone_bind9-qa
@ -981,6 +977,7 @@ cross-version-config-tests:
untracked: true
expire_in: "1 day"
when: always
allow_failure: true # GL!11415
# Jobs for regular GCC builds on Alpine Linux 3.23 (amd64)

View file

@ -81,6 +81,7 @@ path = [
"bin/tests/system/pipelined/inputb",
"bin/tests/system/pipelined/ref",
"bin/tests/system/pipelined/refb",
"bin/tests/system/requirements.txt",
"bin/tests/system/rsabigexponent/ns2/dsset-example.in",
"bin/tests/system/run.gdb",
"bin/tests/system/runtime/ctrl-chars",

View file

@ -17,9 +17,6 @@ import dns.message
import pytest
# isctest.asyncserver requires dnspython >= 2.0.0
pytest.importorskip("dns", minversion="2.0.0")
import isctest
from isctest.instance import NamedInstance

View file

@ -21,7 +21,6 @@ import time
import isctest
import pytest
pytest.importorskip("dns", minversion="2.0.0")
import dns.exception
import dns.message
import dns.name

View file

@ -18,7 +18,8 @@ import shutil
import subprocess
import tempfile
import time
from typing import Any, Dict, List, Optional
from typing import Dict, List, Optional
import sys
import pytest
@ -30,20 +31,8 @@ import isctest
# Silence warnings caused by passing a pytest fixture to another fixture.
# pylint: disable=redefined-outer-name
# ----------------- Older pytest / xdist compatibility -------------------
# As of 2023-01-11, the minimal supported pytest / xdist versions are
# determined by what is available in EL8/EPEL8:
# - pytest 3.4.2
# - pytest-xdist 1.24.1
_pytest_ver = pytest.__version__.split(".")
_pytest_major_ver = int(_pytest_ver[0])
if _pytest_major_ver < 7:
# pytest.Stash/pytest.StashKey mechanism has been added in 7.0.0
# for older versions, use regular dictionary with string keys instead
FIXTURE_OK = "fixture_ok" # type: Any
else:
FIXTURE_OK = pytest.StashKey[bool]() # pylint: disable=no-member
if sys.version_info[1] < 10:
raise RuntimeError("Python 3.10 or newer is required to run system tests.")
# ----------------------- Globals definition -----------------------------
@ -137,7 +126,7 @@ def pytest_configure(config):
config.option.dist = "loadscope"
def pytest_ignore_collect(path):
def pytest_ignore_collect(collection_path):
# System tests are executed in temporary directories inside
# bin/tests/system. These temporary directories contain all files
# needed for the system tests - including tests_*.py files. Make sure to
@ -146,9 +135,9 @@ def pytest_ignore_collect(path):
# convenience symlinks to those test directories. In both of those
# cases, the system test name (directory) contains an underscore, which
# is otherwise and invalid character for a system test name.
match = SYSTEM_TEST_NAME_RE.search(str(path))
match = SYSTEM_TEST_NAME_RE.search(str(collection_path))
if match is None:
isctest.log.warning("unexpected test path: %s (ignored)", path)
isctest.log.warning("unexpected test path: %s (ignored)", collection_path)
return True
system_test_name = match.groups()[0]
return "_" in system_test_name
@ -328,19 +317,10 @@ def system_test_name(request):
return path.parent.name
def _get_marker(node, marker):
try:
# pytest >= 4.x
return node.get_closest_marker(marker)
except AttributeError:
# pytest < 4.x
return node.get_marker(marker)
@pytest.fixture(autouse=True)
def wait_for_zones_loaded(request, servers):
"""Wait for all zones to be loaded by specified named instances."""
instances = _get_marker(request.node, "requires_zones_loaded")
instances = request.node.get_closest_marker("requires_zones_loaded")
if not instances:
return
@ -432,12 +412,6 @@ def system_test_dir(request, env, system_test_name, expected_artifacts):
assert all(res.outcome == "passed" for res in test_results.values())
return "passed"
def unlink(path):
try:
path.unlink() # missing_ok=True isn't available on Python 3.6
except FileNotFoundError:
pass
def check_artifacts(source_dir, run_dir):
def check_artifacts_recursive(dcmp):
def artifact_expected(path, expected):
@ -480,9 +454,9 @@ def system_test_dir(request, env, system_test_name, expected_artifacts):
shutil.copytree(system_test_root / system_test_name, testdir)
# Create a convenience symlink with a stable and predictable name
module_name = SYMLINK_REPLACEMENT_RE.sub(r"\1", str(_get_node_path(request.node)))
module_name = SYMLINK_REPLACEMENT_RE.sub(r"\1", str(request.node.path))
symlink_dst = system_test_root / module_name
unlink(symlink_dst)
symlink_dst.unlink(missing_ok=True)
symlink_dst.symlink_to(os.path.relpath(testdir, start=system_test_root))
isctest.log.init_module_logger(system_test_name, testdir)
@ -514,7 +488,7 @@ def system_test_dir(request, env, system_test_name, expected_artifacts):
"test failure detected, keeping temporary directory %s", testdir
)
keep = True
elif not request.node.stash[FIXTURE_OK]:
elif not request.node.stash["fixture_ok"]:
isctest.log.debug(
"test setup/teardown issue detected, keeping temporary directory %s",
testdir,
@ -531,7 +505,7 @@ def system_test_dir(request, env, system_test_name, expected_artifacts):
isctest.log.deinit_module_logger()
if not keep:
shutil.rmtree(testdir)
unlink(symlink_dst)
symlink_dst.unlink(missing_ok=True)
@pytest.fixture(scope="module")
@ -581,15 +555,6 @@ def _run_script(
isctest.log.debug(" exited with %d", returncode)
def _get_node_path(node) -> Path:
if isinstance(node.parent, pytest.Session):
if _pytest_major_ver >= 8:
return Path()
return Path(node.name)
assert node.parent is not None
return _get_node_path(node.parent) / node.name
@pytest.fixture(scope="module")
def shell(env, system_test_dir):
"""Function to call a shell script with arguments."""
@ -703,13 +668,11 @@ def system_test(
pytest.fail(f"get_core_dumps.sh exited with {exc.returncode}")
os.environ.update(env) # Ensure pytests have the same env vars as shell tests.
isctest.log.info(f"test started: {_get_node_path(request.node)}")
isctest.log.info(f"test started: {request.node.path}")
port = int(env["PORT"])
isctest.log.info("using port range: <%d, %d>", port, port + PORTS_PER_TEST - 1)
if not hasattr(request.node, "stash"): # compatibility with pytest<7.0.0
request.node.stash = {} # use regular dict instead of pytest.Stash
request.node.stash[FIXTURE_OK] = True
request.node.stash["fixture_ok"] = True
# Perform checks which may skip this test.
check_net_interfaces()
@ -718,7 +681,7 @@ def system_test(
# Store the fact that this fixture hasn't successfully finished yet.
# This is checked before temporary directory teardown to decide whether
# it's okay to remove the directory.
request.node.stash[FIXTURE_OK] = False
request.node.stash["fixture_ok"] = False
setup_test()
try:
@ -729,7 +692,7 @@ def system_test(
isctest.log.debug("test(s) finished")
stop_servers()
get_core_dumps()
request.node.stash[FIXTURE_OK] = True
request.node.stash["fixture_ok"] = True
@pytest.fixture(scope="module")

View file

@ -14,7 +14,6 @@
import pytest
import isctest
pytest.importorskip("dns")
import dns.message
pytestmark = pytest.mark.extra_artifacts(

View file

@ -16,9 +16,6 @@ import os
import pytest
pytest.importorskip("cryptography")
pytest.importorskip(
"dns", minversion="2.7.0"
) # dns.dnssec.sign_zone(deterministic=...) needed
from cryptography.hazmat.primitives.asymmetric import ec

View file

@ -17,7 +17,6 @@ import re
import isctest
import pytest
pytest.importorskip("dns", minversion="2.0.0")
import dns.rrset
pytestmark = pytest.mark.extra_artifacts(

View file

@ -18,7 +18,7 @@ import time
import pytest
pytest.importorskip("dns")
import dns
import dns.exception
import dns.name
import dns.rdataclass

View file

@ -12,12 +12,9 @@
import dns.flags
import dns.message
import pytest
import isctest
pytest.importorskip("dns", minversion="2.0.0")
def test_glue_full_glue_set():
"""test that a ccTLD referral gets a full glue set from the root zone"""

View file

@ -11,15 +11,12 @@
from . import check
from . import instance
from . import hypothesis
from . import query
from . import run
from . import template
from . import log
# isctest.hypothesis is intentionally NOT imported, because it detects proper
# hypothesis support and instructs pytest to skip the tests otherwise. It
# should be manually imported only in the modules that require hypothesis.
# isctest.mark module is intentionally NOT imported, because it relies on
# environment variables which might not be set at the time of import of the
# `isctest` package. To use the marks, manual `import isctest.mark` is needed

View file

@ -113,7 +113,6 @@ class AsyncServer:
tcp_handler: Optional[_TcpHandler],
pidfile: Optional[str] = None,
) -> None:
self._abort_if_on_dnspython_version_less_than_2_0_0()
logging.basicConfig(
format="%(asctime)s %(levelname)8s %(message)s",
level=os.environ.get("ANS_LOG_LEVEL", "INFO").upper(),
@ -141,14 +140,6 @@ class AsyncServer:
self._pidfile: Optional[str] = pidfile
self._work_done: Optional[asyncio.Future] = None
@classmethod
def _abort_if_on_dnspython_version_less_than_2_0_0(cls) -> None:
if dns.version.MAJOR < 2:
error = f"Using {cls.__name__} requires dnspython >= 2.0.0; "
error += 'add `pytest.importorskip("dns", minversion="2.0.0")` '
error += "to the test module to skip this test."
raise RuntimeError(error)
def _get_ipv4_address_from_directory_name(self) -> str:
containing_directory = pathlib.Path().absolute().stem
match_result = re.match(r"ans(?P<index>\d+)", containing_directory)

View file

@ -13,13 +13,13 @@ import shutil
from typing import cast, List, Optional
import dns.edns
from dns.edns import EDECode, EDEOption
import dns.flags
import dns.message
import dns.rcode
import dns.zone
import isctest.log
from isctest.compat import dns_rcode, EDECode, EDEOption
def rcode(message: dns.message.Message, expected_rcode) -> None:
@ -27,19 +27,19 @@ def rcode(message: dns.message.Message, expected_rcode) -> None:
def noerror(message: dns.message.Message) -> None:
rcode(message, dns_rcode.NOERROR)
rcode(message, dns.rcode.NOERROR)
def notimp(message: dns.message.Message) -> None:
rcode(message, dns_rcode.NOTIMP)
rcode(message, dns.rcode.NOTIMP)
def refused(message: dns.message.Message) -> None:
rcode(message, dns_rcode.REFUSED)
rcode(message, dns.rcode.REFUSED)
def servfail(message: dns.message.Message) -> None:
rcode(message, dns_rcode.SERVFAIL)
rcode(message, dns.rcode.SERVFAIL)
def adflag(message: dns.message.Message) -> None:
@ -82,10 +82,6 @@ def _extract_ede_options(
def noede(message: dns.message.Message) -> None:
"""Check that message contains no EDE option."""
if not hasattr(dns.edns, "EDECode"):
# dnspython<2.2.0 doesn't support EDE, skip check
return
ede_options = _extract_ede_options(message)
assert not ede_options, f"unexpected EDE options {ede_options} in {message}"
@ -94,10 +90,6 @@ def ede(
message: dns.message.Message, code: EDECode, text: Optional[str] = None
) -> None:
"""Check if message contains expected EDE code (and its text)."""
if not hasattr(dns.edns, "EDECode"):
# dnspython<2.2.0 doesn't support EDE, skip check
return
msg_opts = _extract_ede_options(message)
matching_opts = [opt for opt in msg_opts if opt.code == code]
@ -204,7 +196,7 @@ def is_executable(cmd: str, errmsg: str) -> None:
def named_alive(named_proc, resolver_ip):
assert named_proc.poll() is None, "named isn't running"
msg = isctest.query.create("version.bind", "TXT", "CH")
isctest.query.tcp(msg, resolver_ip, expected_rcode=dns_rcode.NOERROR)
isctest.query.tcp(msg, resolver_ip, expected_rcode=dns.rcode.NOERROR)
def notauth(message: dns.message.Message) -> None:

View file

@ -1,56 +0,0 @@
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# SPDX-License-Identifier: MPL-2.0
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at https://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
from typing import Any, TYPE_CHECKING
import dns.edns
import dns.rcode
# compatiblity with dnspython<2.0.0
try:
# In dnspython>=2.0.0, dns.rcode.Rcode class is available
# pylint: disable=invalid-name
dns_rcode = dns.rcode.Rcode # type: Any
except AttributeError:
# In dnspython<2.0.0, selected rcodes are available as integers directly
# from dns.rcode
dns_rcode = dns.rcode
if TYPE_CHECKING:
EDECode = dns.edns.EDECode
EDEOption = dns.edns.EDEOption
else:
try: # compatiblity with dnspython<2.2.0
EDECode = dns.edns.EDECode
except AttributeError:
# In dnspython<2.2.0, the dns.edns.EDECode doesn't exist.
#
# The primary use-case is for us to use existing EDECode objects from the
# class, e.g. EDECode.FILTERED. To mimick this behavior, use a string
# factory that just turns the attribute name into a string.
#
# The used compatibility hack doesn't really matter (as long as EDECode.xxx
# doesn't raise exception), as with dnspython versions prior to 2.2.0, any
# EDE checking will be skipped anyway.
class _CompatEDECode:
def __getattr__(self, name: str) -> str:
return name
EDECode = _CompatEDECode()
try:
EDEOption = dns.edns.EDEOption
except AttributeError:
# In dnspython<2.2.0, the dns.edns.EDEOption doesn't exist, so we stub it to be
# able to use it in type annotations.
class EDEOption:
def __new__(cls, *args, **kwargs):
raise RuntimeError("Using EDEOption requires dnspython>=2.2.0")

View file

@ -12,17 +12,5 @@
# This ensures we're using a suitable hypothesis version. A newer version is
# required for FIPS-enabled platforms.
import hashlib
import pytest
MIN_HYPOTHESIS_VERSION = None
if "md5" not in hashlib.algorithms_available:
# FIPS mode is enabled, use hypothesis 4.41.2 which doesn't use md5
MIN_HYPOTHESIS_VERSION = "4.41.2"
pytest.importorskip("hypothesis", minversion=MIN_HYPOTHESIS_VERSION)
from . import settings
from . import strategies

View file

@ -143,13 +143,8 @@ def dns_names(
RDATACLASS_MAX = RDATATYPE_MAX = 65535
try:
dns_rdataclasses = builds(dns.rdataclass.RdataClass, integers(0, RDATACLASS_MAX))
dns_rdatatypes = builds(dns.rdatatype.RdataType, integers(0, RDATATYPE_MAX))
except AttributeError:
# In old dnspython versions, RDataTypes and RDataClasses are int and not enums.
dns_rdataclasses = integers(0, RDATACLASS_MAX) # type: ignore
dns_rdatatypes = integers(0, RDATATYPE_MAX) # type: ignore
dns_rdataclasses = builds(dns.rdataclass.RdataClass, integers(0, RDATACLASS_MAX))
dns_rdatatypes = builds(dns.rdatatype.RdataType, integers(0, RDATATYPE_MAX))
dns_rdataclasses_without_meta = dns_rdataclasses.filter(dns.rdataclass.is_metaclass)
# NOTE: This should really be `dns_rdatatypes_without_meta = dns_rdatatypes_without_meta.filter(dns.rdatatype.is_metatype()`,

View file

@ -11,9 +11,6 @@
from typing import Container, Iterable, FrozenSet
import pytest
pytest.importorskip("dns", minversion="2.3.0") # NameRelation
from dns.name import Name, NameRelation
import dns.zone
import dns.rdatatype

View file

@ -17,7 +17,6 @@ import dns.query
import dns.message
import isctest.log
from isctest.compat import dns_rcode
QUERY_TIMEOUT = 10
@ -30,7 +29,7 @@ def generic_query(
source: Optional[str] = None,
timeout: int = QUERY_TIMEOUT,
attempts: int = 10,
expected_rcode: dns_rcode = None,
expected_rcode: Optional[dns.rcode.Rcode] = None,
log_query: bool = True,
log_response: bool = True,
) -> Any:
@ -61,9 +60,9 @@ def generic_query(
return res
time.sleep(1)
if expected_rcode is not None:
last_rcode = dns_rcode.to_text(res.rcode()) if res else None
last_rcode = dns.rcode.to_text(res.rcode()) if res else None
isctest.log.debug(
f"isc.query.{query_func.__name__}(): expected rcode={dns_rcode.to_text(expected_rcode)}, last rcode={last_rcode}"
f"isc.query.{query_func.__name__}(): expected rcode={dns.rcode.to_text(expected_rcode)}, last rcode={last_rcode}"
)
raise dns.exception.Timeout

View file

@ -16,9 +16,9 @@ from typing import Optional
import isctest.log
import isctest.text
from isctest.compat import dns_rcode
import dns.message
import dns.rcode
class CmdResult:
@ -149,4 +149,4 @@ def get_custom_named_instance(assumed_ns, ports):
def assert_custom_named_is_alive(named_proc, resolver_ip):
assert named_proc.poll() is None, "named isn't running"
msg = dns.message.make_query("version.bind", "TXT", "CH")
isctest.query.tcp(msg, resolver_ip, expected_rcode=dns_rcode.NOERROR)
isctest.query.tcp(msg, resolver_ip, expected_rcode=dns.rcode.NOERROR)

View file

@ -14,9 +14,6 @@ import itertools
import isctest
import pytest
# Everything from getting a big answer to creating an RR set with thousands
# of records takes minutes of CPU and real time with dnspython < 2.0.0.
pytest.importorskip("dns", minversion="2.0.0")
import dns.rrset

View file

@ -9,10 +9,6 @@
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
import pytest
pytest.importorskip("dns", minversion="2.7.0")
import isctest

View file

@ -11,6 +11,10 @@
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
# Silence incorrect warnings cause by hypothesis.assume()
# https://github.com/pylint-dev/pylint/issues/10785#issuecomment-3677224217
# pylint: disable=unreachable
from dataclasses import dataclass
import os
from pathlib import Path
@ -18,7 +22,6 @@ from typing import Optional, Set, Tuple
import pytest
pytest.importorskip("dns", minversion="2.5.0")
import dns.dnssec
import dns.message
import dns.name

View file

@ -19,7 +19,7 @@ import sys
import isctest
import pytest
pytest.importorskip("dns", minversion="2.0.0")
import dns
import dns.exception
import dns.message
import dns.name

View file

@ -0,0 +1,13 @@
### Test requirements
dnspython>=2.7.0
cryptography
hypothesis>=4.41.2
jinja2
pytest>=7.0.0
requests
### Utility packages for executing the tests
flaky
pytest-xdist

View file

@ -15,12 +15,11 @@ import os
import pytest
pytest.importorskip("dns", minversion="2.0.0")
import dns
import dns.rcode
import dns.rrset
import isctest
from isctest.compat import dns_rcode
pytestmark = pytest.mark.extra_artifacts(
@ -78,13 +77,13 @@ def test_rpz_multiple_views(qname, source, rcode):
msg,
ip="10.53.0.3",
source="10.53.0.2",
expected_rcode=dns_rcode.NOERROR,
expected_rcode=dns.rcode.NOERROR,
)
isctest.query.tcp(
msg,
ip="10.53.0.3",
source="10.53.0.5",
expected_rcode=dns_rcode.NOERROR,
expected_rcode=dns.rcode.NOERROR,
)
msg = isctest.query.create(qname, "A")

View file

@ -21,7 +21,7 @@ import time
import pytest
pytest.importorskip("dns", minversion="2.0.0")
import dns
import dns.exception
import isctest

View file

@ -14,13 +14,13 @@
from datetime import datetime
import pytest
import requests
import isctest.mark
pytest.register_assert_rewrite("generic")
import generic
requests = pytest.importorskip("requests")
pytestmark = [
isctest.mark.have_json_c,

View file

@ -15,13 +15,13 @@ from datetime import datetime
import xml.etree.ElementTree as ET
import pytest
import requests
import isctest.mark
pytest.register_assert_rewrite("generic")
import generic
requests = pytest.importorskip("requests")
pytestmark = [
isctest.mark.have_libxml2,

View file

@ -19,7 +19,7 @@ import time
import pytest
pytest.importorskip("dns", minversion="2.0.0")
import dns
import dns.message
import dns.query

View file

@ -18,7 +18,7 @@ import time
import pytest
pytest.importorskip("dns", minversion="2.0.0")
import dns
import dns.edns
import dns.message
import dns.name

View file

@ -11,12 +11,14 @@
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
# Silence incorrect warnings cause by hypothesis.assume()
# https://github.com/pylint-dev/pylint/issues/10785#issuecomment-3677224217
# pylint: disable=unreachable
import time
import pytest
pytest.importorskip("dns", minversion="2.7.0") # TSIG parsing without validation
import dns.exception
import dns.message
import dns.name

View file

@ -24,7 +24,7 @@ import pytest
import isctest
pytest.importorskip("dns")
import dns
import dns.message
import dns.name
import dns.rdata

View file

@ -27,9 +27,13 @@ Limitations - untested properties:
- special behavior of rdtypes like CNAME
"""
# Silence incorrect warnings cause by hypothesis.assume()
# https://github.com/pylint-dev/pylint/issues/10785#issuecomment-3677224217
# pylint: disable=unreachable
import pytest
pytest.importorskip("dns", minversion="2.0.0")
import dns
import dns.message
import dns.name
import dns.query

View file

@ -240,7 +240,7 @@ AM_CONDITIONAL([HAVE_PERL], [test -n "$PERL"])
#
# Python is optional, it is used only by some of the system test scripts.
#
AM_PATH_PYTHON([3.6], [], [:])
AM_PATH_PYTHON([3.10], [], [:])
AM_CONDITIONAL([HAVE_PYTHON], [test "$PYTHON" != ":"])
AC_PATH_PROGS([PYTEST], [pytest-3 py.test-3 pytest py.test pytest-pypy], [])