diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py index 7ffb2319b8..e4fd7c7625 100644 --- a/bin/tests/system/conftest.py +++ b/bin/tests/system/conftest.py @@ -10,6 +10,7 @@ # information regarding copyright ownership. from functools import partial +import filecmp import os from pathlib import Path import re @@ -297,7 +298,28 @@ def logger(request, system_test_name): @pytest.fixture(scope="module") -def system_test_dir(request, system_test_name): +def expected_artifacts(request): + common_artifacts = [ + "ns*/named.run", + "ns*/named.run.prev", + "ns*/named.conf", + "ns*/named.memstats", + "pytest.log.txt", + ] + + try: + test_specific_artifacts = request.node.get_closest_marker("extra_artifacts") + except AttributeError: + return None + + if test_specific_artifacts: + return common_artifacts + test_specific_artifacts.args[0] + + return common_artifacts + + +@pytest.fixture(scope="module") +def system_test_dir(request, system_test_name, expected_artifacts): """ Temporary directory for executing the test. @@ -345,6 +367,38 @@ def system_test_dir(request, system_test_name): except FileNotFoundError: pass + def check_artifacts(source_dir, run_dir): + def check_artifacts_recursive(dcmp): + def artifact_expected(path, expected): + for glob in expected: + if path.match(glob): + return True + return False + + # test must not remove any Git-tracked file, ignore libtool and gcov artifacts + for name in dcmp.left_only: + assert name.startswith("lt-") or name.endswith(".gcda") + assert not dcmp.diff_files, "test must not modify any Git-tracked file" + + dir_path = Path(dcmp.left).relative_to(source_dir) + for name in dcmp.right_only: + file = dir_path / Path(name) + if not artifact_expected(file, expected_artifacts): + unexpected_files.append(str(file)) + for subdir in dcmp.subdirs.values(): + check_artifacts_recursive(subdir) + + if expected_artifacts is None: # skip the check if artifact list is unavailable + return + + unexpected_files = [] + dcmp = filecmp.dircmp(source_dir, run_dir) + check_artifacts_recursive(dcmp) + + assert ( + not unexpected_files + ), f"Unexpected files found in test directory: {unexpected_files}" + # Create a temporary directory with a copy of the original system test dir contents system_test_root = Path(os.environ["builddir"]) testdir = Path( @@ -374,6 +428,9 @@ def system_test_dir(request, system_test_name): result = get_test_result() + if result == "passed": + check_artifacts(system_test_root / system_test_name, testdir) + # Clean temporary dir unless it should be kept keep = False if request.config.getoption("--noclean"): diff --git a/bin/tests/system/pytest.ini b/bin/tests/system/pytest.ini index 766b3dad2e..5574b3e611 100644 --- a/bin/tests/system/pytest.ini +++ b/bin/tests/system/pytest.ini @@ -21,3 +21,4 @@ junit_log_passing_tests = 0 markers = requires_zones_loaded: ensures the test does not start until the specified named instances load all configured zones algorithm_set: use to select desired algorithms from isctest/vars/algorithms.py + extra_artifacts: list of files (globs) that are expected to appear in the test directory after the test is run