From 3a9f4edddcfba0d4d303f0ae6873a9f0c07966d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= <michal@isc.org>
Date: Mon, 19 Aug 2024 18:49:08 +0200
Subject: [PATCH] Add pytest fixture for checking test artifacts

Prior to introducing the pytest runner, clean.sh files were used as a
list of files that the test is expected to leave around as artifacts and
check that no extra files were created.

With the pytest runner, those scripts are no longer used, but the
ability to detect extraneous files is still useful. Add a new
"extra_artifacts" mark which can be used for the same purpose.
---
 bin/tests/system/conftest.py | 59 +++++++++++++++++++++++++++++++++++-
 bin/tests/system/pytest.ini  |  1 +
 2 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 7ffb2319b8..e4fd7c7625 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -10,6 +10,7 @@
 # information regarding copyright ownership.
 
 from functools import partial
+import filecmp
 import os
 from pathlib import Path
 import re
@@ -297,7 +298,28 @@ def logger(request, system_test_name):
 
 
 @pytest.fixture(scope="module")
-def system_test_dir(request, system_test_name):
+def expected_artifacts(request):
+    common_artifacts = [
+        "ns*/named.run",
+        "ns*/named.run.prev",
+        "ns*/named.conf",
+        "ns*/named.memstats",
+        "pytest.log.txt",
+    ]
+
+    try:
+        test_specific_artifacts = request.node.get_closest_marker("extra_artifacts")
+    except AttributeError:
+        return None
+
+    if test_specific_artifacts:
+        return common_artifacts + test_specific_artifacts.args[0]
+
+    return common_artifacts
+
+
+@pytest.fixture(scope="module")
+def system_test_dir(request, system_test_name, expected_artifacts):
     """
     Temporary directory for executing the test.
 
@@ -345,6 +367,38 @@ def system_test_dir(request, system_test_name):
         except FileNotFoundError:
             pass
 
+    def check_artifacts(source_dir, run_dir):
+        def check_artifacts_recursive(dcmp):
+            def artifact_expected(path, expected):
+                for glob in expected:
+                    if path.match(glob):
+                        return True
+                return False
+
+            # test must not remove any Git-tracked file, ignore libtool and gcov artifacts
+            for name in dcmp.left_only:
+                assert name.startswith("lt-") or name.endswith(".gcda")
+            assert not dcmp.diff_files, "test must not modify any Git-tracked file"
+
+            dir_path = Path(dcmp.left).relative_to(source_dir)
+            for name in dcmp.right_only:
+                file = dir_path / Path(name)
+                if not artifact_expected(file, expected_artifacts):
+                    unexpected_files.append(str(file))
+            for subdir in dcmp.subdirs.values():
+                check_artifacts_recursive(subdir)
+
+        if expected_artifacts is None:  # skip the check if artifact list is unavailable
+            return
+
+        unexpected_files = []
+        dcmp = filecmp.dircmp(source_dir, run_dir)
+        check_artifacts_recursive(dcmp)
+
+        assert (
+            not unexpected_files
+        ), f"Unexpected files found in test directory: {unexpected_files}"
+
     # Create a temporary directory with a copy of the original system test dir contents
     system_test_root = Path(os.environ["builddir"])
     testdir = Path(
@@ -374,6 +428,9 @@ def system_test_dir(request, system_test_name):
 
         result = get_test_result()
 
+        if result == "passed":
+            check_artifacts(system_test_root / system_test_name, testdir)
+
         # Clean temporary dir unless it should be kept
         keep = False
         if request.config.getoption("--noclean"):
diff --git a/bin/tests/system/pytest.ini b/bin/tests/system/pytest.ini
index 766b3dad2e..5574b3e611 100644
--- a/bin/tests/system/pytest.ini
+++ b/bin/tests/system/pytest.ini
@@ -21,3 +21,4 @@ junit_log_passing_tests = 0
 markers =
     requires_zones_loaded: ensures the test does not start until the specified named instances load all configured zones
     algorithm_set: use to select desired algorithms from isctest/vars/algorithms.py
+    extra_artifacts: list of files (globs) that are expected to appear in the test directory after the test is run