From ddb75fc4d8850eeb964d28f3da37c515e81af285 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= <michal@isc.org>
Date: Mon, 19 Aug 2024 18:49:08 +0200
Subject: [PATCH] Add pytest fixture for checking test artifacts

Prior to introducing the pytest runner, clean.sh files were used as a
list of files that the test is expected to leave around as artifacts and
check that no extra files were created.

With the pytest runner, those scripts are no longer used, but the
ability to detect extraneous files is still useful. Add a new
"extra_artifacts" mark which can be used for the same purpose.

(cherry picked from commit 3a9f4edddcfba0d4d303f0ae6873a9f0c07966d7)
---
 bin/tests/system/conftest.py | 60 +++++++++++++++++++++++++++++++++++-
 bin/tests/system/pytest.ini  |  1 +
 2 files changed, 60 insertions(+), 1 deletion(-)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 42b1a11f33..9a134250e0 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -10,6 +10,7 @@
 # information regarding copyright ownership.
 
 from functools import partial
+import filecmp
 import os
 from pathlib import Path
 import re
@@ -345,7 +346,28 @@ def logger(request, system_test_name):
 
 
 @pytest.fixture(scope="module")
-def system_test_dir(request, env, system_test_name):
+def expected_artifacts(request):
+    common_artifacts = [
+        "ns*/named.run",
+        "ns*/named.run.prev",
+        "ns*/named.conf",
+        "ns*/named.memstats",
+        "pytest.log.txt",
+    ]
+
+    try:
+        test_specific_artifacts = request.node.get_closest_marker("extra_artifacts")
+    except AttributeError:
+        return None
+
+    if test_specific_artifacts:
+        return common_artifacts + test_specific_artifacts.args[0]
+
+    return common_artifacts
+
+
+@pytest.fixture(scope="module")
+def system_test_dir(request, env, system_test_name, expected_artifacts):
     """
     Temporary directory for executing the test.
 
@@ -393,6 +415,39 @@ def system_test_dir(request, env, system_test_name):
         except FileNotFoundError:
             pass
 
+    def check_artifacts(source_dir, run_dir):
+        def check_artifacts_recursive(dcmp):
+            def artifact_expected(path, expected):
+                for glob in expected:
+                    if path.match(glob):
+                        return True
+                return False
+
+            # test must not remove any Git-tracked file, ignore libtool and gcov artifacts
+            for name in dcmp.left_only:
+                path = Path(name)
+                assert path.name.startswith("lt-") or path.suffix == ".gcda"
+            assert not dcmp.diff_files, "test must not modify any Git-tracked file"
+
+            dir_path = Path(dcmp.left).relative_to(source_dir)
+            for name in dcmp.right_only:
+                file = dir_path / Path(name)
+                if not artifact_expected(file, expected_artifacts):
+                    unexpected_files.append(str(file))
+            for subdir in dcmp.subdirs.values():
+                check_artifacts_recursive(subdir)
+
+        if expected_artifacts is None:  # skip the check if artifact list is unavailable
+            return
+
+        unexpected_files = []
+        dcmp = filecmp.dircmp(source_dir, run_dir)
+        check_artifacts_recursive(dcmp)
+
+        assert (
+            not unexpected_files
+        ), f"Unexpected files found in test directory: {unexpected_files}"
+
     # Create a temporary directory with a copy of the original system test dir contents
     system_test_root = Path(f"{env['TOP_BUILDDIR']}/{SYSTEM_TEST_DIR_GIT_PATH}")
     testdir = Path(
@@ -421,6 +476,9 @@ def system_test_dir(request, env, system_test_name):
 
         result = get_test_result()
 
+        if result == "passed":
+            check_artifacts(system_test_root / system_test_name, testdir)
+
         # Clean temporary dir unless it should be kept
         keep = False
         if request.config.getoption("--noclean"):
diff --git a/bin/tests/system/pytest.ini b/bin/tests/system/pytest.ini
index ff17998e35..db85d96b00 100644
--- a/bin/tests/system/pytest.ini
+++ b/bin/tests/system/pytest.ini
@@ -20,3 +20,4 @@ junit_logging = log
 junit_log_passing_tests = 0
 markers =
     requires_zones_loaded: ensures the test does not start until the specified named instances load all configured zones
+    extra_artifacts: list of files (globs) that are expected to appear in the test directory after the test is run