From 0b9d3fbfea216ad78fd2fe45cb876732943fdfae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicki=20K=C5=99=C3=AD=C5=BEek?= <nicki@isc.org>
Date: Tue, 12 Nov 2024 10:07:02 +0100
Subject: [PATCH 1/2] Ensure pytest runner get proper outcome from flaky reruns

When a test is re-run by the flaky plugin, the TestReport outcomes
collected in the pytest_runtest_makereport() hook should be overriden.
Each of the setup/call/teardown phases is reported again and since we
care about the overall outcome, their respective results should be
overriden so that only the outcome from the final test (re)run gets
reported.

Prior to this change, it lead to a situation where an extra_artifact
generated during the test might be ignored. This was caused because the
check was skipped, since the test was incorrectly considered as "failed"
in the case where the test would fail on the first run, but pass on a
subsequent flaky rerun.

(cherry picked from commit b66fb31dcbaeec4e0d4486a468666b40a25cf5e7)
---
 bin/tests/system/conftest.py | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index cba3a90729..0a8e62a113 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -134,16 +134,25 @@ def pytest_collection_modifyitems(items):
 
 class NodeResult:
     def __init__(self, report=None):
-        self.outcome = None
-        self.messages = []
+        self._outcomes = {}
+        self.messages = {}
         if report is not None:
             self.update(report)
 
     def update(self, report):
-        if self.outcome is None or report.outcome != "passed":
-            self.outcome = report.outcome
-        if report.longreprtext:
-            self.messages.append(report.longreprtext)
+        # Allow the same nodeid/when to be overriden. This only happens when
+        # the test is re-run with flaky plugin. In that case, we want the
+        # latest result to override any previous results.
+        key = (report.nodeid, report.when)
+        self._outcomes[key] = report.outcome
+        self.messages[key] = report.longreprtext
+
+    @property
+    def outcome(self):
+        for outcome in self._outcomes.values():
+            if outcome != "passed":
+                return outcome
+        return "passed"
 
 
 @pytest.hookimpl(tryfirst=True, hookwrapper=True)
@@ -347,7 +356,7 @@ def system_test_dir(request, system_test_name, expected_artifacts):
         messages = []
         for node, result in test_results.items():
             isctest.log.debug("%s %s", result.outcome.upper(), node)
-            messages.extend(result.messages)
+            messages.extend(result.messages.values())
         for message in messages:
             isctest.log.debug("\n" + message)
         failed = any(res.outcome == "failed" for res in test_results.values())

From 9ba60be87045c6be5934279931f1ed76549569eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicki=20K=C5=99=C3=AD=C5=BEek?= <nicki@isc.org>
Date: Tue, 12 Nov 2024 09:25:34 +0100
Subject: [PATCH 2/2] Fix clean.sh removal omissions

- Ensure keyfromlabel token is cleaned up
- Remove forgotten clean.sh file
- Add missing enginepkcs11 test artifact

(cherry picked from commit 7dde34afacf414fbef37e0198ee0bff5452c656e)
---
 bin/tests/system/dialup/clean.sh                | 17 -----------------
 .../enginepkcs11/tests_sh_enginepkcs11.py       |  1 +
 bin/tests/system/keyfromlabel/setup.sh          |  1 +
 3 files changed, 2 insertions(+), 17 deletions(-)
 delete mode 100644 bin/tests/system/dialup/clean.sh

diff --git a/bin/tests/system/dialup/clean.sh b/bin/tests/system/dialup/clean.sh
deleted file mode 100644
index 9ee72bc536..0000000000
--- a/bin/tests/system/dialup/clean.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-rm -f ns2/example.bk
-rm -f ns3/example.bk
-rm -f */named.memstats */named.run
-rm -f ns*/named.conf
diff --git a/bin/tests/system/enginepkcs11/tests_sh_enginepkcs11.py b/bin/tests/system/enginepkcs11/tests_sh_enginepkcs11.py
index 66f2fd9bf8..5a5fef62d5 100644
--- a/bin/tests/system/enginepkcs11/tests_sh_enginepkcs11.py
+++ b/bin/tests/system/enginepkcs11/tests_sh_enginepkcs11.py
@@ -46,6 +46,7 @@ pytestmark = pytest.mark.extra_artifacts(
         "ns1/*.weird.db",
         "ns1/*.weird.db.signed",
         "ns2/keys",
+        "ns2/named.args",
         "ns2/*.view*.db",
         "ns2/*.view*.db.signed",
     ]
diff --git a/bin/tests/system/keyfromlabel/setup.sh b/bin/tests/system/keyfromlabel/setup.sh
index 0d02b6b411..e302c60c68 100644
--- a/bin/tests/system/keyfromlabel/setup.sh
+++ b/bin/tests/system/keyfromlabel/setup.sh
@@ -16,6 +16,7 @@
 
 set -e
 
+OPENSSL_CONF= softhsm2-util --delete-token --token "softhsm2-keyfromlabel" >/dev/null 2>&1 || true
 OPENSSL_CONF= softhsm2-util --init-token --free --pin 1234 --so-pin 1234 --label "softhsm2-keyfromlabel" | awk '/^The token has been initialized and is reassigned to slot/ { print $NF }'
 
 printf '%s' "${HSMPIN:-1234}" >pin