[9.20] fix: test: minor fixes for extra_artifacts detection

Some omissions of !9426 discovered during the backports

Backport of MR !9739

Merge branch 'backport-nicki/extra-artifacts-fixups-9.20' into 'bind-9.20'

See merge request isc-projects/bind9!9741

bin/tests/system/conftest.py

@@ -134,16 +134,25 @@ def pytest_collection_modifyitems(items):
 
 class NodeResult:
     def __init__(self, report=None):
-        self.outcome = None
-        self.messages = []
+        self._outcomes = {}
+        self.messages = {}
         if report is not None:
             self.update(report)
 
     def update(self, report):
-        if self.outcome is None or report.outcome != "passed":
-            self.outcome = report.outcome
-        if report.longreprtext:
-            self.messages.append(report.longreprtext)
+        # Allow the same nodeid/when to be overriden. This only happens when
+        # the test is re-run with flaky plugin. In that case, we want the
+        # latest result to override any previous results.
+        key = (report.nodeid, report.when)
+        self._outcomes[key] = report.outcome
+        self.messages[key] = report.longreprtext
+
+    @property
+    def outcome(self):
+        for outcome in self._outcomes.values():
+            if outcome != "passed":
+                return outcome
+        return "passed"
 
 
 @pytest.hookimpl(tryfirst=True, hookwrapper=True)
@@ -347,7 +356,7 @@ def system_test_dir(request, system_test_name, expected_artifacts):
         messages = []
         for node, result in test_results.items():
             isctest.log.debug("%s %s", result.outcome.upper(), node)
-            messages.extend(result.messages)
+            messages.extend(result.messages.values())
         for message in messages:
             isctest.log.debug("\n" + message)
         failed = any(res.outcome == "failed" for res in test_results.values())

bin/tests/system/dialup/clean.sh

@@ -1,17 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-rm -f ns2/example.bk
-rm -f ns3/example.bk
-rm -f */named.memstats */named.run
-rm -f ns*/named.conf

bin/tests/system/enginepkcs11/tests_sh_enginepkcs11.py

@@ -46,6 +46,7 @@ pytestmark = pytest.mark.extra_artifacts(
         "ns1/*.weird.db",
         "ns1/*.weird.db.signed",
         "ns2/keys",
+        "ns2/named.args",
         "ns2/*.view*.db",
         "ns2/*.view*.db.signed",
     ]

bin/tests/system/keyfromlabel/setup.sh

@@ -16,6 +16,7 @@
 
 set -e
 
+OPENSSL_CONF= softhsm2-util --delete-token --token "softhsm2-keyfromlabel" >/dev/null 2>&1 || true
 OPENSSL_CONF= softhsm2-util --init-token --free --pin 1234 --so-pin 1234 --label "softhsm2-keyfromlabel" | awk '/^The token has been initialized and is reassigned to slot/ { print $NF }'
 
 printf '%s' "${HSMPIN:-1234}" >pin