diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index cba3a90729..0a8e62a113 100644
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -134,16 +134,25 @@ def pytest_collection_modifyitems(items):
 
 class NodeResult:
     def __init__(self, report=None):
-        self.outcome = None
-        self.messages = []
+        self._outcomes = {}
+        self.messages = {}
         if report is not None:
             self.update(report)
 
     def update(self, report):
-        if self.outcome is None or report.outcome != "passed":
-            self.outcome = report.outcome
-        if report.longreprtext:
-            self.messages.append(report.longreprtext)
+        # Allow the same nodeid/when to be overriden. This only happens when
+        # the test is re-run with flaky plugin. In that case, we want the
+        # latest result to override any previous results.
+        key = (report.nodeid, report.when)
+        self._outcomes[key] = report.outcome
+        self.messages[key] = report.longreprtext
+
+    @property
+    def outcome(self):
+        for outcome in self._outcomes.values():
+            if outcome != "passed":
+                return outcome
+        return "passed"
 
 
 @pytest.hookimpl(tryfirst=True, hookwrapper=True)
@@ -347,7 +356,7 @@ def system_test_dir(request, system_test_name, expected_artifacts):
         messages = []
         for node, result in test_results.items():
             isctest.log.debug("%s %s", result.outcome.upper(), node)
-            messages.extend(result.messages)
+            messages.extend(result.messages.values())
         for message in messages:
             isctest.log.debug("\n" + message)
         failed = any(res.outcome == "failed" for res in test_results.values())
diff --git a/bin/tests/system/dialup/clean.sh b/bin/tests/system/dialup/clean.sh
deleted file mode 100644
index 9ee72bc536..0000000000
--- a/bin/tests/system/dialup/clean.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-rm -f ns2/example.bk
-rm -f ns3/example.bk
-rm -f */named.memstats */named.run
-rm -f ns*/named.conf
diff --git a/bin/tests/system/enginepkcs11/tests_sh_enginepkcs11.py b/bin/tests/system/enginepkcs11/tests_sh_enginepkcs11.py
index 66f2fd9bf8..5a5fef62d5 100644
--- a/bin/tests/system/enginepkcs11/tests_sh_enginepkcs11.py
+++ b/bin/tests/system/enginepkcs11/tests_sh_enginepkcs11.py
@@ -46,6 +46,7 @@ pytestmark = pytest.mark.extra_artifacts(
         "ns1/*.weird.db",
         "ns1/*.weird.db.signed",
         "ns2/keys",
+        "ns2/named.args",
         "ns2/*.view*.db",
         "ns2/*.view*.db.signed",
     ]
diff --git a/bin/tests/system/keyfromlabel/setup.sh b/bin/tests/system/keyfromlabel/setup.sh
index 0d02b6b411..e302c60c68 100644
--- a/bin/tests/system/keyfromlabel/setup.sh
+++ b/bin/tests/system/keyfromlabel/setup.sh
@@ -16,6 +16,7 @@
 
 set -e
 
+OPENSSL_CONF= softhsm2-util --delete-token --token "softhsm2-keyfromlabel" >/dev/null 2>&1 || true
 OPENSSL_CONF= softhsm2-util --init-token --free --pin 1234 --so-pin 1234 --label "softhsm2-keyfromlabel" | awk '/^The token has been initialized and is reassigned to slot/ { print $NF }'
 
 printf '%s' "${HSMPIN:-1234}" >pin