diff --git a/bin/tests/system/checkconf/clean.sh b/bin/tests/system/checkconf/clean.sh
index 0d6001da76..8981c4586b 100644
--- a/bin/tests/system/checkconf/clean.sh
+++ b/bin/tests/system/checkconf/clean.sh
@@ -21,5 +21,6 @@ rm -f diff.out*
 rm -f good-kasp.conf.in
 rm -f good-server-christmas-tree.conf
 rm -f good.conf.in good.conf.out badzero.conf *.out
+rm -rf keys
 rm -f ns*/named.lock
 rm -rf test.keydir
diff --git a/bin/tests/system/checkconf/tests.sh b/bin/tests/system/checkconf/tests.sh
index c848136a05..6604a2ab58 100644
--- a/bin/tests/system/checkconf/tests.sh
+++ b/bin/tests/system/checkconf/tests.sh
@@ -18,7 +18,7 @@ set -e
 status=0
 n=0
 
-mkdir keys
+mkdir -p keys
 
 n=$((n + 1))
 echo_i "checking that named-checkconf handles a known good config ($n)"
@@ -657,7 +657,5 @@ grep -F "warning: ownername 'foo.*.check-wildcard' contains an non-terminal wild
 if [ $ret != 0 ]; then echo_i "failed"; ret=1; fi
 status=$((status + ret))
 
-rmdir keys
-
 echo_i "exit status: $status"
 [ $status -eq 0 ] || exit 1