From bdf983b72c1524c8febc92e50e58330ee1b39bb3 Mon Sep 17 00:00:00 2001
From: Matthijs Mekking <matthijs@isc.org>
Date: Fri, 10 Oct 2025 10:57:50 +0200
Subject: [PATCH 1/8] Convert model2.multisigner test to pytest

This converts the model2.multisigner tests from the multisigner system
test to pytest based code. Crappy shell test functions such as
'zsks_are_published', 'records_published' and others are replaced with
the standard test code from isctest.kasp and by setting 'private=False'
and 'legacy=True' on the keys from the other providers so we don't do
any key file testing.

(cherry picked from commit 773ce8d99b4c5074e0368e0c1ef2af250b043255)
---
 bin/tests/system/multisigner/ns3/setup.sh     |   2 -
 bin/tests/system/multisigner/ns4/setup.sh     |   2 -
 bin/tests/system/multisigner/tests.sh         | 415 ----------------
 .../system/multisigner/tests_multisigner.py   | 454 ++++++++++++++++++
 4 files changed, 454 insertions(+), 419 deletions(-)
 create mode 100644 bin/tests/system/multisigner/tests_multisigner.py

diff --git a/bin/tests/system/multisigner/ns3/setup.sh b/bin/tests/system/multisigner/ns3/setup.sh
index 123528f35c..80327a1300 100644
--- a/bin/tests/system/multisigner/ns3/setup.sh
+++ b/bin/tests/system/multisigner/ns3/setup.sh
@@ -28,8 +28,6 @@ KSK=$($KEYGEN -a $DEFAULT_ALGORITHM -f KSK -L 3600 $ksktimes $zone 2>keygen.out.
 ZSK=$($KEYGEN -a $DEFAULT_ALGORITHM -L 3600 $zsktimes $zone 2>keygen.out.$zone.2)
 $SETTIME -s -g $O -k $O now -r $O now -d $O now "$KSK" >settime.out.$zone.1 2>&1
 $SETTIME -s -g $O -k $O now -z $O now "$ZSK" >settime.out.$zone.2 2>&1
-# ZSK will be added to the other provider with nsupdate.
-cat "${ZSK}.key" | grep -v ";.*" >"${zone}.zsk"
 
 zone="model2.secondary"
 echo_i "setting up zone: $zone"
diff --git a/bin/tests/system/multisigner/ns4/setup.sh b/bin/tests/system/multisigner/ns4/setup.sh
index dc3fc7cebd..c58ddce495 100644
--- a/bin/tests/system/multisigner/ns4/setup.sh
+++ b/bin/tests/system/multisigner/ns4/setup.sh
@@ -28,8 +28,6 @@ KSK=$($KEYGEN -a $DEFAULT_ALGORITHM -f KSK -L 3600 $ksktimes $zone 2>keygen.out.
 ZSK=$($KEYGEN -a $DEFAULT_ALGORITHM -L 3600 $zsktimes $zone 2>keygen.out.$zone.2)
 $SETTIME -s -g $O -k $O now -r $O now -d $O now "$KSK" >settime.out.$zone.1 2>&1
 $SETTIME -s -g $O -k $O now -z $O now "$ZSK" >settime.out.$zone.2 2>&1
-# ZSK will be added to the other provider with nsupdate.
-cat "${ZSK}.key" | grep -v ";.*" >"${zone}.zsk"
 
 zone="model2.secondary"
 echo_i "setting up zone: $zone"
diff --git a/bin/tests/system/multisigner/tests.sh b/bin/tests/system/multisigner/tests.sh
index abe19ff215..c0141e8d85 100644
--- a/bin/tests/system/multisigner/tests.sh
+++ b/bin/tests/system/multisigner/tests.sh
@@ -26,82 +26,6 @@ start_time="$(TZ=UTC date +%s)"
 status=0
 n=0
 
-set_zone "model2.multisigner"
-set_policy "model2" "2" "3600"
-
-# Key properties and states.
-key_clear "KEY1"
-set_keyrole "KEY1" "ksk"
-set_keylifetime "KEY1" "0"
-set_keyalgorithm "KEY1" "13" "ECDSAP256SHA256" "256"
-set_keysigning "KEY1" "yes"
-set_zonesigning "KEY1" "no"
-set_keystate "KEY1" "GOAL" "omnipresent"
-set_keystate "KEY1" "STATE_DNSKEY" "omnipresent"
-set_keystate "KEY1" "STATE_KRRSIG" "omnipresent"
-set_keystate "KEY1" "STATE_DS" "omnipresent"
-
-key_clear "KEY2"
-set_keyrole "KEY2" "zsk"
-set_keylifetime "KEY2" "0"
-set_keyalgorithm "KEY2" "13" "ECDSAP256SHA256" "256"
-set_keysigning "KEY2" "no"
-set_zonesigning "KEY2" "yes"
-set_keystate "KEY2" "GOAL" "omnipresent"
-set_keystate "KEY2" "STATE_DNSKEY" "omnipresent"
-set_keystate "KEY2" "STATE_ZRRSIG" "omnipresent"
-
-key_clear "KEY3"
-key_clear "KEY4"
-
-set_keytimes_model2() {
-  # The first KSK is immediately published and activated.
-  created=$(key_get KEY1 CREATED)
-  set_keytime "KEY1" "PUBLISHED" "${created}"
-  set_keytime "KEY1" "ACTIVE" "${created}"
-  set_keytime "KEY1" "SYNCPUBLISH" "${created}"
-
-  # The first ZSKs are immediately published and activated.
-  created=$(key_get KEY2 CREATED)
-  set_keytime "KEY2" "PUBLISHED" "${created}"
-  set_keytime "KEY2" "ACTIVE" "${created}"
-}
-
-set_server "ns3" "10.53.0.3"
-check_keys
-check_dnssecstatus "$SERVER" "$POLICY" "$ZONE"
-set_keytimes_model2
-check_keytimes
-check_apex
-dnssec_verify
-
-set_server "ns4" "10.53.0.4"
-check_keys
-check_dnssecstatus "$SERVER" "$POLICY" "$ZONE"
-set_keytimes_model2
-check_keytimes
-check_apex
-dnssec_verify
-
-#
-# Update DNSKEY RRset.
-#
-
-# Check that the ZSKs from the other provider are published.
-zsks_are_published() {
-  dig_with_opts "$ZONE" "@${SERVER}" DNSKEY >"dig.out.$DIR.test$n" || return 1
-  cat dig.out.$DIR.test$n | tr [:blank:] ' ' >dig.out.$DIR.test$n.tr || return 1
-  # We should have two ZSKs.
-  lines=$(grep "256 3 13" dig.out.$DIR.test$n.tr | wc -l)
-  test "$lines" -eq 2 || return 1
-  # Both ZSKs are published.
-  grep "$(cat ns3/${ZONE}.zsk | tr [:blank:] ' ')" dig.out.$DIR.test$n.tr >/dev/null || return 1
-  grep "$(cat ns4/${ZONE}.zsk | tr [:blank:] ' ')" dig.out.$DIR.test$n.tr >/dev/null || return 1
-  # And one KSK.
-  lines=$(grep "257 3 13" dig.out.$DIR.test$n.tr | wc -l)
-  test "$lines" -eq 1 || return 1
-}
-
 # Test to make sure no DNSSEC records end up in the raw journal.
 no_dnssec_in_journal() {
   n=$((n + 1))
@@ -124,150 +48,6 @@ rrset_exists() (
   test "$lines" -gt 0
 )
 
-n=$((n + 1))
-echo_i "add dnskey record: update zone ${ZONE} at ns3 with ZSK from provider ns4 ($n)"
-ret=0
-set_server "ns3" "10.53.0.3"
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update add $(cat "ns4/${ZONE}.zsk")
-  echo send
-) | $NSUPDATE || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Check the new DNSKEY RRset.
-n=$((n + 1))
-echo_i "check zone ${ZONE} DNSKEY RRset after update ($n)"
-ret=0
-retry_quiet 10 zsks_are_published || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Check the logs for find zone keys errors.
-n=$((n + 1))
-echo_i "make sure we did not try to sign with the keys added with nsupdate for zone ${ZONE} ($n)"
-ret=0
-grep "dns_zone_findkeys: error reading ./K${ZONE}.*\.private: file not found" "${DIR}/named.run" && ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Verify again.
-dnssec_verify
-
-n=$((n + 1))
-echo_i "add dnskey record: - update zone ${ZONE} at ns4 with ZSK from provider ns3 ($n)"
-ret=0
-set_server "ns4" "10.53.0.4"
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update add $(cat "ns3/${ZONE}.zsk")
-  echo send
-) | $NSUPDATE || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Check the new DNSKEY RRset.
-n=$((n + 1))
-echo_i "check zone ${ZONE} DNSKEY RRset after update ($n)"
-ret=0
-retry_quiet 10 zsks_are_published || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Check the logs for find zone keys errors.
-n=$((n + 1))
-echo_i "make sure we did not try to sign with the keys added with nsupdate for zone ${ZONE} ($n)"
-ret=0
-grep "dns_zone_findkeys: error reading ./K${ZONE}.*\.private: file not found" "${DIR}/named.run" && ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Verify again.
-dnssec_verify
-no_dnssec_in_journal
-
-n=$((n + 1))
-echo_i "remove dnskey record: - try to remove ns3 ZSK from provider ns3 (should fail) ($n)"
-ret=0
-set_server "ns3" "10.53.0.3"
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update del $(cat "ns3/${ZONE}.zsk")
-  echo send
-) | $NSUPDATE || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Both ZSKs should still be published.
-n=$((n + 1))
-echo_i "check zone ${ZONE} DNSKEY RRset after failed update ($n)"
-ret=0
-retry_quiet 10 zsks_are_published || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-
-n=$((n + 1))
-echo_i "remove dnskey record: remove ns4 ZSK from provider ns3 ($n)"
-ret=0
-set_server "ns3" "10.53.0.3"
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update del $(cat "ns4/${ZONE}.zsk")
-  echo send
-) | $NSUPDATE || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# We should have only the KSK and ZSK from provider ns3.
-n=$((n + 1))
-echo_i "check zone ${ZONE} DNSKEY RRset after update ($n)"
-ret=0
-check_keys
-check_apex
-dnssec_verify
-
-n=$((n + 1))
-echo_i "remove dnskey record: try to remove ns4 ZSK from provider ns4 (should fail) ($n)"
-ret=0
-set_server "ns4" "10.53.0.4"
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update del $(cat "ns4/${ZONE}.zsk")
-  echo send
-) | $NSUPDATE || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Both ZSKs should still be published.
-n=$((n + 1))
-echo_i "check zone ${ZONE} DNSKEY RRset after failed update ($n)"
-ret=0
-retry_quiet 10 zsks_are_published || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-
-n=$((n + 1))
-echo_i "remove dnskey record: remove ns3 ZSK from provider ns4 ($n)"
-ret=0
-set_server "ns4" "10.53.0.4"
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update del $(cat "ns3/${ZONE}.zsk")
-  echo send
-) | $NSUPDATE
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# We should have only the KSK and ZSK from provider ns4.
-n=$((n + 1))
-echo_i "check zone ${ZONE} DNSKEY RRset after update ($n)"
-ret=0
-check_keys
-check_apex
-dnssec_verify
-no_dnssec_in_journal
-
-#
-# Update CDNSKEY RRset.
-#
-
 # Check that the CDNSKEY from both providers are published.
 records_published() {
   _rrtype=$1
@@ -278,201 +58,6 @@ records_published() {
   test "$lines" -eq "$_expect" || return 1
 }
 
-# Retrieve CDNSKEY records from the other provider.
-dig_with_opts ${ZONE} @10.53.0.3 CDNSKEY >dig.out.ns3.cdnskey
-awk '$4 == "CDNSKEY" {print}' dig.out.ns3.cdnskey >cdnskey.ns3
-dig_with_opts ${ZONE} @10.53.0.4 CDNSKEY >dig.out.ns4.cdnskey
-awk '$4 == "CDNSKEY" {print}' dig.out.ns4.cdnskey >cdnskey.ns4
-
-n=$((n + 1))
-echo_i "add cdnskey record: update zone ${ZONE} at ns3 with CDNSKEY from provider ns4 ($n)"
-ret=0
-set_server "ns3" "10.53.0.3"
-# Initially there should be one CDNSKEY.
-retry_quiet 10 records_published CDNSKEY 1 || ret=1
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update add $(cat "cdnskey.ns4")
-  echo send
-) | $NSUPDATE || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Now there should be two CDNSKEY records (we test that BIND does not
-# skip it during DNSSEC maintenance).
-n=$((n + 1))
-echo_i "check zone ${ZONE} CDNSKEY RRset after update ($n)"
-ret=0
-retry_quiet 10 records_published CDNSKEY 2 || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-
-n=$((n + 1))
-echo_i "add cdnskey record: update zone ${ZONE} at ns4 with CDNSKEY from provider ns3 ($n)"
-ret=0
-set_server "ns4" "10.53.0.4"
-# Initially there should be one CDNSKEY.
-retry_quiet 10 records_published CDNSKEY 1 || ret=1
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update add $(cat "cdnskey.ns3")
-  echo send
-) | $NSUPDATE || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Now there should be two CDNSKEY records (we test that BIND does not
-# skip it during DNSSEC maintenance).
-n=$((n + 1))
-echo_i "check zone ${ZONE} CDNSKEY RRset after update ($n)"
-ret=0
-retry_quiet 10 records_published CDNSKEY 2 || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# No DNSSEC in raw journal.
-no_dnssec_in_journal
-
-n=$((n + 1))
-echo_i "remove cdnskey record: remove ns4 CDNSKEY from provider ns3 ($n)"
-ret=0
-set_server "ns3" "10.53.0.3"
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update del $(cat "cdnskey.ns4")
-  echo send
-) | $NSUPDATE || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Now there should be one CDNSKEY record again.
-n=$((n + 1))
-echo_i "check zone ${ZONE} CDNSKEY RRset after update ($n)"
-ret=0
-retry_quiet 10 records_published CDNSKEY 1 || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-
-n=$((n + 1))
-echo_i "remove cdnskey record: remove ns3 CDNSKEY from provider ns4 ($n)"
-ret=0
-set_server "ns4" "10.53.0.4"
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update del $(cat "cdnskey.ns3")
-  echo send
-) | $NSUPDATE || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Now there should be one CDNSKEY record again.
-n=$((n + 1))
-echo_i "check zone ${ZONE} CDNSKEY RRset after update ($n)"ret=0
-retry_quiet 10 records_published CDNSKEY 1 || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# No DNSSEC in raw journal.
-no_dnssec_in_journal
-
-#
-# Update CDS RRset.
-#
-
-# Retrieve CDS records from the other provider.
-dig_with_opts ${ZONE} @10.53.0.3 CDS >dig.out.ns3.cds
-awk '$4 == "CDS" {print}' dig.out.ns3.cds >cds.ns3
-dig_with_opts ${ZONE} @10.53.0.4 CDS >dig.out.ns4.cds
-awk '$4 == "CDS" {print}' dig.out.ns4.cds >cds.ns4
-
-n=$((n + 1))
-echo_i "add cds record: update zone ${ZONE} at ns3 with CDS from provider ns4 ($n)"
-ret=0
-set_server "ns3" "10.53.0.3"
-# Initially there should be one CDS.
-retry_quiet 10 records_published CDS 1 || ret=1
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update add $(cat "cds.ns4")
-  echo send
-) | $NSUPDATE || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Now there should be two CDS records (we test that BIND does not
-# skip it during DNSSEC maintenance).
-n=$((n + 1))
-echo_i "check zone ${ZONE} CDS RRset after update ($n)"
-ret=0
-retry_quiet 10 records_published CDS 2 || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-
-n=$((n + 1))
-echo_i "add cds record: update zone ${ZONE} at ns4 with CDS from provider ns3 ($n)"
-ret=0
-set_server "ns4" "10.53.0.4"
-# Initially there should be one CDS.
-retry_quiet 10 records_published CDS 1 || ret=1
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update add $(cat "cds.ns3")
-  echo send
-) | $NSUPDATE || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Now there should be two CDS records (we test that BIND does not
-# skip it during DNSSEC maintenance).
-n=$((n + 1))
-echo_i "check zone ${ZONE} CDS RRset after update ($n)"
-ret=0
-retry_quiet 10 records_published CDS 2 || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# No DNSSEC in raw journal.
-no_dnssec_in_journal
-
-n=$((n + 1))
-echo_i "remove cds record: remove ns4 CDS from provider ns3 ($n)"
-ret=0
-set_server "ns3" "10.53.0.3"
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update del $(cat "cds.ns4")
-  echo send
-) | $NSUPDATE || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Now there should be one CDS record again.
-n=$((n + 1))
-echo_i "check zone ${ZONE} CDS RRset after update ($n)"
-ret=0
-retry_quiet 10 records_published CDS 1 || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-
-n=$((n + 1))
-echo_i "remove cds record: remove ns3 CDS from provider ns4 ($n)"
-ret=0
-set_server "ns4" "10.53.0.4"
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update del $(cat "cds.ns3")
-  echo send
-) | $NSUPDATE || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Now there should be one CDS record again.
-n=$((n + 1))
-echo_i "check zone ${ZONE} CDS RRset after update ($n)"
-ret=0
-retry_quiet 10 records_published CDS 1 || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# No DNSSEC in raw journal.
-no_dnssec_in_journal
-
 #
 # Check secondary server behaviour.
 #
diff --git a/bin/tests/system/multisigner/tests_multisigner.py b/bin/tests/system/multisigner/tests_multisigner.py
new file mode 100644
index 0000000000..9638bc3e26
--- /dev/null
+++ b/bin/tests/system/multisigner/tests_multisigner.py
@@ -0,0 +1,454 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+from datetime import timedelta
+import os
+import re
+
+import pytest
+
+pytest.importorskip("dns", minversion="2.0.0")
+import dns
+import dns.update
+
+import isctest
+
+pytestmark = pytest.mark.extra_artifacts(
+    [
+        "*.axfr",
+        "*.created",
+        "cdnskey.ns*",
+        "cds.ns*",
+        "dig.out.*",
+        "rndc.dnssec.status.out.*",
+        "secondary.cdnskey.ns*",
+        "secondary.cds.ns*",
+        "unused.*",
+        "verify.out.*",
+        "ns*/K*",
+        "ns*/db-*",
+        "ns*/keygen.out.*",
+        "ns*/*.jbk",
+        "ns*/*.jnl",
+        "ns*/*.zsk",
+        "ns*/*.signed",
+        "ns*/*.journal.out.*",
+        "ns*/settime.out.*",
+        "ns*/model2.secondary.db",
+    ]
+)
+
+ALGORITHM = os.environ["DEFAULT_ALGORITHM_NUMBER"]
+SIZE = os.environ["DEFAULT_BITS"]
+CONFIG = {
+    "dnskey-ttl": timedelta(hours=1),
+    "ds-ttl": timedelta(days=1),
+    "max-zone-ttl": timedelta(days=1),
+    "parent-propagation-delay": timedelta(hours=1),
+    "publish-safety": timedelta(hours=1),
+    "retire-safety": timedelta(hours=1),
+    "signatures-refresh": timedelta(days=5),
+    "signatures-validity": timedelta(days=14),
+    "zone-propagation-delay": timedelta(minutes=5),
+}
+TTL = 3600
+
+
+def dsfromkey(key):
+    dsfromkey_command = [
+        os.environ.get("DSFROMKEY"),
+        "-T",
+        str(TTL),
+        "-a",
+        "SHA-256",
+        "-C",
+        "-w",
+        str(key.keyfile),
+    ]
+    out = isctest.run.cmd(dsfromkey_command)
+    return out.stdout.decode("utf-8").split()
+
+
+def check_dnssec(server, zone, keys, expected):
+    ksks = [k for k in keys if k.is_ksk()]
+    zsks = [k for k in keys if not k.is_ksk()]
+
+    isctest.kasp.check_keys(zone, keys, expected)
+
+    for kp in expected:
+        kp.set_expected_keytimes(CONFIG)
+        kp.set_expected_keytimes(CONFIG)
+        start = kp.key.get_timing("Created")
+        kp.timing["Published"] = start
+        kp.timing["Active"] = start
+        if kp.role != "zsk":
+            kp.timing["PublishCDS"] = start
+
+    isctest.kasp.check_dnssec_verify(server, zone)
+    isctest.kasp.check_apex(server, zone, ksks, zsks)
+
+
+def check_no_dnssec_in_journal(server, zone):
+    journalprint = [
+        os.environ.get("JOURNALPRINT"),
+        f"{server.identifier}/{zone}.db.jnl",
+    ]
+
+    out = isctest.run.cmd(journalprint)
+    contents = out.stdout.decode("utf-8")
+    pattern = re.compile(
+        r"^\s*(?:\S+\s+){4}(NSEC|NSEC3|NSEC3PARAM|RRSIG)", flags=re.MULTILINE
+    )
+    match = pattern.search(contents)
+    assert not match, f"{match.group(1)} record found in journal"
+
+
+def check_add_zsk(server, zone, keys, expected, zsk, extra):
+    isctest.log.info("add dnskey record:")
+
+    isctest.log.info(
+        f"- zone {zone} {server.identifier}: update zone with ZSK from other provider"
+    )
+
+    dnskey = zsk.dnskey().split()
+    rdata = " ".join(dnskey[4:])
+    update_msg = dns.update.UpdateMessage(zone)
+    update_msg.add(f"{zone}.", TTL, "DNSKEY", rdata)
+    server.nsupdate(update_msg)
+
+    # Check the new DNSKEY RRset.
+    isctest.log.info(
+        f"- zone {zone} {server.identifier}: check DNSKEY RRset after update add"
+    )
+    check_dnssec(server, zone, keys + [zsk], expected + extra)
+
+    # Check the logs for find zone keys errors.
+    isctest.log.info(
+        f"- zone {zone} {server.identifier}: make sure we did not try to sign with the keys added with nsupdate"
+    )
+    server.log.prohibit(f"dns_zone_findkeys: error reading ./K{zone}")
+
+    # Trigger keymgr.
+    with server.watch_log_from_here() as watcher:
+        server.rndc(f"loadkeys {zone}", log=False)
+        watcher.wait_for_line(f"keymgr: {zone} done")
+
+    # Check again.
+    isctest.log.info(f"- zone {zone} {server.identifier}: check again after keymgr run")
+    check_dnssec(server, zone, keys + [zsk], expected + extra)
+    server.log.prohibit(f"dns_zone_findkeys: error reading ./K{zone}")
+
+
+def check_remove_zsk(server, zone, keys, expected, zsk, extra):
+    isctest.log.info("remove dnskey record:")
+
+    isctest.log.info(
+        f"- zone {zone} {server.identifier}: try to remove own ZSK (should fail)"
+    )
+
+    zsks = [k for k in keys if not k.is_ksk()]
+    dnskey = zsks[0].dnskey().split()
+    rdata = " ".join(dnskey[4:])
+    update_msg = dns.update.UpdateMessage(zone)
+    update_msg.delete(f"{zone}.", "DNSKEY", rdata)
+    with server.watch_log_from_here() as watcher:
+        server.nsupdate(update_msg)
+        watcher.wait_for_line(
+            f"updating zone '{zone}/IN': attempt to delete in use DNSKEY ignored"
+        )
+
+    # Both ZSKs should still be published.
+    isctest.log.info(
+        f"- zone {zone} {server.identifier}: check DNSKEY RRset after update remove"
+    )
+    check_dnssec(server, zone, keys + [zsk], expected + extra)
+
+    # Trigger keymgr.
+    with server.watch_log_from_here() as watcher:
+        server.rndc(f"loadkeys {zone}", log=False)
+        watcher.wait_for_line(f"keymgr: {zone} done")
+
+    # Check again.
+    isctest.log.info(f"- zone {zone} {server.identifier}: check again after keymgr run")
+    check_dnssec(server, zone, keys + [zsk], expected + extra)
+
+    # Remove actual ZSK.
+    isctest.log.info(
+        f"- zone {zone} {server.identifier}: remove ZSK from other provider"
+    )
+
+    dnskey = zsk.dnskey().split()
+    rdata = " ".join(dnskey[4:])
+    update_msg = dns.update.UpdateMessage(zone)
+    update_msg.delete(f"{zone}.", "DNSKEY", rdata)
+    server.nsupdate(update_msg)
+
+    # We should have only the KSK and ZSK from server.
+    isctest.log.info(
+        f"- zone {zone} {server.identifier}: check DNSKEY RRset after update remove"
+    )
+    check_dnssec(server, zone, keys, expected)
+
+    # Trigger keymgr.
+    with server.watch_log_from_here() as watcher:
+        server.rndc(f"loadkeys {zone}", log=False)
+        watcher.wait_for_line(f"keymgr: {zone} done")
+
+    # Check again.
+    isctest.log.info(f"- zone {zone} {server.identifier}: check again after keymgr run")
+    check_dnssec(server, zone, keys, expected)
+
+
+def check_add_cdnskey(server, zone, keys, expected, ksk, extra):
+    isctest.log.info("add cdnskey record:")
+
+    isctest.log.info(
+        f"- zone {zone} {server.identifier}: update zone with CDNSKEY from other provider"
+    )
+
+    # Retrieve CDNSKEY records from the other provider.
+    dnskey = ksk.dnskey().split()
+    rdata = " ".join(dnskey[4:])
+    update_msg = dns.update.UpdateMessage(zone)
+    update_msg.add(f"{zone}.", TTL, "CDNSKEY", rdata)
+    server.nsupdate(update_msg)
+
+    # Now there should be two CDNSKEY records.
+    isctest.log.info(
+        f"- zone {zone} {server.identifier}: check CDNSKEY RRset after update add"
+    )
+    check_dnssec(server, zone, keys + [ksk], expected + extra)
+
+    # Trigger keymgr.
+    with server.watch_log_from_here() as watcher:
+        server.rndc(f"loadkeys {zone}", log=False)
+        watcher.wait_for_line(f"keymgr: {zone} done")
+
+    # Check again.
+    isctest.log.info(f"- zone {zone} {server.identifier}: check again after keymgr run")
+    check_dnssec(server, zone, keys + [ksk], expected + extra)
+
+
+def check_remove_cdnskey(server, zone, keys, expected, ksk, extra):
+    isctest.log.info("remove cdnskey record:")
+
+    isctest.log.info(
+        f"- zone {zone} {server.identifier}: try to remove own CDNSKEY (should fail)"
+    )
+
+    ksks = [k for k in keys if not k.is_ksk()]
+    dnskey = ksks[0].dnskey().split()
+    rdata = " ".join(dnskey[4:])
+    update_msg = dns.update.UpdateMessage(zone)
+    update_msg.delete(f"{zone}.", "CDNSKEY", rdata)
+    with server.watch_log_from_here() as watcher:
+        server.nsupdate(update_msg)
+        watcher.wait_for_line(
+            f"updating zone '{zone}/IN': attempt to delete in use CDNSKEY ignored"
+        )
+
+    # Both CDNSKEY records should still be published.
+    isctest.log.info(
+        f"- zone {zone} {server.identifier}: check CDNSKEY RRset after update remove"
+    )
+    check_dnssec(server, zone, keys + [ksk], expected + extra)
+
+    # Trigger keymgr.
+    with server.watch_log_from_here() as watcher:
+        server.rndc(f"loadkeys {zone}", log=False)
+        watcher.wait_for_line(f"keymgr: {zone} done")
+
+    # Check again.
+    isctest.log.info(f"- zone {zone} {server.identifier}: check again after keymgr run")
+    check_dnssec(server, zone, keys + [ksk], expected + extra)
+
+    # Remove actual CDNSKEY.
+    isctest.log.info(
+        f"- zone {zone} {server.identifier}: remove CDNSKEY from other provider"
+    )
+
+    dnskey = ksk.dnskey().split()
+    rdata = " ".join(dnskey[4:])
+    update_msg = dns.update.UpdateMessage(zone)
+    update_msg.delete(f"{zone}.", "CDNSKEY", rdata)
+    server.nsupdate(update_msg)
+
+    # Now there should be one CDNSKEY record again.
+    isctest.log.info(
+        f"- zone {zone} {server.identifier}: check CDNSKEY RRset after update remove"
+    )
+    check_dnssec(server, zone, keys, expected)
+
+    # Trigger keymgr.
+    with server.watch_log_from_here() as watcher:
+        server.rndc(f"loadkeys {zone}", log=False)
+        watcher.wait_for_line(f"keymgr: {zone} done")
+
+    # Check again.
+    isctest.log.info(f"- zone {zone} {server.identifier}: check again after keymgr run")
+    check_dnssec(server, zone, keys, expected)
+
+
+def check_add_cds(server, zone, keys, expected, ksk, extra):
+    isctest.log.info("add cds record:")
+
+    isctest.log.info(
+        f"- zone {zone} {server.identifier}: update zone with CDS from other provider"
+    )
+
+    # Retrieve CDS records from the other provider.
+    ds = dsfromkey(ksk)
+    rdata = " ".join(ds[4:])
+    update_msg = dns.update.UpdateMessage(zone)
+    update_msg.add(f"{zone}.", TTL, "CDS", rdata)
+    server.nsupdate(update_msg)
+
+    # Now there should be two CDS records.
+    isctest.log.info(
+        f"- zone {zone} {server.identifier}: check CDS RRset after update add"
+    )
+    check_dnssec(server, zone, keys + [ksk], expected + extra)
+
+    # Trigger keymgr.
+    with server.watch_log_from_here() as watcher:
+        server.rndc(f"loadkeys {zone}", log=False)
+        watcher.wait_for_line(f"keymgr: {zone} done")
+
+    # Check again.
+    isctest.log.info(f"- zone {zone} {server.identifier}: check again after keymgr run")
+    check_dnssec(server, zone, keys + [ksk], expected + extra)
+
+
+def check_remove_cds(server, zone, keys, expected, ksk, extra):
+    isctest.log.info("remove cds record:")
+
+    isctest.log.info(
+        f"- zone {zone} {server.identifier}: try to remove own CDS (should fail)"
+    )
+
+    ksks = [k for k in keys if not k.is_ksk()]
+    ds = dsfromkey(ksks[0])
+    rdata = " ".join(ds[4:])
+    update_msg = dns.update.UpdateMessage(zone)
+    update_msg.delete(f"{zone}.", "CDS", rdata)
+    with server.watch_log_from_here() as watcher:
+        server.nsupdate(update_msg)
+        watcher.wait_for_line(
+            f"updating zone '{zone}/IN': attempt to delete in use CDS ignored"
+        )
+
+    # Both CDS records should still be published.
+    isctest.log.info(
+        f"- zone {zone} {server.identifier}: check CDS RRset after update remove"
+    )
+    check_dnssec(server, zone, keys + [ksk], expected + extra)
+
+    # Trigger keymgr.
+    with server.watch_log_from_here() as watcher:
+        server.rndc(f"loadkeys {zone}", log=False)
+        watcher.wait_for_line(f"keymgr: {zone} done")
+
+    # Check again.
+    isctest.log.info(f"- zone {zone} {server.identifier}: check again after keymgr run")
+    check_dnssec(server, zone, keys + [ksk], expected + extra)
+
+    # Remove actual CDS.
+    isctest.log.info(
+        f"- zone {zone} {server.identifier}: remove CDS from other provider"
+    )
+
+    ds = dsfromkey(ksk)
+    rdata = " ".join(ds[4:])
+    update_msg = dns.update.UpdateMessage(zone)
+    update_msg.delete(f"{zone}.", "CDS", rdata)
+    server.nsupdate(update_msg)
+
+    # Now there should be one CDS record again.
+    isctest.log.info(
+        f"- zone {zone} {server.identifier}: check CDS RRset after update remove"
+    )
+    check_dnssec(server, zone, keys, expected)
+
+    # Trigger keymgr.
+    with server.watch_log_from_here() as watcher:
+        server.rndc(f"loadkeys {zone}", log=False)
+        watcher.wait_for_line(f"keymgr: {zone} done")
+
+    # Check again.
+    isctest.log.info(f"- zone {zone} {server.identifier}: check again after keymgr run")
+    check_dnssec(server, zone, keys, expected)
+
+
+def test_multisigner(ns3, ns4):
+    zone = "model2.multisigner"
+    keyprops = [
+        f"ksk 0 {ALGORITHM} {SIZE} goal:omnipresent dnskey:omnipresent krrsig:omnipresent ds:omnipresent",
+        f"zsk 0 {ALGORITHM} {SIZE} goal:omnipresent dnskey:omnipresent zrrsig:omnipresent",
+    ]
+
+    # First make sure the zone is properly signed.
+    isctest.log.info(f"basic DNSSEC tests for {zone}")
+    isctest.kasp.wait_keymgr_done(ns3, zone)
+    isctest.kasp.wait_keymgr_done(ns4, zone)
+
+    keys3 = isctest.kasp.keydir_to_keylist(zone, ns3.identifier)
+    ksks3 = [k for k in keys3 if k.is_ksk()]
+    zsks3 = [k for k in keys3 if not k.is_ksk()]
+    expected3 = isctest.kasp.policy_to_properties(ttl=TTL, keys=keyprops)
+
+    check_dnssec(ns3, zone, keys3, expected3)
+
+    keys4 = isctest.kasp.keydir_to_keylist(zone, ns4.identifier)
+    ksks4 = [k for k in keys4 if k.is_ksk()]
+    zsks4 = [k for k in keys4 if not k.is_ksk()]
+    expected4 = isctest.kasp.policy_to_properties(ttl=TTL, keys=keyprops)
+
+    check_dnssec(ns4, zone, keys4, expected4)
+
+    # Add DNSKEY to RRset.
+    newprops = [f"zsk unlimited {ALGORITHM} {SIZE}"]
+    extra = isctest.kasp.policy_to_properties(ttl=TTL, keys=newprops)
+    extra[0].private = False  # noqa
+    extra[0].legacy = True  # noqa
+
+    check_add_zsk(ns3, zone, keys3, expected3, zsks4[0], extra)
+    check_add_zsk(ns4, zone, keys4, expected4, zsks3[0], extra)
+    check_no_dnssec_in_journal(ns4, zone)
+
+    # Remove DNSKEY from RRset.
+    check_remove_zsk(ns3, zone, keys3, expected3, zsks4[0], extra)
+    check_remove_zsk(ns4, zone, keys4, expected4, zsks3[0], extra)
+    check_no_dnssec_in_journal(ns4, zone)
+
+    # Add CDNSKEY RRset.
+    newprops = [f"ksk unlimited {ALGORITHM} {SIZE}"]
+    extra = isctest.kasp.policy_to_properties(ttl=TTL, keys=newprops)
+    extra[0].private = False  # noqa
+    extra[0].legacy = True  # noqa
+
+    check_add_cdnskey(ns3, zone, keys3, expected3, ksks4[0], extra)
+    check_add_cdnskey(ns4, zone, keys4, expected4, ksks3[0], extra)
+    check_no_dnssec_in_journal(ns4, zone)
+
+    # Remove CDNSKEY RRset.
+    check_remove_cdnskey(ns3, zone, keys3, expected3, ksks4[0], extra)
+    check_remove_cdnskey(ns4, zone, keys4, expected4, ksks3[0], extra)
+    check_no_dnssec_in_journal(ns4, zone)
+
+    # Update CDS RRset.
+    check_add_cds(ns3, zone, keys3, expected3, ksks4[0], extra)
+    check_add_cds(ns4, zone, keys4, expected4, ksks3[0], extra)
+    check_no_dnssec_in_journal(ns4, zone)
+
+    # Remove CDS RRset.
+    check_remove_cds(ns3, zone, keys3, expected3, ksks4[0], extra)
+    check_remove_cds(ns4, zone, keys4, expected4, ksks3[0], extra)
+    check_no_dnssec_in_journal(ns4, zone)

From 941cd550102c3def5740970de72bdea2c14505ea Mon Sep 17 00:00:00 2001
From: Matthijs Mekking <matthijs@isc.org>
Date: Fri, 10 Oct 2025 17:17:38 +0200
Subject: [PATCH 2/8] Make test code flexible for more providers

The initial test code only allowed for one additional provider. Update
the test function such that more extra keys can be tested.

(cherry picked from commit 9ae449afd153692b5a81ea7d93d7477d0cb7945e)
---
 .../system/multisigner/tests_multisigner.py   | 118 +++++++++---------
 1 file changed, 62 insertions(+), 56 deletions(-)

diff --git a/bin/tests/system/multisigner/tests_multisigner.py b/bin/tests/system/multisigner/tests_multisigner.py
index 9638bc3e26..86610e80de 100644
--- a/bin/tests/system/multisigner/tests_multisigner.py
+++ b/bin/tests/system/multisigner/tests_multisigner.py
@@ -111,24 +111,25 @@ def check_no_dnssec_in_journal(server, zone):
     assert not match, f"{match.group(1)} record found in journal"
 
 
-def check_add_zsk(server, zone, keys, expected, zsk, extra):
+def check_add_zsk(server, zone, keys, expected, extra_keys, extra):
     isctest.log.info("add dnskey record:")
 
     isctest.log.info(
-        f"- zone {zone} {server.identifier}: update zone with ZSK from other provider"
+        f"- zone {zone} {server.identifier}: update zone with ZSK from other providers"
     )
 
-    dnskey = zsk.dnskey().split()
-    rdata = " ".join(dnskey[4:])
     update_msg = dns.update.UpdateMessage(zone)
-    update_msg.add(f"{zone}.", TTL, "DNSKEY", rdata)
+    for zsk in extra_keys:
+        dnskey = zsk.dnskey().split()
+        rdata = " ".join(dnskey[4:])
+        update_msg.add(f"{zone}.", TTL, "DNSKEY", rdata)
     server.nsupdate(update_msg)
 
     # Check the new DNSKEY RRset.
     isctest.log.info(
         f"- zone {zone} {server.identifier}: check DNSKEY RRset after update add"
     )
-    check_dnssec(server, zone, keys + [zsk], expected + extra)
+    check_dnssec(server, zone, keys + extra_keys, expected + extra)
 
     # Check the logs for find zone keys errors.
     isctest.log.info(
@@ -143,11 +144,11 @@ def check_add_zsk(server, zone, keys, expected, zsk, extra):
 
     # Check again.
     isctest.log.info(f"- zone {zone} {server.identifier}: check again after keymgr run")
-    check_dnssec(server, zone, keys + [zsk], expected + extra)
+    check_dnssec(server, zone, keys + extra_keys, expected + extra)
     server.log.prohibit(f"dns_zone_findkeys: error reading ./K{zone}")
 
 
-def check_remove_zsk(server, zone, keys, expected, zsk, extra):
+def check_remove_zsk(server, zone, keys, expected, extra_keys, extra):
     isctest.log.info("remove dnskey record:")
 
     isctest.log.info(
@@ -169,7 +170,7 @@ def check_remove_zsk(server, zone, keys, expected, zsk, extra):
     isctest.log.info(
         f"- zone {zone} {server.identifier}: check DNSKEY RRset after update remove"
     )
-    check_dnssec(server, zone, keys + [zsk], expected + extra)
+    check_dnssec(server, zone, keys + extra_keys, expected + extra)
 
     # Trigger keymgr.
     with server.watch_log_from_here() as watcher:
@@ -178,17 +179,18 @@ def check_remove_zsk(server, zone, keys, expected, zsk, extra):
 
     # Check again.
     isctest.log.info(f"- zone {zone} {server.identifier}: check again after keymgr run")
-    check_dnssec(server, zone, keys + [zsk], expected + extra)
+    check_dnssec(server, zone, keys + extra_keys, expected + extra)
 
     # Remove actual ZSK.
     isctest.log.info(
-        f"- zone {zone} {server.identifier}: remove ZSK from other provider"
+        f"- zone {zone} {server.identifier}: remove ZSK from other providers"
     )
 
-    dnskey = zsk.dnskey().split()
-    rdata = " ".join(dnskey[4:])
     update_msg = dns.update.UpdateMessage(zone)
-    update_msg.delete(f"{zone}.", "DNSKEY", rdata)
+    for zsk in extra_keys:
+        dnskey = zsk.dnskey().split()
+        rdata = " ".join(dnskey[4:])
+        update_msg.delete(f"{zone}.", "DNSKEY", rdata)
     server.nsupdate(update_msg)
 
     # We should have only the KSK and ZSK from server.
@@ -207,25 +209,26 @@ def check_remove_zsk(server, zone, keys, expected, zsk, extra):
     check_dnssec(server, zone, keys, expected)
 
 
-def check_add_cdnskey(server, zone, keys, expected, ksk, extra):
+def check_add_cdnskey(server, zone, keys, expected, extra_keys, extra):
     isctest.log.info("add cdnskey record:")
 
     isctest.log.info(
-        f"- zone {zone} {server.identifier}: update zone with CDNSKEY from other provider"
+        f"- zone {zone} {server.identifier}: update zone with CDNSKEY from other providers"
     )
 
-    # Retrieve CDNSKEY records from the other provider.
-    dnskey = ksk.dnskey().split()
-    rdata = " ".join(dnskey[4:])
+    # Retrieve CDNSKEY records from the other providers.
     update_msg = dns.update.UpdateMessage(zone)
-    update_msg.add(f"{zone}.", TTL, "CDNSKEY", rdata)
+    for ksk in extra_keys:
+        dnskey = ksk.dnskey().split()
+        rdata = " ".join(dnskey[4:])
+        update_msg.add(f"{zone}.", TTL, "CDNSKEY", rdata)
     server.nsupdate(update_msg)
 
     # Now there should be two CDNSKEY records.
     isctest.log.info(
         f"- zone {zone} {server.identifier}: check CDNSKEY RRset after update add"
     )
-    check_dnssec(server, zone, keys + [ksk], expected + extra)
+    check_dnssec(server, zone, keys + extra_keys, expected + extra)
 
     # Trigger keymgr.
     with server.watch_log_from_here() as watcher:
@@ -234,10 +237,10 @@ def check_add_cdnskey(server, zone, keys, expected, ksk, extra):
 
     # Check again.
     isctest.log.info(f"- zone {zone} {server.identifier}: check again after keymgr run")
-    check_dnssec(server, zone, keys + [ksk], expected + extra)
+    check_dnssec(server, zone, keys + extra_keys, expected + extra)
 
 
-def check_remove_cdnskey(server, zone, keys, expected, ksk, extra):
+def check_remove_cdnskey(server, zone, keys, expected, extra_keys, extra):
     isctest.log.info("remove cdnskey record:")
 
     isctest.log.info(
@@ -259,7 +262,7 @@ def check_remove_cdnskey(server, zone, keys, expected, ksk, extra):
     isctest.log.info(
         f"- zone {zone} {server.identifier}: check CDNSKEY RRset after update remove"
     )
-    check_dnssec(server, zone, keys + [ksk], expected + extra)
+    check_dnssec(server, zone, keys + extra_keys, expected + extra)
 
     # Trigger keymgr.
     with server.watch_log_from_here() as watcher:
@@ -268,17 +271,18 @@ def check_remove_cdnskey(server, zone, keys, expected, ksk, extra):
 
     # Check again.
     isctest.log.info(f"- zone {zone} {server.identifier}: check again after keymgr run")
-    check_dnssec(server, zone, keys + [ksk], expected + extra)
+    check_dnssec(server, zone, keys + extra_keys, expected + extra)
 
     # Remove actual CDNSKEY.
     isctest.log.info(
-        f"- zone {zone} {server.identifier}: remove CDNSKEY from other provider"
+        f"- zone {zone} {server.identifier}: remove CDNSKEY from other providers"
     )
 
-    dnskey = ksk.dnskey().split()
-    rdata = " ".join(dnskey[4:])
     update_msg = dns.update.UpdateMessage(zone)
-    update_msg.delete(f"{zone}.", "CDNSKEY", rdata)
+    for ksk in extra_keys:
+        dnskey = ksk.dnskey().split()
+        rdata = " ".join(dnskey[4:])
+        update_msg.delete(f"{zone}.", "CDNSKEY", rdata)
     server.nsupdate(update_msg)
 
     # Now there should be one CDNSKEY record again.
@@ -297,25 +301,26 @@ def check_remove_cdnskey(server, zone, keys, expected, ksk, extra):
     check_dnssec(server, zone, keys, expected)
 
 
-def check_add_cds(server, zone, keys, expected, ksk, extra):
+def check_add_cds(server, zone, keys, expected, extra_keys, extra):
     isctest.log.info("add cds record:")
 
     isctest.log.info(
-        f"- zone {zone} {server.identifier}: update zone with CDS from other provider"
+        f"- zone {zone} {server.identifier}: update zone with CDS from other providers"
     )
 
-    # Retrieve CDS records from the other provider.
-    ds = dsfromkey(ksk)
-    rdata = " ".join(ds[4:])
+    # Retrieve CDS records from the other providers.
     update_msg = dns.update.UpdateMessage(zone)
-    update_msg.add(f"{zone}.", TTL, "CDS", rdata)
+    for ksk in extra_keys:
+        ds = dsfromkey(ksk)
+        rdata = " ".join(ds[4:])
+        update_msg.add(f"{zone}.", TTL, "CDS", rdata)
     server.nsupdate(update_msg)
 
     # Now there should be two CDS records.
     isctest.log.info(
         f"- zone {zone} {server.identifier}: check CDS RRset after update add"
     )
-    check_dnssec(server, zone, keys + [ksk], expected + extra)
+    check_dnssec(server, zone, keys + extra_keys, expected + extra)
 
     # Trigger keymgr.
     with server.watch_log_from_here() as watcher:
@@ -324,10 +329,10 @@ def check_add_cds(server, zone, keys, expected, ksk, extra):
 
     # Check again.
     isctest.log.info(f"- zone {zone} {server.identifier}: check again after keymgr run")
-    check_dnssec(server, zone, keys + [ksk], expected + extra)
+    check_dnssec(server, zone, keys + extra_keys, expected + extra)
 
 
-def check_remove_cds(server, zone, keys, expected, ksk, extra):
+def check_remove_cds(server, zone, keys, expected, extra_keys, extra):
     isctest.log.info("remove cds record:")
 
     isctest.log.info(
@@ -349,7 +354,7 @@ def check_remove_cds(server, zone, keys, expected, ksk, extra):
     isctest.log.info(
         f"- zone {zone} {server.identifier}: check CDS RRset after update remove"
     )
-    check_dnssec(server, zone, keys + [ksk], expected + extra)
+    check_dnssec(server, zone, keys + extra_keys, expected + extra)
 
     # Trigger keymgr.
     with server.watch_log_from_here() as watcher:
@@ -358,17 +363,18 @@ def check_remove_cds(server, zone, keys, expected, ksk, extra):
 
     # Check again.
     isctest.log.info(f"- zone {zone} {server.identifier}: check again after keymgr run")
-    check_dnssec(server, zone, keys + [ksk], expected + extra)
+    check_dnssec(server, zone, keys + extra_keys, expected + extra)
 
     # Remove actual CDS.
     isctest.log.info(
-        f"- zone {zone} {server.identifier}: remove CDS from other provider"
+        f"- zone {zone} {server.identifier}: remove CDS from other providers"
     )
 
-    ds = dsfromkey(ksk)
-    rdata = " ".join(ds[4:])
     update_msg = dns.update.UpdateMessage(zone)
-    update_msg.delete(f"{zone}.", "CDS", rdata)
+    for ksk in extra_keys:
+        ds = dsfromkey(ksk)
+        rdata = " ".join(ds[4:])
+        update_msg.delete(f"{zone}.", "CDS", rdata)
     server.nsupdate(update_msg)
 
     # Now there should be one CDS record again.
@@ -419,13 +425,13 @@ def test_multisigner(ns3, ns4):
     extra[0].private = False  # noqa
     extra[0].legacy = True  # noqa
 
-    check_add_zsk(ns3, zone, keys3, expected3, zsks4[0], extra)
-    check_add_zsk(ns4, zone, keys4, expected4, zsks3[0], extra)
+    check_add_zsk(ns3, zone, keys3, expected3, [zsks4[0]], extra)
+    check_add_zsk(ns4, zone, keys4, expected4, [zsks3[0]], extra)
     check_no_dnssec_in_journal(ns4, zone)
 
     # Remove DNSKEY from RRset.
-    check_remove_zsk(ns3, zone, keys3, expected3, zsks4[0], extra)
-    check_remove_zsk(ns4, zone, keys4, expected4, zsks3[0], extra)
+    check_remove_zsk(ns3, zone, keys3, expected3, [zsks4[0]], extra)
+    check_remove_zsk(ns4, zone, keys4, expected4, [zsks3[0]], extra)
     check_no_dnssec_in_journal(ns4, zone)
 
     # Add CDNSKEY RRset.
@@ -434,21 +440,21 @@ def test_multisigner(ns3, ns4):
     extra[0].private = False  # noqa
     extra[0].legacy = True  # noqa
 
-    check_add_cdnskey(ns3, zone, keys3, expected3, ksks4[0], extra)
-    check_add_cdnskey(ns4, zone, keys4, expected4, ksks3[0], extra)
+    check_add_cdnskey(ns3, zone, keys3, expected3, [ksks4[0]], extra)
+    check_add_cdnskey(ns4, zone, keys4, expected4, [ksks3[0]], extra)
     check_no_dnssec_in_journal(ns4, zone)
 
     # Remove CDNSKEY RRset.
-    check_remove_cdnskey(ns3, zone, keys3, expected3, ksks4[0], extra)
-    check_remove_cdnskey(ns4, zone, keys4, expected4, ksks3[0], extra)
+    check_remove_cdnskey(ns3, zone, keys3, expected3, [ksks4[0]], extra)
+    check_remove_cdnskey(ns4, zone, keys4, expected4, [ksks3[0]], extra)
     check_no_dnssec_in_journal(ns4, zone)
 
     # Update CDS RRset.
-    check_add_cds(ns3, zone, keys3, expected3, ksks4[0], extra)
-    check_add_cds(ns4, zone, keys4, expected4, ksks3[0], extra)
+    check_add_cds(ns3, zone, keys3, expected3, [ksks4[0]], extra)
+    check_add_cds(ns4, zone, keys4, expected4, [ksks3[0]], extra)
     check_no_dnssec_in_journal(ns4, zone)
 
     # Remove CDS RRset.
-    check_remove_cds(ns3, zone, keys3, expected3, ksks4[0], extra)
-    check_remove_cds(ns4, zone, keys4, expected4, ksks3[0], extra)
+    check_remove_cds(ns3, zone, keys3, expected3, [ksks4[0]], extra)
+    check_remove_cds(ns4, zone, keys4, expected4, [ksks3[0]], extra)
     check_no_dnssec_in_journal(ns4, zone)

From dc0ac0ca0abd5fa5de5de8f65e9e71d336d0c36c Mon Sep 17 00:00:00 2001
From: Matthijs Mekking <matthijs@isc.org>
Date: Fri, 10 Oct 2025 18:02:58 +0200
Subject: [PATCH 3/8] Update multisigner system test to set primary

When testing multi-signer as bump-in-the-wire (upcoming test), we want
to be able to do dynamically updates to a hidden primary. Update the
test functions such that we can set a specific primary server.

(cherry picked from commit fdf8a171c5058df7beb3500cad6bdcdbe731410c)
---
 .../system/multisigner/tests_multisigner.py   | 142 +++++++++++++-----
 1 file changed, 103 insertions(+), 39 deletions(-)

diff --git a/bin/tests/system/multisigner/tests_multisigner.py b/bin/tests/system/multisigner/tests_multisigner.py
index 86610e80de..2e9e5cd588 100644
--- a/bin/tests/system/multisigner/tests_multisigner.py
+++ b/bin/tests/system/multisigner/tests_multisigner.py
@@ -111,11 +111,14 @@ def check_no_dnssec_in_journal(server, zone):
     assert not match, f"{match.group(1)} record found in journal"
 
 
-def check_add_zsk(server, zone, keys, expected, extra_keys, extra):
+def check_add_zsk(server, zone, keys, expected, extra_keys, extra, primary=None):
+    if primary is None:
+        primary = server
+
     isctest.log.info("add dnskey record:")
 
     isctest.log.info(
-        f"- zone {zone} {server.identifier}: update zone with ZSK from other providers"
+        f"- zone {zone} {primary.identifier}: update zone with ZSK from other providers"
     )
 
     update_msg = dns.update.UpdateMessage(zone)
@@ -123,7 +126,7 @@ def check_add_zsk(server, zone, keys, expected, extra_keys, extra):
         dnskey = zsk.dnskey().split()
         rdata = " ".join(dnskey[4:])
         update_msg.add(f"{zone}.", TTL, "DNSKEY", rdata)
-    server.nsupdate(update_msg)
+    primary.nsupdate(update_msg)
 
     # Check the new DNSKEY RRset.
     isctest.log.info(
@@ -148,11 +151,14 @@ def check_add_zsk(server, zone, keys, expected, extra_keys, extra):
     server.log.prohibit(f"dns_zone_findkeys: error reading ./K{zone}")
 
 
-def check_remove_zsk(server, zone, keys, expected, extra_keys, extra):
-    isctest.log.info("remove dnskey record:")
+def _check_remove_zsk_fail(
+    server, zone, keys, expected, extra_keys, extra, primary=None
+):
+    if primary is None:
+        primary = server
 
     isctest.log.info(
-        f"- zone {zone} {server.identifier}: try to remove own ZSK (should fail)"
+        f"- zone {zone} {primary.identifier}: try to remove own ZSK (should fail)"
     )
 
     zsks = [k for k in keys if not k.is_ksk()]
@@ -160,15 +166,15 @@ def check_remove_zsk(server, zone, keys, expected, extra_keys, extra):
     rdata = " ".join(dnskey[4:])
     update_msg = dns.update.UpdateMessage(zone)
     update_msg.delete(f"{zone}.", "DNSKEY", rdata)
-    with server.watch_log_from_here() as watcher:
-        server.nsupdate(update_msg)
+    with primary.watch_log_from_here() as watcher:
+        primary.nsupdate(update_msg)
         watcher.wait_for_line(
             f"updating zone '{zone}/IN': attempt to delete in use DNSKEY ignored"
         )
 
     # Both ZSKs should still be published.
     isctest.log.info(
-        f"- zone {zone} {server.identifier}: check DNSKEY RRset after update remove"
+        f"- zone {zone} {server.identifier}: check DNSKEY RRset after ignored remove"
     )
     check_dnssec(server, zone, keys + extra_keys, expected + extra)
 
@@ -181,9 +187,23 @@ def check_remove_zsk(server, zone, keys, expected, extra_keys, extra):
     isctest.log.info(f"- zone {zone} {server.identifier}: check again after keymgr run")
     check_dnssec(server, zone, keys + extra_keys, expected + extra)
 
+
+def check_remove_zsk(
+    server, zone, keys, expected, extra_keys, extra, primary=None, check_fail=False
+):
+    isctest.log.info("remove dnskey record:")
+
+    if primary is None:
+        primary = server
+
+    if check_fail:
+        _check_remove_zsk_fail(
+            server, zone, keys, expected, extra_keys, extra, primary=primary
+        )
+
     # Remove actual ZSK.
     isctest.log.info(
-        f"- zone {zone} {server.identifier}: remove ZSK from other providers"
+        f"- zone {zone} {primary.identifier}: remove ZSK from other providers"
     )
 
     update_msg = dns.update.UpdateMessage(zone)
@@ -191,7 +211,7 @@ def check_remove_zsk(server, zone, keys, expected, extra_keys, extra):
         dnskey = zsk.dnskey().split()
         rdata = " ".join(dnskey[4:])
         update_msg.delete(f"{zone}.", "DNSKEY", rdata)
-    server.nsupdate(update_msg)
+    primary.nsupdate(update_msg)
 
     # We should have only the KSK and ZSK from server.
     isctest.log.info(
@@ -209,11 +229,14 @@ def check_remove_zsk(server, zone, keys, expected, extra_keys, extra):
     check_dnssec(server, zone, keys, expected)
 
 
-def check_add_cdnskey(server, zone, keys, expected, extra_keys, extra):
+def check_add_cdnskey(server, zone, keys, expected, extra_keys, extra, primary=None):
+    if primary is None:
+        primary = server
+
     isctest.log.info("add cdnskey record:")
 
     isctest.log.info(
-        f"- zone {zone} {server.identifier}: update zone with CDNSKEY from other providers"
+        f"- zone {zone} {primary.identifier}: update zone with CDNSKEY from other providers"
     )
 
     # Retrieve CDNSKEY records from the other providers.
@@ -222,7 +245,7 @@ def check_add_cdnskey(server, zone, keys, expected, extra_keys, extra):
         dnskey = ksk.dnskey().split()
         rdata = " ".join(dnskey[4:])
         update_msg.add(f"{zone}.", TTL, "CDNSKEY", rdata)
-    server.nsupdate(update_msg)
+    primary.nsupdate(update_msg)
 
     # Now there should be two CDNSKEY records.
     isctest.log.info(
@@ -240,11 +263,14 @@ def check_add_cdnskey(server, zone, keys, expected, extra_keys, extra):
     check_dnssec(server, zone, keys + extra_keys, expected + extra)
 
 
-def check_remove_cdnskey(server, zone, keys, expected, extra_keys, extra):
-    isctest.log.info("remove cdnskey record:")
+def _check_remove_cdnskey_fail(
+    server, zone, keys, expected, extra_keys, extra, primary=None
+):
+    if primary is None:
+        primary = server
 
     isctest.log.info(
-        f"- zone {zone} {server.identifier}: try to remove own CDNSKEY (should fail)"
+        f"- zone {zone} {primary.identifier}: try to remove own CDNSKEY (should fail)"
     )
 
     ksks = [k for k in keys if not k.is_ksk()]
@@ -252,15 +278,15 @@ def check_remove_cdnskey(server, zone, keys, expected, extra_keys, extra):
     rdata = " ".join(dnskey[4:])
     update_msg = dns.update.UpdateMessage(zone)
     update_msg.delete(f"{zone}.", "CDNSKEY", rdata)
-    with server.watch_log_from_here() as watcher:
-        server.nsupdate(update_msg)
+    with primary.watch_log_from_here() as watcher:
+        primary.nsupdate(update_msg)
         watcher.wait_for_line(
             f"updating zone '{zone}/IN': attempt to delete in use CDNSKEY ignored"
         )
 
     # Both CDNSKEY records should still be published.
     isctest.log.info(
-        f"- zone {zone} {server.identifier}: check CDNSKEY RRset after update remove"
+        f"- zone {zone} {server.identifier}: check CDNSKEY RRset after ignored remove"
     )
     check_dnssec(server, zone, keys + extra_keys, expected + extra)
 
@@ -273,9 +299,23 @@ def check_remove_cdnskey(server, zone, keys, expected, extra_keys, extra):
     isctest.log.info(f"- zone {zone} {server.identifier}: check again after keymgr run")
     check_dnssec(server, zone, keys + extra_keys, expected + extra)
 
+
+def check_remove_cdnskey(
+    server, zone, keys, expected, extra_keys, extra, primary=None, check_fail=False
+):
+    isctest.log.info("remove cdnskey record:")
+
+    if primary is None:
+        primary = server
+
+    if check_fail:
+        _check_remove_cdnskey_fail(
+            server, zone, keys, expected, extra_keys, extra, primary=primary
+        )
+
     # Remove actual CDNSKEY.
     isctest.log.info(
-        f"- zone {zone} {server.identifier}: remove CDNSKEY from other providers"
+        f"- zone {zone} {primary.identifier}: remove CDNSKEY from other providers"
     )
 
     update_msg = dns.update.UpdateMessage(zone)
@@ -283,7 +323,7 @@ def check_remove_cdnskey(server, zone, keys, expected, extra_keys, extra):
         dnskey = ksk.dnskey().split()
         rdata = " ".join(dnskey[4:])
         update_msg.delete(f"{zone}.", "CDNSKEY", rdata)
-    server.nsupdate(update_msg)
+    primary.nsupdate(update_msg)
 
     # Now there should be one CDNSKEY record again.
     isctest.log.info(
@@ -301,11 +341,14 @@ def check_remove_cdnskey(server, zone, keys, expected, extra_keys, extra):
     check_dnssec(server, zone, keys, expected)
 
 
-def check_add_cds(server, zone, keys, expected, extra_keys, extra):
+def check_add_cds(server, zone, keys, expected, extra_keys, extra, primary=None):
     isctest.log.info("add cds record:")
 
+    if primary is None:
+        primary = server
+
     isctest.log.info(
-        f"- zone {zone} {server.identifier}: update zone with CDS from other providers"
+        f"- zone {zone} {primary.identifier}: update zone with CDS from other providers"
     )
 
     # Retrieve CDS records from the other providers.
@@ -314,7 +357,7 @@ def check_add_cds(server, zone, keys, expected, extra_keys, extra):
         ds = dsfromkey(ksk)
         rdata = " ".join(ds[4:])
         update_msg.add(f"{zone}.", TTL, "CDS", rdata)
-    server.nsupdate(update_msg)
+    primary.nsupdate(update_msg)
 
     # Now there should be two CDS records.
     isctest.log.info(
@@ -332,11 +375,14 @@ def check_add_cds(server, zone, keys, expected, extra_keys, extra):
     check_dnssec(server, zone, keys + extra_keys, expected + extra)
 
 
-def check_remove_cds(server, zone, keys, expected, extra_keys, extra):
-    isctest.log.info("remove cds record:")
+def _check_remove_cds_fail(
+    server, zone, keys, expected, extra_keys, extra, primary=None
+):
+    if primary is None:
+        primary = server
 
     isctest.log.info(
-        f"- zone {zone} {server.identifier}: try to remove own CDS (should fail)"
+        f"- zone {zone} {primary.identifier}: try to remove own CDS (should fail)"
     )
 
     ksks = [k for k in keys if not k.is_ksk()]
@@ -344,15 +390,15 @@ def check_remove_cds(server, zone, keys, expected, extra_keys, extra):
     rdata = " ".join(ds[4:])
     update_msg = dns.update.UpdateMessage(zone)
     update_msg.delete(f"{zone}.", "CDS", rdata)
-    with server.watch_log_from_here() as watcher:
-        server.nsupdate(update_msg)
+    with primary.watch_log_from_here() as watcher:
+        primary.nsupdate(update_msg)
         watcher.wait_for_line(
             f"updating zone '{zone}/IN': attempt to delete in use CDS ignored"
         )
 
     # Both CDS records should still be published.
     isctest.log.info(
-        f"- zone {zone} {server.identifier}: check CDS RRset after update remove"
+        f"- zone {zone} {server.identifier}: check CDS RRset after ignored remove"
     )
     check_dnssec(server, zone, keys + extra_keys, expected + extra)
 
@@ -365,9 +411,23 @@ def check_remove_cds(server, zone, keys, expected, extra_keys, extra):
     isctest.log.info(f"- zone {zone} {server.identifier}: check again after keymgr run")
     check_dnssec(server, zone, keys + extra_keys, expected + extra)
 
+
+def check_remove_cds(
+    server, zone, keys, expected, extra_keys, extra, primary=None, check_fail=False
+):
+    isctest.log.info("remove cds record:")
+
+    if primary is None:
+        primary = server
+
+    if check_fail:
+        _check_remove_cds_fail(
+            server, zone, keys, expected, extra_keys, extra, primary=primary
+        )
+
     # Remove actual CDS.
     isctest.log.info(
-        f"- zone {zone} {server.identifier}: remove CDS from other providers"
+        f"- zone {zone} {primary.identifier}: remove CDS from other providers"
     )
 
     update_msg = dns.update.UpdateMessage(zone)
@@ -375,7 +435,7 @@ def check_remove_cds(server, zone, keys, expected, extra_keys, extra):
         ds = dsfromkey(ksk)
         rdata = " ".join(ds[4:])
         update_msg.delete(f"{zone}.", "CDS", rdata)
-    server.nsupdate(update_msg)
+    primary.nsupdate(update_msg)
 
     # Now there should be one CDS record again.
     isctest.log.info(
@@ -430,8 +490,8 @@ def test_multisigner(ns3, ns4):
     check_no_dnssec_in_journal(ns4, zone)
 
     # Remove DNSKEY from RRset.
-    check_remove_zsk(ns3, zone, keys3, expected3, [zsks4[0]], extra)
-    check_remove_zsk(ns4, zone, keys4, expected4, [zsks3[0]], extra)
+    check_remove_zsk(ns3, zone, keys3, expected3, [zsks4[0]], extra, check_fail=True)
+    check_remove_zsk(ns4, zone, keys4, expected4, [zsks3[0]], extra, check_fail=True)
     check_no_dnssec_in_journal(ns4, zone)
 
     # Add CDNSKEY RRset.
@@ -445,8 +505,12 @@ def test_multisigner(ns3, ns4):
     check_no_dnssec_in_journal(ns4, zone)
 
     # Remove CDNSKEY RRset.
-    check_remove_cdnskey(ns3, zone, keys3, expected3, [ksks4[0]], extra)
-    check_remove_cdnskey(ns4, zone, keys4, expected4, [ksks3[0]], extra)
+    check_remove_cdnskey(
+        ns3, zone, keys3, expected3, [ksks4[0]], extra, check_fail=True
+    )
+    check_remove_cdnskey(
+        ns4, zone, keys4, expected4, [ksks3[0]], extra, check_fail=True
+    )
     check_no_dnssec_in_journal(ns4, zone)
 
     # Update CDS RRset.
@@ -455,6 +519,6 @@ def test_multisigner(ns3, ns4):
     check_no_dnssec_in_journal(ns4, zone)
 
     # Remove CDS RRset.
-    check_remove_cds(ns3, zone, keys3, expected3, [ksks4[0]], extra)
-    check_remove_cds(ns4, zone, keys4, expected4, [ksks3[0]], extra)
+    check_remove_cds(ns3, zone, keys3, expected3, [ksks4[0]], extra, check_fail=True)
+    check_remove_cds(ns4, zone, keys4, expected4, [ksks3[0]], extra, check_fail=True)
     check_no_dnssec_in_journal(ns4, zone)

From 9379d4f1dfb60b2fea861d8824c7c570006c2752 Mon Sep 17 00:00:00 2001
From: Matthijs Mekking <matthijs@isc.org>
Date: Sun, 12 Oct 2025 11:49:00 +0200
Subject: [PATCH 4/8] Convert model2.secondary test to pytest

This test is similar to model2.multisigner, but now the two providers
are both secondary, both using the same hidden primary. The DNSKEY,
CDNSKEY, and CDS records need to be published at the hidden primary,
ns5, the zone is transferred to both secondaries, ns3 and ns4.

To avoid intermittent test failures, we wait for the line
"zone {zone}/IN (signed): serial {serial2} (unsigned {serial1})" in
the secondary server logs. This is a signal that the unsigned zone
with serial <serial1> has a signed version ready with serial <serial2>.

To speed up the test, disable 'notify-delay'.

(cherry picked from commit c96f8964825084a662ad7f3682810a446f2646c5)
---
 .../system/multisigner/ns5/named.conf.in      |   1 +
 bin/tests/system/multisigner/tests.sh         | 294 ------------------
 .../system/multisigner/tests_multisigner.py   | 126 ++++++++
 3 files changed, 127 insertions(+), 294 deletions(-)

diff --git a/bin/tests/system/multisigner/ns5/named.conf.in b/bin/tests/system/multisigner/ns5/named.conf.in
index 0a8e4f556c..eeae1309c5 100644
--- a/bin/tests/system/multisigner/ns5/named.conf.in
+++ b/bin/tests/system/multisigner/ns5/named.conf.in
@@ -27,6 +27,7 @@ options {
 	recursion no;
 	key-directory ".";
 	dnssec-validation no;
+	notify-delay 0;
 };
 
 key rndc_key {
diff --git a/bin/tests/system/multisigner/tests.sh b/bin/tests/system/multisigner/tests.sh
index c0141e8d85..ee06afa7d4 100644
--- a/bin/tests/system/multisigner/tests.sh
+++ b/bin/tests/system/multisigner/tests.sh
@@ -58,299 +58,5 @@ records_published() {
   test "$lines" -eq "$_expect" || return 1
 }
 
-#
-# Check secondary server behaviour.
-#
-set_zone "model2.secondary"
-set_policy "model2" "2" "3600"
-
-set_server "ns3" "10.53.0.3"
-check_keys
-check_dnssecstatus "$SERVER" "$POLICY" "$ZONE"
-set_keytimes_model2
-check_keytimes
-check_apex
-dnssec_verify
-
-set_server "ns4" "10.53.0.4"
-check_keys
-check_dnssecstatus "$SERVER" "$POLICY" "$ZONE"
-set_keytimes_model2
-check_keytimes
-check_apex
-dnssec_verify
-
-#
-# Update DNSKEY RRset.
-#
-n=$((n + 1))
-echo_i "add dnskey record: update zone ${ZONE} at ns5 with ZSKs from providers ns3 and ns4 ($n)"
-ret=0
-set_server "ns5" "10.53.0.5"
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update add $(cat "ns3/${ZONE}.zsk")
-  echo update add $(cat "ns4/${ZONE}.zsk")
-  echo send
-) | $NSUPDATE || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# NS3
-n=$((n + 1))
-set_server "ns3" "10.53.0.3"
-echo_i "check server ${DIR} zone ${ZONE} DNSKEY RRset after update ($n)"
-ret=0
-retry_quiet 10 zsks_are_published || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-dnssec_verify
-no_dnssec_in_journal
-grep "dns_zone_findkeys: error reading ./K${ZONE}.*\.private: file not found" "${DIR}/named.run" && ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# NS4
-n=$((n + 1))
-set_server "ns4" "10.53.0.4"
-echo_i "check server ${DIR} zone ${ZONE} DNSKEY RRset after update ($n)"
-ret=0
-retry_quiet 10 zsks_are_published || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-dnssec_verify
-no_dnssec_in_journal
-grep "dns_zone_findkeys: error reading ./K${ZONE}.*\.private: file not found" "${DIR}/named.run" && ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-
-n=$((n + 1))
-echo_i "remove dnskey record: remove ns3 and ns4 DNSKEY records from primary ns5 ($n)"
-ret=0
-set_server "ns5" "10.53.0.5"
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update del $(cat "ns3/${ZONE}.zsk")
-  echo update del $(cat "ns4/${ZONE}.zsk")
-  echo send
-) | $NSUPDATE || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Now there should be one DNSKEY record again.
-# While we did remove both DNSKEY records, the bump in the wire signer, i.e
-# the secondary inline-signing zone, should add back the DNSKEY belonging to
-# its own KSK when re-signing the zone.
-#
-# NS3
-n=$((n + 1))
-set_server "ns3" "10.53.0.3"
-echo_i "check server ${DIR} zone ${ZONE} DNSKEY RRset after update ($n)"
-ret=0
-check_keys
-check_apex
-dnssec_verify
-no_dnssec_in_journal
-# NS4
-n=$((n + 1))
-set_server "ns4" "10.53.0.4"
-echo_i "check server ${DIR} zone ${ZONE} DNSKEY RRset after update ($n)"
-ret=0
-check_keys
-check_apex
-dnssec_verify
-no_dnssec_in_journal
-
-#
-# Update CDNSKEY RRset.
-#
-
-# Retrieve CDNSKEY records from the providers.
-n=$((n + 1))
-echo_i "check initial CDSNKEY response for zone ${ZONE} at ns3 and ns4 ($n)"
-ret=0
-dig_with_opts ${ZONE} @10.53.0.3 CDNSKEY >dig.out.ns3.secondary.cdnskey
-awk '$4 == "CDNSKEY" {print}' dig.out.ns3.secondary.cdnskey >secondary.cdnskey.ns3
-dig_with_opts ${ZONE} @10.53.0.4 CDNSKEY >dig.out.ns4.secondary.cdnskey
-awk '$4 == "CDNSKEY" {print}' dig.out.ns4.secondary.cdnskey >secondary.cdnskey.ns4
-# Initially there should be one CDNSKEY.
-set_server "ns3" "10.53.0.3"
-retry_quiet 10 records_published CDNSKEY 1 || ret=1
-set_server "ns4" "10.53.0.4"
-retry_quiet 10 records_published CDNSKEY 1 || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-
-n=$((n + 1))
-echo_i "add cdnskey record: update zone ${ZONE} at ns5 with CDNSKEY records from providers ns3 and ns4 ($n)"
-ret=0
-set_server "ns5" "10.53.0.5"
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update add $(cat "secondary.cdnskey.ns3")
-  echo update add $(cat "secondary.cdnskey.ns4")
-  echo send
-) | $NSUPDATE || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Now there should be two CDNSKEY records (we test that BIND does not
-# skip it during DNSSEC maintenance).
-#
-# NS3
-n=$((n + 1))
-set_server "ns3" "10.53.0.3"
-echo_i "check server ${DIR} zone ${ZONE} CDNSKEY RRset after update ($n)"
-ret=0
-retry_quiet 10 records_published CDNSKEY 2 || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-dnssec_verify
-no_dnssec_in_journal
-# NS4
-n=$((n + 1))
-set_server "ns4" "10.53.0.4"
-echo_i "check server ${DIR} zone ${ZONE} CDNSKEY RRset after update ($n)"
-ret=0
-retry_quiet 10 records_published CDNSKEY 2 || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-dnssec_verify
-no_dnssec_in_journal
-
-n=$((n + 1))
-echo_i "remove cdnskey record: remove ns3 and ns4 CDNSKEY records from primary ns5 ($n)"
-ret=0
-set_server "ns5" "10.53.0.5"
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update del $(cat "secondary.cdnskey.ns3")
-  echo update del $(cat "secondary.cdnskey.ns4")
-  echo send
-) | $NSUPDATE || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Now there should be one CDNSKEY record again.
-# While we did remove both CDNSKEY records, the bump in the wire signer, i.e
-# the secondary inline-signing zone, should add back the CDNSKEY belonging to
-# its own KSK when re-signing the zone.
-#
-# NS3
-n=$((n + 1))
-set_server "ns3" "10.53.0.3"
-echo_i "check server ${DIR} zone ${ZONE} CDNSKEY RRset after update ($n)"
-ret=0
-retry_quiet 10 records_published CDNSKEY 1 || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-dnssec_verify
-no_dnssec_in_journal
-# NS4
-n=$((n + 1))
-set_server "ns4" "10.53.0.4"
-echo_i "check server ${DIR} zone ${ZONE} CDNSKEY RRset after update ($n)"
-ret=0
-retry_quiet 10 records_published CDNSKEY 1 || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-dnssec_verify
-no_dnssec_in_journal
-
-#
-# Update CDS RRset.
-#
-
-# Retrieve CDS records from the other provider.
-n=$((n + 1))
-echo_i "check initial CDS response for zone ${ZONE} at ns3 and ns4 ($n)"
-ret=0
-dig_with_opts ${ZONE} @10.53.0.3 CDS >dig.out.ns3.secondary.cds
-awk '$4 == "CDS" {print}' dig.out.ns3.secondary.cds >secondary.cds.ns3
-dig_with_opts ${ZONE} @10.53.0.4 CDS >dig.out.ns4.secondary.cds
-awk '$4 == "CDS" {print}' dig.out.ns4.secondary.cds >secondary.cds.ns4
-# Initially there should be one CDS.
-set_server "ns3" "10.53.0.3"
-retry_quiet 10 records_published CDS 1 || ret=1
-set_server "ns4" "10.53.0.4"
-retry_quiet 10 records_published CDS 1 || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-
-n=$((n + 1))
-echo_i "add cds record: update zone ${ZONE} at ns5 with CDS from provider ns4 ($n)"
-ret=0
-set_server "ns5" "10.53.0.5"
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update add $(cat "secondary.cds.ns3")
-  echo update add $(cat "secondary.cds.ns4")
-  echo send
-) | $NSUPDATE || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Now there should be two CDS records (we test that BIND does not
-# skip it during DNSSEC maintenance).
-#
-# NS3
-n=$((n + 1))
-set_server "ns3" "10.53.0.3"
-echo_i "check server ${DIR} zone ${ZONE} CDS RRset after update ($n)"
-ret=0
-retry_quiet 10 records_published CDS 2 || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-dnssec_verify
-no_dnssec_in_journal
-# NS4
-n=$((n + 1))
-set_server "ns4" "10.53.0.4"
-echo_i "check server ${DIR} zone ${ZONE} CDS RRset after update ($n)"
-ret=0
-retry_quiet 10 records_published CDS 2 || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-dnssec_verify
-no_dnssec_in_journal
-
-n=$((n + 1))
-echo_i "remove cds record: remove ns3 and ns4 CDS records from primary ns5 ($n)"
-ret=0
-set_server "ns5" "10.53.0.5"
-(
-  echo zone "${ZONE}"
-  echo server "${SERVER}" "${PORT}"
-  echo update del $(cat "secondary.cds.ns3")
-  echo update del $(cat "secondary.cds.ns4")
-  echo send
-) | $NSUPDATE || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-# Now there should be one CDS record again.
-# While we did remove both CDS records, the bump in the wire signer, i.e
-# the secondary inline-signing zone, should add back the CDS belonging to
-# its own KSK when re-signing the zone.
-#
-# NS3
-n=$((n + 1))
-set_server "ns3" "10.53.0.3"
-echo_i "check server ${DIR} zone ${ZONE} CDS RRset after update ($n)"
-ret=0
-retry_quiet 10 records_published CDS 1 || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-dnssec_verify
-no_dnssec_in_journal
-# NS4
-n=$((n + 1))
-set_server "ns4" "10.53.0.4"
-echo_i "check server ${DIR} zone ${ZONE} CDS RRset after update ($n)"
-ret=0
-retry_quiet 10 records_published CDS 1 || ret=1
-test "$ret" -eq 0 || echo_i "failed"
-status=$((status + ret))
-dnssec_verify
-no_dnssec_in_journal
-
 echo_i "exit status: $status"
 [ $status -eq 0 ] || exit 1
diff --git a/bin/tests/system/multisigner/tests_multisigner.py b/bin/tests/system/multisigner/tests_multisigner.py
index 2e9e5cd588..80288aee6d 100644
--- a/bin/tests/system/multisigner/tests_multisigner.py
+++ b/bin/tests/system/multisigner/tests_multisigner.py
@@ -111,6 +111,47 @@ def check_no_dnssec_in_journal(server, zone):
     assert not match, f"{match.group(1)} record found in journal"
 
 
+def wait_for_serial(primary, server, zone):
+    if primary.identifier == server.identifier:
+        # No need to check if the transfer has been done.
+        return
+
+    def check_serial():
+        response = isctest.query.tcp(
+            query, primary.ip, primary.ports.dns, timeout=3, attempts=1
+        )
+        assert response.rcode() == dns.rcode.NOERROR
+        soa = response.get_rrset(
+            response.answer,
+            dns.name.from_text(fqdn),
+            dns.rdataclass.IN,
+            dns.rdatatype.SOA,
+        )
+        serial1 = soa[0].serial
+
+        response = isctest.query.tcp(
+            query, server.ip, server.ports.dns, timeout=3, attempts=1
+        )
+        assert response.rcode() == dns.rcode.NOERROR
+        soa = response.get_rrset(
+            response.answer,
+            dns.name.from_text(fqdn),
+            dns.rdataclass.IN,
+            dns.rdatatype.SOA,
+        )
+        serial2 = soa[0].serial
+
+        return (
+            f"zone {zone}/IN (signed): serial {serial2} (unsigned {serial1})"
+            in server.log
+        )
+
+    fqdn = f"{zone}."
+    query = isctest.query.create(fqdn, dns.rdatatype.SOA)
+
+    isctest.run.retry_with_timeout(check_serial, timeout=10)
+
+
 def check_add_zsk(server, zone, keys, expected, extra_keys, extra, primary=None):
     if primary is None:
         primary = server
@@ -128,6 +169,8 @@ def check_add_zsk(server, zone, keys, expected, extra_keys, extra, primary=None)
         update_msg.add(f"{zone}.", TTL, "DNSKEY", rdata)
     primary.nsupdate(update_msg)
 
+    wait_for_serial(primary, server, zone)
+
     # Check the new DNSKEY RRset.
     isctest.log.info(
         f"- zone {zone} {server.identifier}: check DNSKEY RRset after update add"
@@ -213,6 +256,8 @@ def check_remove_zsk(
         update_msg.delete(f"{zone}.", "DNSKEY", rdata)
     primary.nsupdate(update_msg)
 
+    wait_for_serial(primary, server, zone)
+
     # We should have only the KSK and ZSK from server.
     isctest.log.info(
         f"- zone {zone} {server.identifier}: check DNSKEY RRset after update remove"
@@ -247,6 +292,8 @@ def check_add_cdnskey(server, zone, keys, expected, extra_keys, extra, primary=N
         update_msg.add(f"{zone}.", TTL, "CDNSKEY", rdata)
     primary.nsupdate(update_msg)
 
+    wait_for_serial(primary, server, zone)
+
     # Now there should be two CDNSKEY records.
     isctest.log.info(
         f"- zone {zone} {server.identifier}: check CDNSKEY RRset after update add"
@@ -325,6 +372,8 @@ def check_remove_cdnskey(
         update_msg.delete(f"{zone}.", "CDNSKEY", rdata)
     primary.nsupdate(update_msg)
 
+    wait_for_serial(primary, server, zone)
+
     # Now there should be one CDNSKEY record again.
     isctest.log.info(
         f"- zone {zone} {server.identifier}: check CDNSKEY RRset after update remove"
@@ -359,6 +408,8 @@ def check_add_cds(server, zone, keys, expected, extra_keys, extra, primary=None)
         update_msg.add(f"{zone}.", TTL, "CDS", rdata)
     primary.nsupdate(update_msg)
 
+    wait_for_serial(primary, server, zone)
+
     # Now there should be two CDS records.
     isctest.log.info(
         f"- zone {zone} {server.identifier}: check CDS RRset after update add"
@@ -437,6 +488,8 @@ def check_remove_cds(
         update_msg.delete(f"{zone}.", "CDS", rdata)
     primary.nsupdate(update_msg)
 
+    wait_for_serial(primary, server, zone)
+
     # Now there should be one CDS record again.
     isctest.log.info(
         f"- zone {zone} {server.identifier}: check CDS RRset after update remove"
@@ -522,3 +575,76 @@ def test_multisigner(ns3, ns4):
     check_remove_cds(ns3, zone, keys3, expected3, [ksks4[0]], extra, check_fail=True)
     check_remove_cds(ns4, zone, keys4, expected4, [ksks3[0]], extra, check_fail=True)
     check_no_dnssec_in_journal(ns4, zone)
+
+
+def test_multisigner_secondary(ns3, ns4, ns5):
+    zone = "model2.secondary"
+    keyprops = [
+        f"ksk 0 {ALGORITHM} {SIZE} goal:omnipresent dnskey:omnipresent krrsig:omnipresent ds:omnipresent",
+        f"zsk 0 {ALGORITHM} {SIZE} goal:omnipresent dnskey:omnipresent zrrsig:omnipresent",
+    ]
+
+    # First make sure the zone is properly signed.
+    isctest.log.info(f"basic DNSSEC tests for {zone}")
+    isctest.kasp.wait_keymgr_done(ns3, zone)
+    isctest.kasp.wait_keymgr_done(ns4, zone)
+
+    keys3 = isctest.kasp.keydir_to_keylist(zone, ns3.identifier)
+    ksks3 = [k for k in keys3 if k.is_ksk()]
+    zsks3 = [k for k in keys3 if not k.is_ksk()]
+    expected3 = isctest.kasp.policy_to_properties(ttl=TTL, keys=keyprops)
+
+    check_dnssec(ns3, zone, keys3, expected3)
+
+    keys4 = isctest.kasp.keydir_to_keylist(zone, ns4.identifier)
+    ksks4 = [k for k in keys4 if k.is_ksk()]
+    zsks4 = [k for k in keys4 if not k.is_ksk()]
+    expected4 = isctest.kasp.policy_to_properties(ttl=TTL, keys=keyprops)
+
+    check_dnssec(ns4, zone, keys4, expected4)
+
+    # Add DNSKEY to RRset.
+    newprops = [f"zsk unlimited {ALGORITHM} {SIZE}"]
+    extra = isctest.kasp.policy_to_properties(ttl=TTL, keys=newprops)
+    extra[0].private = False  # noqa
+    extra[0].legacy = True  # noqa
+
+    check_add_zsk(ns3, zone, keys3, expected3, [zsks4[0]], extra, primary=ns5)
+    check_add_zsk(ns4, zone, keys4, expected4, [zsks3[0]], extra, primary=ns5)
+    check_no_dnssec_in_journal(ns3, zone)
+    check_no_dnssec_in_journal(ns4, zone)
+
+    # Remove DNSKEY from RRset.
+    check_remove_zsk(ns3, zone, keys3, expected3, [zsks4[0]], extra, primary=ns5)
+    check_remove_zsk(ns4, zone, keys4, expected4, [zsks3[0]], extra, primary=ns5)
+    check_no_dnssec_in_journal(ns3, zone)
+    check_no_dnssec_in_journal(ns4, zone)
+
+    # Add CDNSKEY RRset.
+    newprops = [f"ksk unlimited {ALGORITHM} {SIZE}"]
+    extra = isctest.kasp.policy_to_properties(ttl=TTL, keys=newprops)
+    extra[0].private = False  # noqa
+    extra[0].legacy = True  # noqa
+
+    check_add_cdnskey(ns3, zone, keys3, expected3, [ksks4[0]], extra, primary=ns5)
+    check_add_cdnskey(ns4, zone, keys4, expected4, [ksks3[0]], extra, primary=ns5)
+    check_no_dnssec_in_journal(ns3, zone)
+    check_no_dnssec_in_journal(ns4, zone)
+
+    # Remove CDNSKEY RRset.
+    check_remove_cdnskey(ns3, zone, keys3, expected3, [ksks4[0]], extra, primary=ns5)
+    check_remove_cdnskey(ns4, zone, keys4, expected4, [ksks3[0]], extra, primary=ns5)
+    check_no_dnssec_in_journal(ns3, zone)
+    check_no_dnssec_in_journal(ns4, zone)
+
+    # Update CDS RRset.
+    check_add_cds(ns3, zone, keys3, expected3, [ksks4[0]], extra, primary=ns5)
+    check_add_cds(ns4, zone, keys4, expected4, [ksks3[0]], extra, primary=ns5)
+    check_no_dnssec_in_journal(ns3, zone)
+    check_no_dnssec_in_journal(ns4, zone)
+
+    # Remove CDS RRset.
+    check_remove_cds(ns3, zone, keys3, expected3, [ksks4[0]], extra, primary=ns5)
+    check_remove_cds(ns4, zone, keys4, expected4, [ksks3[0]], extra, primary=ns5)
+    check_no_dnssec_in_journal(ns3, zone)
+    check_no_dnssec_in_journal(ns4, zone)

From 58f7bbdf6c3fd07b498ba5b64550cac33e0a6ded Mon Sep 17 00:00:00 2001
From: Matthijs Mekking <matthijs@isc.org>
Date: Sun, 12 Oct 2025 11:54:54 +0200
Subject: [PATCH 5/8] Clean up shell script remnants

All the cases in this system test have been converted to pytest, so
we can clean up the shell script remnants.

(cherry picked from commit 97b38a1fbc86a78ea672b907f0e369203a9a71fc)
---
 bin/tests/system/multisigner/tests.sh         | 62 -------------------
 .../multisigner/tests_sh_multisigner.py       | 41 ------------
 2 files changed, 103 deletions(-)
 delete mode 100644 bin/tests/system/multisigner/tests.sh
 delete mode 100644 bin/tests/system/multisigner/tests_sh_multisigner.py

diff --git a/bin/tests/system/multisigner/tests.sh b/bin/tests/system/multisigner/tests.sh
deleted file mode 100644
index ee06afa7d4..0000000000
--- a/bin/tests/system/multisigner/tests.sh
+++ /dev/null
@@ -1,62 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-set -e
-
-# shellcheck source=conf.sh
-. ../conf.sh
-# shellcheck source=kasp.sh
-. ../kasp.sh
-
-dig_with_opts() {
-  $DIG +tcp +noadd +nosea +nostat +nocmd +dnssec -p $PORT "$@"
-}
-
-start_time="$(TZ=UTC date +%s)"
-status=0
-n=0
-
-# Test to make sure no DNSSEC records end up in the raw journal.
-no_dnssec_in_journal() {
-  n=$((n + 1))
-  ret=0
-  echo_i "check zone ${ZONE} raw journal has no DNSSEC ($n)"
-  $JOURNALPRINT "${DIR}/${ZONE}.db.jnl" >"${DIR}/${ZONE}.journal.out.test$n"
-  rrset_exists NSEC "${DIR}/${ZONE}.journal.out.test$n" && ret=1
-  rrset_exists NSEC3 "${DIR}/${ZONE}.journal.out.test$n" && ret=1
-  rrset_exists NSEC3PARAM "${DIR}/${ZONE}.journal.out.test$n" && ret=1
-  rrset_exists RRSIG "${DIR}/${ZONE}.journal.out.test$n" && ret= 1
-  test "$ret" -eq 0 || echo_i "failed"
-  status=$((status + ret))
-}
-
-# Check if a certain RRtype is present in the journal file.
-rrset_exists() (
-  rrtype=$1
-  file=$2
-  lines=$(awk -v rt="${rrtype}" '$5 == rt {print}' ${file} | wc -l)
-  test "$lines" -gt 0
-)
-
-# Check that the CDNSKEY from both providers are published.
-records_published() {
-  _rrtype=$1
-  _expect=$2
-
-  dig_with_opts "$ZONE" "@${SERVER}" "${_rrtype}" >"dig.out.$DIR.test$n" || return 1
-  lines=$(awk -v rt="${_rrtype}" '$4 == rt {print}' dig.out.$DIR.test$n | wc -l)
-  test "$lines" -eq "$_expect" || return 1
-}
-
-echo_i "exit status: $status"
-[ $status -eq 0 ] || exit 1
diff --git a/bin/tests/system/multisigner/tests_sh_multisigner.py b/bin/tests/system/multisigner/tests_sh_multisigner.py
deleted file mode 100644
index 6269823872..0000000000
--- a/bin/tests/system/multisigner/tests_sh_multisigner.py
+++ /dev/null
@@ -1,41 +0,0 @@
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-import pytest
-
-pytestmark = pytest.mark.extra_artifacts(
-    [
-        "*.created",
-        "cdnskey.ns*",
-        "cds.ns*",
-        "created.*",
-        "dig.out.*",
-        "rndc.dnssec.status.out.*",
-        "secondary.cdnskey.ns*",
-        "secondary.cds.ns*",
-        "unused.*",
-        "verify.out.*",
-        "ns*/K*",
-        "ns*/db-*",
-        "ns*/keygen.out.*",
-        "ns*/*.jbk",
-        "ns*/*.jnl",
-        "ns*/*.zsk",
-        "ns*/*.signed",
-        "ns*/*.journal.out.*",
-        "ns*/settime.out.*",
-        "ns*/model2.secondary.db",
-    ]
-)
-
-
-def test_multisigner(run_tests_sh):
-    run_tests_sh()

From a8a080f6f6b31637e47ce16d563fe35c4d858bee Mon Sep 17 00:00:00 2001
From: Matthijs Mekking <matthijs@isc.org>
Date: Sun, 12 Oct 2025 12:23:12 +0200
Subject: [PATCH 6/8] Remove common kasp shell script

No system test is using the common kasp shell script test anymore, so we
can remove it.

(cherry picked from commit 61d1209c311bc24790cb96f92b2f7794b24fbdda)
---
 bin/tests/system/kasp.sh | 1299 --------------------------------------
 1 file changed, 1299 deletions(-)
 delete mode 100644 bin/tests/system/kasp.sh

diff --git a/bin/tests/system/kasp.sh b/bin/tests/system/kasp.sh
deleted file mode 100644
index be0f6082b7..0000000000
--- a/bin/tests/system/kasp.sh
+++ /dev/null
@@ -1,1299 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
-#
-# SPDX-License-Identifier: MPL-2.0
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, you can obtain one at https://mozilla.org/MPL/2.0/.
-#
-# See the COPYRIGHT file distributed with this work for additional
-# information regarding copyright ownership.
-
-#
-# Common configuration data for kasp system tests, to be sourced into
-# other shell scripts.
-#
-
-# shellcheck source=conf.sh
-. ../conf.sh
-
-###############################################################################
-# Constants                                                                   #
-###############################################################################
-DEFAULT_TTL=300
-
-###############################################################################
-# Query properties                                                            #
-###############################################################################
-TSIG=""
-SHA1="FrSt77yPTFx6hTs4i2tKLB9LmE0="
-SHA224="hXfwwwiag2QGqblopofai9NuW28q/1rH4CaTnA=="
-SHA256="R16NojROxtxH/xbDl//ehDsHm5DjWTQ2YXV+hGC2iBY="
-VIEW1="YPfMoAk6h+3iN8MDRQC004iSNHY="
-VIEW2="4xILSZQnuO1UKubXHkYUsvBRPu8="
-VIEW3="C1Azf+gGPMmxrUg/WQINP6eV9Y0="
-MINDEPTH=1
-MAXDEPTH=3
-
-###############################################################################
-# Key properties                                                              #
-###############################################################################
-# ID
-# BASEFILE
-# EXPECT
-# ROLE
-# KSK
-# ZSK
-# FLAGS
-# LIFETIME
-# ALG_NUM
-# ALG_STR
-# ALG_LEN
-# CREATED
-# PUBLISHED
-# ACTIVE
-# RETIRED
-# REVOKED
-# REMOVED
-# GOAL
-# STATE_DNSKEY
-# STATE_ZRRSIG
-# STATE_KRRSIG
-# STATE_DS
-# EXPECT_ZRRSIG
-# EXPECT_KRRSIG
-# LEGACY
-# PRIVATE
-# PRIVKEY_STAT
-# PUBKEY_STAT
-# STATE_STAT
-# FLAGS
-# KEYDIR
-
-key_key() {
-  echo "${1}__${2}"
-}
-
-key_get() {
-  eval "echo \${$(key_key "$1" "$2")}"
-}
-
-key_set() {
-  eval "$(key_key "$1" "$2")='$3'"
-}
-
-key_stat() {
-  $PERL -e 'print((stat @ARGV[0])[9] . "\n");' "$1"
-}
-
-# Save certain values in the KEY array.
-key_save() {
-  # Save key id.
-  key_set "$1" ID "$KEY_ID"
-  key_set "$1" RID "$KEY_RID"
-  # Save base filename.
-  key_set "$1" BASEFILE "$BASE_FILE"
-  # Save creation date.
-  key_set "$1" CREATED "${KEY_CREATED}"
-  # Save key change time.
-  key_set "$1" PRIVKEY_STAT $(key_stat "${BASE_FILE}.private")
-  key_set "$1" PUBKEY_STAT $(key_stat "${BASE_FILE}.key")
-  key_set "$1" STATE_STAT $(key_stat "${BASE_FILE}.state")
-}
-
-# Clear key state.
-#
-# This will update either the KEY1, KEY2, or KEY3 array.
-key_clear() {
-  key_set "$1" "ID" 'no'
-  key_set "$1" "RID" 'no'
-  key_set "$1" "IDPAD" 'no'
-  key_set "$1" "EXPECT" 'no'
-  key_set "$1" "ROLE" 'none'
-  key_set "$1" "KSK" 'no'
-  key_set "$1" "ZSK" 'no'
-  key_set "$1" "FLAGS" '0'
-  key_set "$1" "LIFETIME" 'none'
-  key_set "$1" "ALG_NUM" '0'
-  key_set "$1" "ALG_STR" 'none'
-  key_set "$1" "ALG_LEN" '0'
-  key_set "$1" "CREATED" '0'
-  key_set "$1" "PUBLISHED" 'none'
-  key_set "$1" "SYNCPUBLISH" 'none'
-  key_set "$1" "ACTIVE" 'none'
-  key_set "$1" "RETIRED" 'none'
-  key_set "$1" "REVOKED" 'none'
-  key_set "$1" "REMOVED" 'none'
-  key_set "$1" "GOAL" 'none'
-  key_set "$1" "STATE_DNSKEY" 'none'
-  key_set "$1" "STATE_KRRSIG" 'none'
-  key_set "$1" "STATE_ZRRSIG" 'none'
-  key_set "$1" "STATE_DS" 'none'
-  key_set "$1" "EXPECT_ZRRSIG" 'no'
-  key_set "$1" "EXPECT_KRRSIG" 'no'
-  key_set "$1" "LEGACY" 'no'
-  key_set "$1" "PRIVATE" 'yes'
-  key_set "$1" "PRIVKEY_STAT" '0'
-  key_set "$1" "PUBKEY_STAT" '0'
-  key_set "$1" "STATE_STAT" '0'
-  key_set "$1" "KEYDIR" 'none'
-}
-
-# Start clear.
-# There can be at most 4 keys at the same time during a rollover:
-# 2x KSK, 2x ZSK
-key_clear "KEY1"
-key_clear "KEY2"
-key_clear "KEY3"
-key_clear "KEY4"
-
-###############################################################################
-# Utilities                                                                   #
-###############################################################################
-
-# Call dig with default options.
-_dig_with_opts() {
-
-  if [ -n "$TSIG" ]; then
-    "$DIG" +tcp +noadd +nosea +nostat +nocmd +dnssec -p "$PORT" -y "$TSIG" "$@"
-  else
-    "$DIG" +tcp +noadd +nosea +nostat +nocmd +dnssec -p "$PORT" "$@"
-  fi
-}
-
-# RNDC.
-_rndccmd() {
-  "$RNDC" -c ../_common/rndc.conf -p "$CONTROLPORT" -s "$@"
-}
-
-# Print IDs of keys used for generating RRSIG records for RRsets of type $1,
-# matching algorithm number $2, found in dig output file $3.
-# If $2 is equal to 0, any algorithm matches.
-get_keys_which_signed() {
-  _qtype=$1
-  _alg=$2
-  _output=$3
-  # The key ID is the 11th column of the RRSIG record line.
-  if [ "$_alg" = "0" ]; then
-    awk -v qt="$_qtype" '$4 == "RRSIG" && $5 == qt {print $11}' <"$_output"
-  else
-    awk -v alg="$_alg" -v qt="$_qtype" '$4 == "RRSIG" && $5 == qt && $6 == alg {print $11}' <"$_output"
-  fi
-}
-
-# Get the key ids from key files for zone $2 in directory $1.
-get_keyids() {
-  _dir=$1
-  _zone=$2
-  _regex="K${_zone}.+*+*.key"
-
-  find "${_dir}" -mindepth $MINDEPTH -maxdepth $MAXDEPTH -name "${_regex}" | sed "s,.*/K${_zone}.+\([0-9]\{3\}\)+\([0-9]\{5\}\).key,\2,"
-}
-
-# By default log errors and don't quit immediately.
-_log=1
-_log_error() {
-  test $_log -eq 1 && echo_i "error: $1"
-  ret=$((ret + 1))
-}
-disable_logerror() {
-  _log=0
-}
-enable_logerror() {
-  _log=1
-}
-
-# Set server key-directory ($1) and address ($2) for testing keys.
-set_server() {
-  DIR=$1
-  SERVER=$2
-}
-# Set zone name for testing keys.
-set_zone() {
-  ZONE=$1
-  DYNAMIC="no"
-}
-# By default zones are considered static.
-# When testing dynamic zones, call 'set_dynamic' after 'set_zone'.
-set_dynamic() {
-  DYNAMIC="yes"
-}
-
-# Set policy settings (name $1, number of keys $2, dnskey ttl $3).
-set_policy() {
-  POLICY=$1
-  NUM_KEYS=$2
-  DNSKEY_TTL=$3
-  KEYFILE_TTL=$3
-  CDS_DELETE="no"
-  CDS_SHA256="yes"
-  CDS_SHA384="no"
-  CDNSKEY="yes"
-}
-# By default policies are considered to be secure.
-# If a zone sets its policy to "insecure", call 'set_cdsdelete' to tell the
-# system test to expect a CDS and CDNSKEY Delete record.
-set_cdsdelete() {
-  CDS_DELETE="yes"
-}
-
-# Set key properties for testing keys.
-# $1: Key to update (KEY1, KEY2, ...)
-# $2: Value
-set_keyrole() {
-  key_set "$1" "EXPECT" "yes"
-  key_set "$1" "ROLE" "$2"
-  key_set "$1" "KSK" "no"
-  key_set "$1" "ZSK" "no"
-  key_set "$1" "FLAGS" "0"
-
-  test "$2" = "ksk" && key_set "$1" "KSK" "yes"
-  test "$2" = "ksk" && key_set "$1" "FLAGS" "257"
-
-  test "$2" = "zsk" && key_set "$1" "ZSK" "yes"
-  test "$2" = "zsk" && key_set "$1" "FLAGS" "256"
-
-  test "$2" = "csk" && key_set "$1" "KSK" "yes"
-  test "$2" = "csk" && key_set "$1" "ZSK" "yes"
-  test "$2" = "csk" && key_set "$1" "FLAGS" "257"
-
-  return 0
-}
-set_keylifetime() {
-  key_set "$1" "EXPECT" "yes"
-  key_set "$1" "LIFETIME" "$2"
-}
-# The algorithm value consists of three parts:
-# $2: Algorithm (number)
-# $3: Algorithm (string-format)
-# $4: Algorithm length
-set_keyalgorithm() {
-  key_set "$1" "EXPECT" "yes"
-  key_set "$1" "ALG_NUM" "$2"
-  key_set "$1" "ALG_STR" "$3"
-  key_set "$1" "ALG_LEN" "$4"
-}
-set_keysigning() {
-  key_set "$1" "EXPECT" "yes"
-  key_set "$1" "EXPECT_KRRSIG" "$2"
-}
-set_zonesigning() {
-  key_set "$1" "EXPECT" "yes"
-  key_set "$1" "EXPECT_ZRRSIG" "$2"
-}
-
-# Set key timing metadata. Set to "none" to unset.
-# $1: Key to update (KEY1, KEY2, ...)
-# $2: Time to update (PUBLISHED, SYNCPUBLISH, ACTIVE, RETIRED, REVOKED, or REMOVED).
-# $3: Value
-set_keytime() {
-  key_set "$1" "EXPECT" "yes"
-  key_set "$1" "$2" "$3"
-}
-
-# Set key timing metadata to a value plus additional time.
-# $1: Key to update (KEY1, KEY2, ...)
-# $2: Time to update (PUBLISHED, SYNCPUBLISH, ACTIVE, RETIRED, REVOKED, or REMOVED).
-# $3: Value
-# $4: Additional time.
-set_addkeytime() {
-  # Convert "%Y%m%d%H%M%S" format to epoch seconds.
-  # Then, add the additional time (can be negative).
-  _value=$3
-  _plus=$4
-  $PYTHON >python.out.$ZONE.$1.$2 <<EOF
-from datetime import datetime
-from datetime import timedelta
-_now = datetime.strptime("$_value", "%Y%m%d%H%M%S")
-_delta = timedelta(seconds=$_plus)
-_then = _now + _delta
-print(_then.strftime("%Y%m%d%H%M%S"));
-EOF
-  # Set the expected timing metadata.
-  key_set "$1" "$2" $(cat python.out.$ZONE.$1.$2)
-}
-
-# Set key state metadata. Set to "none" to unset.
-# $1: Key to update (KEY1, KEY2, ...)
-# $2: Key state to update (GOAL, STATE_DNSKEY, STATE_ZRRSIG, STATE_KRRSIG, or STATE_DS)
-# $3: Value
-set_keystate() {
-  key_set "$1" "EXPECT" "yes"
-  key_set "$1" "$2" "$3"
-}
-
-# Set key directory.
-# $1: Key to update (KEY1, KEY2, ...)
-# $2: Directory.
-set_keydir() {
-  key_set "$1" "KEYDIR" "$2"
-}
-
-# Check the key $1 with id $2.
-# This requires environment variables to be set.
-#
-# This will set the following environment variables for testing:
-# BASE_FILE="${_dir}/K${_zone}.+${_alg_numpad}+${_key_idpad}"
-# KEY_FILE="${BASE_FILE}.key"
-# PRIVATE_FILE="${BASE_FILE}.private"
-# STATE_FILE="${BASE_FILE}.state"
-# KEY_ID=$(echo $1 | sed 's/^0\{0,4\}//')
-# KEY_CREATED (from the KEY_FILE)
-check_key() {
-  _dir=$(key_get "$1" KEYDIR)
-  if [ "$_dir" = "none" ]; then
-    _dir="$DIR"
-  fi
-  _zone="$ZONE"
-  _role=$(key_get "$1" ROLE)
-  _key_idpad="$2"
-  _key_id=$(echo "$_key_idpad" | sed 's/^0\{0,4\}//')
-  _alg_num=$(key_get "$1" ALG_NUM)
-  _alg_numpad=$(printf "%03d" "$_alg_num")
-  _alg_string=$(key_get "$1" ALG_STR)
-  _length=$(key_get "$1" "ALG_LEN")
-  _dnskey_ttl="$KEYFILE_TTL"
-  _lifetime=$(key_get "$1" LIFETIME)
-  _legacy=$(key_get "$1" LEGACY)
-  _private=$(key_get "$1" PRIVATE)
-  _flags=$(key_get "$1" FLAGS)
-
-  _published=$(key_get "$1" PUBLISHED)
-  _active=$(key_get "$1" ACTIVE)
-  _retired=$(key_get "$1" RETIRED)
-  _revoked=$(key_get "$1" REVOKED)
-  _removed=$(key_get "$1" REMOVED)
-
-  _goal=$(key_get "$1" GOAL)
-  _state_dnskey=$(key_get "$1" STATE_DNSKEY)
-  _state_zrrsig=$(key_get "$1" STATE_ZRRSIG)
-  _state_krrsig=$(key_get "$1" STATE_KRRSIG)
-  _state_ds=$(key_get "$1" STATE_DS)
-
-  _ksk="no"
-  _zsk="no"
-  if [ "$_role" = "ksk" ]; then
-    _ksk="yes"
-  elif [ "$_role" = "zsk" ]; then
-    _zsk="yes"
-  elif [ "$_role" = "csk" ]; then
-    _zsk="yes"
-    _ksk="yes"
-  fi
-
-  _role2="none"
-  if [ "$_flags" = "257" ]; then
-    _role2="key-signing"
-  elif [ "$_flags" = "256" ]; then
-    _role2="zone-signing"
-  fi
-
-  BASE_FILE="${_dir}/K${_zone}.+${_alg_numpad}+${_key_idpad}"
-  KEY_FILE="${BASE_FILE}.key"
-  PRIVATE_FILE="${BASE_FILE}.private"
-  STATE_FILE="${BASE_FILE}.state"
-  KEY_ID="${_key_id}"
-
-  # Check file existence.
-  [ -s "$KEY_FILE" ] || ret=1
-  if [ "$_private" = "yes" ]; then
-    [ -s "$PRIVATE_FILE" ] || ret=1
-  fi
-  if [ "$_legacy" = "no" ]; then
-    [ -s "$STATE_FILE" ] || ret=1
-  fi
-  [ "$ret" -eq 0 ] || _log_error "${BASE_FILE} files missing"
-  [ "$ret" -eq 0 ] || return 0
-
-  # Retrieve revoked key id
-  KEY_RID=$($REVOKE -R ${BASE_FILE})
-
-  # Retrieve creation date.
-  grep "; Created:" "$KEY_FILE" >"${ZONE}.${KEY_ID}.${_alg_num}.created" || _log_error "mismatch created comment in $KEY_FILE"
-  KEY_CREATED=$(awk '{print $3}' <"${ZONE}.${KEY_ID}.${_alg_num}.created")
-
-  if [ "$_private" = "yes" ]; then
-    grep "Created: ${KEY_CREATED}" "$PRIVATE_FILE" >/dev/null || _log_error "mismatch created in $PRIVATE_FILE"
-  fi
-  if [ "$_legacy" = "no" ]; then
-    grep "Generated: ${KEY_CREATED}" "$STATE_FILE" >/dev/null || _log_error "mismatch generated in $STATE_FILE"
-  fi
-
-  test $_log -eq 1 && echo_i "check key file $BASE_FILE"
-
-  # Check the public key file.
-  grep "This is a ${_role2} key, keyid ${_key_id}, for ${_zone}." "$KEY_FILE" >/dev/null || _log_error "mismatch top comment in $KEY_FILE"
-  grep "${_zone}\. ${_dnskey_ttl} IN DNSKEY ${_flags} 3 ${_alg_num}" "$KEY_FILE" >/dev/null || _log_error "mismatch DNSKEY record in $KEY_FILE"
-  # Now check the private key file.
-  if [ "$_private" = "yes" ]; then
-    grep "Private-key-format: v1.3" "$PRIVATE_FILE" >/dev/null || _log_error "mismatch private key format in $PRIVATE_FILE"
-    grep "Algorithm: ${_alg_num} (${_alg_string})" "$PRIVATE_FILE" >/dev/null || _log_error "mismatch algorithm in $PRIVATE_FILE"
-  fi
-  # Now check the key state file.
-  if [ "$_legacy" = "no" ]; then
-    grep "This is the state of key ${_key_id}, for ${_zone}." "$STATE_FILE" >/dev/null || _log_error "mismatch top comment in $STATE_FILE"
-    if [ "$_lifetime" = "none" ]; then
-      grep "Lifetime: " "$STATE_FILE" >/dev/null && _log_error "unexpected lifetime in $STATE_FILE"
-    else
-      grep "Lifetime: ${_lifetime}" "$STATE_FILE" >/dev/null || _log_error "mismatch lifetime in $STATE_FILE"
-    fi
-    grep "Algorithm: ${_alg_num}" "$STATE_FILE" >/dev/null || _log_error "mismatch algorithm in $STATE_FILE"
-    grep "Length: ${_length}" "$STATE_FILE" >/dev/null || _log_error "mismatch length in $STATE_FILE"
-    grep "KSK: ${_ksk}" "$STATE_FILE" >/dev/null || _log_error "mismatch ksk in $STATE_FILE"
-    grep "ZSK: ${_zsk}" "$STATE_FILE" >/dev/null || _log_error "mismatch zsk in $STATE_FILE"
-
-    # Check key states.
-    if [ "$_goal" = "none" ]; then
-      grep "GoalState: " "$STATE_FILE" >/dev/null && _log_error "unexpected goal state in $STATE_FILE"
-    else
-      grep "GoalState: ${_goal}" "$STATE_FILE" >/dev/null || _log_error "mismatch goal state in $STATE_FILE"
-    fi
-
-    if [ "$_state_dnskey" = "none" ]; then
-      grep "DNSKEYState: " "$STATE_FILE" >/dev/null && _log_error "unexpected dnskey state in $STATE_FILE"
-      grep "DNSKEYChange: " "$STATE_FILE" >/dev/null && _log_error "unexpected dnskey change in $STATE_FILE"
-    else
-      grep "DNSKEYState: ${_state_dnskey}" "$STATE_FILE" >/dev/null || _log_error "mismatch dnskey state in $STATE_FILE"
-      grep "DNSKEYChange: " "$STATE_FILE" >/dev/null || _log_error "mismatch dnskey change in $STATE_FILE"
-    fi
-
-    if [ "$_state_zrrsig" = "none" ]; then
-      grep "ZRRSIGState: " "$STATE_FILE" >/dev/null && _log_error "unexpected zrrsig state in $STATE_FILE"
-      grep "ZRRSIGChange: " "$STATE_FILE" >/dev/null && _log_error "unexpected zrrsig change in $STATE_FILE"
-    else
-      grep "ZRRSIGState: ${_state_zrrsig}" "$STATE_FILE" >/dev/null || _log_error "mismatch zrrsig state in $STATE_FILE"
-      grep "ZRRSIGChange: " "$STATE_FILE" >/dev/null || _log_error "mismatch zrrsig change in $STATE_FILE"
-    fi
-
-    if [ "$_state_krrsig" = "none" ]; then
-      grep "KRRSIGState: " "$STATE_FILE" >/dev/null && _log_error "unexpected krrsig state in $STATE_FILE"
-      grep "KRRSIGChange: " "$STATE_FILE" >/dev/null && _log_error "unexpected krrsig change in $STATE_FILE"
-    else
-      grep "KRRSIGState: ${_state_krrsig}" "$STATE_FILE" >/dev/null || _log_error "mismatch krrsig state in $STATE_FILE"
-      grep "KRRSIGChange: " "$STATE_FILE" >/dev/null || _log_error "mismatch krrsig change in $STATE_FILE"
-    fi
-
-    if [ "$_state_ds" = "none" ]; then
-      grep "DSState: " "$STATE_FILE" >/dev/null && _log_error "unexpected ds state in $STATE_FILE"
-      grep "DSChange: " "$STATE_FILE" >/dev/null && _log_error "unexpected ds change in $STATE_FILE"
-    else
-      grep "DSState: ${_state_ds}" "$STATE_FILE" >/dev/null || _log_error "mismatch ds state in $STATE_FILE"
-      grep "DSChange: " "$STATE_FILE" >/dev/null || _log_error "mismatch ds change in $STATE_FILE"
-    fi
-  fi
-
-  return 0
-}
-
-# Check the key timing metadata for key $1.
-check_timingmetadata() {
-  _dir=$(key_get "$1" KEYDIR)
-  if [ "$_dir" = "none" ]; then
-    _dir="$DIR"
-  fi
-  _zone="$ZONE"
-  _key_idpad=$(key_get "$1" ID)
-  _key_id=$(echo "$_key_idpad" | sed 's/^0\{0,4\}//')
-  _alg_num=$(key_get "$1" ALG_NUM)
-  _alg_numpad=$(printf "%03d" "$_alg_num")
-
-  _published=$(key_get "$1" PUBLISHED)
-  _active=$(key_get "$1" ACTIVE)
-  _retired=$(key_get "$1" RETIRED)
-  _revoked=$(key_get "$1" REVOKED)
-  _removed=$(key_get "$1" REMOVED)
-
-  _goal=$(key_get "$1" GOAL)
-  _state_dnskey=$(key_get "$1" STATE_DNSKEY)
-  _state_zrrsig=$(key_get "$1" STATE_ZRRSIG)
-  _state_krrsig=$(key_get "$1" STATE_KRRSIG)
-  _state_ds=$(key_get "$1" STATE_DS)
-
-  _base_file=$(key_get "$1" BASEFILE)
-  _key_file="${_base_file}.key"
-  _private_file="${_base_file}.private"
-  _state_file="${_base_file}.state"
-  _legacy=$(key_get "$1" LEGACY)
-  _private=$(key_get "$1" PRIVATE)
-
-  _published=$(key_get "$1" PUBLISHED)
-  _syncpublish=$(key_get "$1" SYNCPUBLISH)
-  _active=$(key_get "$1" ACTIVE)
-  _retired=$(key_get "$1" RETIRED)
-  _revoked=$(key_get "$1" REVOKED)
-  _removed=$(key_get "$1" REMOVED)
-
-  # Check timing metadata.
-  n=$((n + 1))
-  echo_i "check key timing metadata for key $1 id ${_key_id} zone ${ZONE} ($n)"
-  ret=0
-
-  if [ "$_published" = "none" ]; then
-    grep "; Publish:" "${_key_file}" >/dev/null && _log_error "unexpected publish comment in ${_key_file}"
-    if [ "$_private" = "yes" ]; then
-      grep "Publish:" "${_private_file}" >/dev/null && _log_error "unexpected publish in ${_private_file}"
-    fi
-    if [ "$_legacy" = "no" ]; then
-      grep "Published: " "${_state_file}" >/dev/null && _log_error "unexpected publish in ${_state_file}"
-    fi
-  else
-    grep "; Publish: $_published" "${_key_file}" >/dev/null || _log_error "mismatch publish comment in ${_key_file} (expected ${_published})"
-    if [ "$_private" = "yes" ]; then
-      grep "Publish: $_published" "${_private_file}" >/dev/null || _log_error "mismatch publish in ${_private_file} (expected ${_published})"
-    fi
-    if [ "$_legacy" = "no" ]; then
-      grep "Published: $_published" "${_state_file}" >/dev/null || _log_error "mismatch publish in ${_state_file} (expected ${_published})"
-    fi
-  fi
-
-  if [ "$_syncpublish" = "none" ]; then
-    grep "; SyncPublish:" "${_key_file}" >/dev/null && _log_error "unexpected syncpublish comment in ${_key_file}"
-    if [ "$_private" = "yes" ]; then
-      grep "SyncPublish:" "${_private_file}" >/dev/null && _log_error "unexpected syncpublish in ${_private_file}"
-    fi
-    if [ "$_legacy" = "no" ]; then
-      grep "PublishCDS: " "${_state_file}" >/dev/null && _log_error "unexpected syncpublish in ${_state_file}"
-    fi
-  else
-    grep "; SyncPublish: $_syncpublish" "${_key_file}" >/dev/null || _log_error "mismatch syncpublish comment in ${_key_file} (expected ${_syncpublish})"
-    if [ "$_private" = "yes" ]; then
-      grep "SyncPublish: $_syncpublish" "${_private_file}" >/dev/null || _log_error "mismatch syncpublish in ${_private_file} (expected ${_syncpublish})"
-    fi
-    if [ "$_legacy" = "no" ]; then
-      grep "PublishCDS: $_syncpublish" "${_state_file}" >/dev/null || _log_error "mismatch syncpublish in ${_state_file} (expected ${_syncpublish})"
-    fi
-  fi
-
-  if [ "$_active" = "none" ]; then
-    grep "; Activate:" "${_key_file}" >/dev/null && _log_error "unexpected active comment in ${_key_file}"
-    if [ "$_private" = "yes" ]; then
-      grep "Activate:" "${_private_file}" >/dev/null && _log_error "unexpected active in ${_private_file}"
-    fi
-    if [ "$_legacy" = "no" ]; then
-      grep "Active: " "${_state_file}" >/dev/null && _log_error "unexpected active in ${_state_file}"
-    fi
-  else
-    grep "; Activate: $_active" "${_key_file}" >/dev/null || _log_error "mismatch active comment in ${_key_file} (expected ${_active})"
-    if [ "$_private" = "yes" ]; then
-      grep "Activate: $_active" "${_private_file}" >/dev/null || _log_error "mismatch active in ${_private_file} (expected ${_active})"
-    fi
-    if [ "$_legacy" = "no" ]; then
-      grep "Active: $_active" "${_state_file}" >/dev/null || _log_error "mismatch active in ${_state_file} (expected ${_active})"
-    fi
-  fi
-
-  if [ "$_retired" = "none" ]; then
-    grep "; Inactive:" "${_key_file}" >/dev/null && _log_error "unexpected retired comment in ${_key_file}"
-    if [ "$_private" = "yes" ]; then
-      grep "Inactive:" "${_private_file}" >/dev/null && _log_error "unexpected retired in ${_private_file}"
-    fi
-    if [ "$_legacy" = "no" ]; then
-      grep "Retired: " "${_state_file}" >/dev/null && _log_error "unexpected retired in ${_state_file}"
-    fi
-  else
-    grep "; Inactive: $_retired" "${_key_file}" >/dev/null || _log_error "mismatch retired comment in ${_key_file} (expected ${_retired})"
-    if [ "$_private" = "yes" ]; then
-      grep "Inactive: $_retired" "${_private_file}" >/dev/null || _log_error "mismatch retired in ${_private_file} (expected ${_retired})"
-    fi
-    if [ "$_legacy" = "no" ]; then
-      grep "Retired: $_retired" "${_state_file}" >/dev/null || _log_error "mismatch retired in ${_state_file} (expected ${_retired})"
-    fi
-  fi
-
-  if [ "$_revoked" = "none" ]; then
-    grep "; Revoke:" "${_key_file}" >/dev/null && _log_error "unexpected revoked comment in ${_key_file}"
-    if [ "$_private" = "yes" ]; then
-      grep "Revoke:" "${_private_file}" >/dev/null && _log_error "unexpected revoked in ${_private_file}"
-    fi
-    if [ "$_legacy" = "no" ]; then
-      grep "Revoked: " "${_state_file}" >/dev/null && _log_error "unexpected revoked in ${_state_file}"
-    fi
-  else
-    grep "; Revoke: $_revoked" "${_key_file}" >/dev/null || _log_error "mismatch revoked comment in ${_key_file} (expected ${_revoked})"
-    if [ "$_private" = "yes" ]; then
-      grep "Revoke: $_revoked" "${_private_file}" >/dev/null || _log_error "mismatch revoked in ${_private_file} (expected ${_revoked})"
-    fi
-    if [ "$_legacy" = "no" ]; then
-      grep "Revoked: $_revoked" "${_state_file}" >/dev/null || _log_error "mismatch revoked in ${_state_file} (expected ${_revoked})"
-    fi
-  fi
-
-  if [ "$_removed" = "none" ]; then
-    grep "; Delete:" "${_key_file}" >/dev/null && _log_error "unexpected removed comment in ${_key_file}"
-    if [ "$_private" = "yes" ]; then
-      grep "Delete:" "${_private_file}" >/dev/null && _log_error "unexpected removed in ${_private_file}"
-    fi
-    if [ "$_legacy" = "no" ]; then
-      grep "Removed: " "${_state_file}" >/dev/null && _log_error "unexpected removed in ${_state_file}"
-    fi
-  else
-    grep "; Delete: $_removed" "${_key_file}" >/dev/null || _log_error "mismatch removed comment in ${_key_file} (expected ${_removed})"
-    if [ "$_private" = "yes" ]; then
-      grep "Delete: $_removed" "${_private_file}" >/dev/null || _log_error "mismatch removed in ${_private_file} (expected ${_removed})"
-    fi
-    if [ "$_legacy" = "no" ]; then
-      grep "Removed: $_removed" "${_state_file}" >/dev/null || _log_error "mismatch removed in ${_state_file} (expected ${_removed})"
-    fi
-  fi
-
-  test "$ret" -eq 0 || echo_i "failed"
-  status=$((status + ret))
-}
-
-check_keytimes() {
-  # The script relies on Python to set keytimes.
-  if [ "$(key_get KEY1 EXPECT)" = "yes" ]; then
-    check_timingmetadata "KEY1"
-  fi
-  if [ "$(key_get KEY2 EXPECT)" = "yes" ]; then
-    check_timingmetadata "KEY2"
-  fi
-  if [ "$(key_get KEY3 EXPECT)" = "yes" ]; then
-    check_timingmetadata "KEY3"
-  fi
-  if [ "$(key_get KEY4 EXPECT)" = "yes" ]; then
-    check_timingmetadata "KEY4"
-  fi
-}
-
-# Check the key with key id $1 and see if it is unused.
-# This requires environment variables to be set.
-#
-# This will set the following environment variables for testing:
-# BASE_FILE="${_dir}/K${_zone}.+${_alg_numpad}+${_key_idpad}"
-# KEY_FILE="${BASE_FILE}.key"
-# PRIVATE_FILE="${BASE_FILE}.private"
-# STATE_FILE="${BASE_FILE}.state"
-# KEY_ID=$(echo $1 | sed 's/^0\{0,4\}//')
-key_unused() {
-  _dir="$DIR"
-  _zone="$ZONE"
-  _key_idpad="$1"
-  _key_id=$(echo "$_key_idpad" | sed 's/^0\{0,4\}//')
-  _alg_num="$2"
-  _alg_numpad=$(printf "%03d" "$_alg_num")
-
-  BASE_FILE="${_dir}/K${_zone}.+${_alg_numpad}+${_key_idpad}"
-  KEY_FILE="${BASE_FILE}.key"
-  PRIVATE_FILE="${BASE_FILE}.private"
-  STATE_FILE="${BASE_FILE}.state"
-  KEY_ID="${_key_id}"
-
-  test $_log -eq 1 && echo_i "key unused $KEY_ID?"
-
-  # Check file existence.
-  [ -s "$KEY_FILE" ] || ret=1
-  [ -s "$PRIVATE_FILE" ] || ret=1
-  [ -s "$STATE_FILE" ] || ret=1
-  [ "$ret" -eq 0 ] || return 0
-
-  # Treat keys that have been removed from the zone as unused.
-  _check_removed=1
-  grep "; Created:" "$KEY_FILE" >created.key-${KEY_ID}.test${n} || _check_removed=0
-  grep "; Delete:" "$KEY_FILE" >unused.key-${KEY_ID}.test${n} || _check_removed=0
-  if [ "$_check_removed" -eq 1 ]; then
-    _created=$(awk '{print $3}' <created.key-${KEY_ID}.test${n})
-    _removed=$(awk '{print $3}' <unused.key-${KEY_ID}.test${n})
-    [ "$_removed" -le "$_created" ] && return
-  fi
-
-  # If no timing metadata is set, this key is unused.
-  grep "; Publish:" "$KEY_FILE" >/dev/null && _log_error "unexpected publish comment in $KEY_FILE"
-  grep "; Activate:" "$KEY_FILE" >/dev/null && _log_error "unexpected active comment in $KEY_FILE"
-  grep "; Inactive:" "$KEY_FILE" >/dev/null && _log_error "unexpected retired comment in $KEY_FILE"
-  grep "; Revoke:" "$KEY_FILE" >/dev/null && _log_error "unexpected revoked comment in $KEY_FILE"
-  grep "; Delete:" "$KEY_FILE" >/dev/null && _log_error "unexpected removed comment in $KEY_FILE"
-
-  grep "Publish:" "$PRIVATE_FILE" >/dev/null && _log_error "unexpected publish in $PRIVATE_FILE"
-  grep "Activate:" "$PRIVATE_FILE" >/dev/null && _log_error "unexpected active in $PRIVATE_FILE"
-  grep "Inactive:" "$PRIVATE_FILE" >/dev/null && _log_error "unexpected retired in $PRIVATE_FILE"
-  grep "Revoke:" "$PRIVATE_FILE" >/dev/null && _log_error "unexpected revoked in $PRIVATE_FILE"
-  grep "Delete:" "$PRIVATE_FILE" >/dev/null && _log_error "unexpected removed in $PRIVATE_FILE"
-
-  grep "Published: " "$STATE_FILE" >/dev/null && _log_error "unexpected publish in $STATE_FILE"
-  grep "Active: " "$STATE_FILE" >/dev/null && _log_error "unexpected active in $STATE_FILE"
-  grep "Retired: " "$STATE_FILE" >/dev/null && _log_error "unexpected retired in $STATE_FILE"
-  grep "Revoked: " "$STATE_FILE" >/dev/null && _log_error "unexpected revoked in $STATE_FILE"
-  grep "Removed: " "$STATE_FILE" >/dev/null && _log_error "unexpected removed in $STATE_FILE"
-
-  return 0
-}
-
-# Test: dnssec-verify zone $1.
-dnssec_verify() {
-  n=$((n + 1))
-  echo_i "dnssec-verify zone ${ZONE} ($n)"
-  ret=0
-  _dig_with_opts "$ZONE" "@${SERVER}" AXFR >dig.out.axfr.test$n || _log_error "dig ${ZONE} AXFR failed"
-  $VERIFY -z -o "$ZONE" dig.out.axfr.test$n >verify.out.$ZONE.test$n || _log_error "dnssec verify zone $ZONE failed"
-  test "$ret" -eq 0 || echo_i "failed"
-  status=$((status + ret))
-}
-
-# Wait for the zone to be signed.
-# The apex NSEC record indicates that it is signed.
-_wait_for_nsec() {
-  _dig_with_opts "@${SERVER}" "$ZONE" NSEC >"dig.out.nsec.test$n" || return 1
-  grep "NS SOA" "dig.out.nsec.test$n" >/dev/null || return 1
-  grep "${ZONE}\..*IN.*RRSIG" "dig.out.nsec.test$n" >/dev/null || return 1
-  return 0
-}
-wait_for_nsec() {
-  n=$((n + 1))
-  ret=0
-  echo_i "wait for ${ZONE} to be signed ($n)"
-  retry_quiet 10 _wait_for_nsec || _log_error "wait for ${ZONE} to be signed failed"
-  test "$ret" -eq 0 || echo_i "failed"
-  status=$((status + ret))
-}
-
-check_numkeys() {
-  _numkeys=$(get_keyids "$DIR" "$ZONE" | wc -l)
-  test "$_numkeys" -eq "$NUM_KEYS" || return 1
-  return 0
-}
-
-_check_keys() {
-  ret=0
-  _ret=0
-
-  # Clear key ids.
-  if [ "$1" != "keep" ]; then
-    key_set KEY1 ID "no"
-    key_set KEY2 ID "no"
-    key_set KEY3 ID "no"
-    key_set KEY4 ID "no"
-  fi
-
-  # Check key files.
-  _ids=$(get_keyids "$DIR" "$ZONE")
-  for _id in $_ids; do
-    # There are multiple key files with the same algorithm.
-    # Check them until a match is found.
-    ret=0
-    echo_i "check key id $_id"
-
-    if [ "no" = "$(key_get KEY1 ID)" ] && [ "$(key_get KEY1 EXPECT)" = "yes" ]; then
-      ret=0
-      check_key "KEY1" "$_id"
-      test "$ret" -eq 0 && key_save KEY1 && continue
-    fi
-    if [ "no" = "$(key_get KEY2 ID)" ] && [ "$(key_get KEY2 EXPECT)" = "yes" ]; then
-      ret=0
-      check_key "KEY2" "$_id"
-      test "$ret" -eq 0 && key_save KEY2 && continue
-    fi
-    if [ "no" = "$(key_get KEY3 ID)" ] && [ "$(key_get KEY3 EXPECT)" = "yes" ]; then
-      ret=0
-      check_key "KEY3" "$_id"
-      test "$ret" -eq 0 && key_save KEY3 && continue
-    fi
-    if [ "no" = "$(key_get KEY4 ID)" ] && [ "$(key_get KEY4 EXPECT)" = "yes" ]; then
-      ret=0
-      check_key "KEY4" "$_id"
-      test "$ret" -eq 0 && key_save KEY4 && continue
-    fi
-
-    # This may be an unused key. Assume algorithm of KEY1.
-    ret=0 && key_unused "$_id" "$(key_get KEY1 ALG_NUM)"
-    test "$ret" -eq 0 && continue
-
-    # If ret is still non-zero, none of the files matched.
-    echo_i "failed"
-    _ret=1
-  done
-
-  return $_ret
-}
-
-# Check keys for a configured zone. This verifies:
-# 1. The right number of keys exist in the key pool ($1).
-# 2. The right number of keys is active. Checks KEY1, KEY2, KEY3, and KEY4.
-#
-# It is expected that KEY1, KEY2, KEY3, and KEY4 arrays are set correctly.
-# Found key identifiers are stored in the right key array.
-# Keys are found if they are stored inside $DIR or in a subdirectory up to
-# three levels deeper.
-#
-# If $1 is set, we keep keys that are already found and don't look for them
-# again.
-check_keys() {
-  n=$((n + 1))
-  echo_i "check keys are created for zone ${ZONE} ($n)"
-  ret=0
-
-  echo_i "check number of keys for zone ${ZONE} in dir ${DIR} ($n)"
-  retry_quiet 10 check_numkeys || ret=1
-  if [ $ret -ne 0 ]; then
-    _numkeys=$(get_keyids "$DIR" "$ZONE" | wc -l)
-    _log_error "bad number of key files ($_numkeys) for zone $ZONE (expected $NUM_KEYS)"
-    status=$((status + ret))
-  fi
-
-  # Temporarily don't log errors because we are searching multiple files.
-  disable_logerror
-
-  retry_quiet 3 _check_keys $1 || ret=1
-  test "$ret" -eq 0 || echo_i "failed"
-  status=$((status + ret))
-
-  # Turn error logs on again.
-  enable_logerror
-
-  ret=0
-  if [ "$(key_get KEY1 EXPECT)" = "yes" ]; then
-    echo_i "KEY1 ID $(key_get KEY1 ID) ALG $(key_get KEY1 ALG_STR)"
-    test "no" = "$(key_get KEY1 ID)" && _log_error "No KEY1 found for zone ${ZONE}"
-  fi
-  if [ "$(key_get KEY2 EXPECT)" = "yes" ]; then
-    echo_i "KEY2 ID $(key_get KEY2 ID) ALG $(key_get KEY2 ALG_STR)"
-    test "no" = "$(key_get KEY2 ID)" && _log_error "No KEY2 found for zone ${ZONE}"
-  fi
-  if [ "$(key_get KEY3 EXPECT)" = "yes" ]; then
-    echo_i "KEY3 ID $(key_get KEY3 ID) ALG $(key_get KEY3 ALG_STR)"
-    test "no" = "$(key_get KEY3 ID)" && _log_error "No KEY3 found for zone ${ZONE}"
-  fi
-  if [ "$(key_get KEY4 EXPECT)" = "yes" ]; then
-    echo_i "KEY4 ID $(key_get KEY4 ID) ALG $(key_get KEY4 ALG_STR)"
-    test "no" = "$(key_get KEY4 ID)" && _log_error "No KEY4 found for zone ${ZONE}"
-  fi
-  test "$ret" -eq 0 || echo_i "failed"
-  status=$((status + ret))
-}
-
-# Call rndc dnssec -status on server $1 for zone $3 in view $4 with policy $2
-# and check output. This is a loose verification, it just tests if the right
-# policy name is returned, and if all expected keys are listed.  The rndc
-# dnssec -status output also lists whether a key is published,
-# used for signing, is retired, or is removed, and if not when
-# it is scheduled to do so, and it shows the states for the various
-# DNSSEC records.
-check_dnssecstatus() {
-  _server=$1
-  _policy=$2
-  _zone=$3
-  _view=$4
-
-  n=$((n + 1))
-  echo_i "check rndc dnssec -status output for ${_zone} (policy: $_policy) ($n)"
-  ret=0
-
-  _rndccmd $_server dnssec -status $_zone in $_view >rndc.dnssec.status.out.$_zone.$n || _log_error "rndc dnssec -status zone ${_zone} failed"
-
-  if [ "$_policy" = "none" ]; then
-    grep "Zone does not have dnssec-policy" rndc.dnssec.status.out.$_zone.$n >/dev/null || log_error "bad dnssec status for unsigned zone ${_zone}"
-  else
-    grep "dnssec-policy: ${_policy}" rndc.dnssec.status.out.$_zone.$n >/dev/null || _log_error "bad dnssec status for signed zone ${_zone}"
-    if [ "$(key_get KEY1 EXPECT)" = "yes" ]; then
-      grep "key: $(key_get KEY1 ID)" rndc.dnssec.status.out.$_zone.$n >/dev/null || _log_error "missing key $(key_get KEY1 ID) from dnssec status"
-    fi
-    if [ "$(key_get KEY2 EXPECT)" = "yes" ]; then
-      grep "key: $(key_get KEY2 ID)" rndc.dnssec.status.out.$_zone.$n >/dev/null || _log_error "missing key $(key_get KEY2 ID) from dnssec status"
-    fi
-    if [ "$(key_get KEY3 EXPECT)" = "yes" ]; then
-      grep "key: $(key_get KEY3 ID)" rndc.dnssec.status.out.$_zone.$n >/dev/null || _log_error "missing key $(key_get KEY3 ID) from dnssec status"
-    fi
-    if [ "$(key_get KEY4 EXPECT)" = "yes" ]; then
-      grep "key: $(key_get KEY4 ID)" rndc.dnssec.status.out.$_zone.$n >/dev/null || _log_error "missing key $(key_get KEY4 ID) from dnssec status"
-    fi
-  fi
-
-  test "$ret" -eq 0 || echo_i "failed"
-  status=$((status + ret))
-}
-
-# Call rndc zonestatus on server $1 for zone $2 in view $3 and check output if
-# inline-signing is enabled.
-check_inlinesigning() {
-  _server=$1
-  _zone=$2
-  _view=$3
-
-  _rndccmd $_server zonestatus $_zone in $_view >rndc.zonestatus.out.$_zone.$n || return 1
-  grep "inline signing: yes" rndc.zonestatus.out.$_zone.$n >/dev/null || return 1
-}
-
-# Call rndc zonestatus on server $1 for zone $2 in view $3 and check output if
-# the zone is dynamic.
-check_isdynamic() {
-  _server=$1
-  _zone=$2
-  _view=$3
-
-  _rndccmd $_server zonestatus $_zone in $_view >rndc.zonestatus.out.$_zone.$n || return 1
-  grep "dynamic: yes" rndc.zonestatus.out.$_zone.$n >/dev/null || return 1
-}
-
-# Check if RRset of type $1 in file $2 is signed with the right keys.
-# The right keys are the ones that expect a signature and matches the role $3.
-_check_signatures() {
-  _qtype=$1
-  _file=$2
-  _role=$3
-
-  numsigs=0
-
-  if [ "$_role" = "KSK" ]; then
-    _expect_type=EXPECT_KRRSIG
-  elif [ "$_role" = "ZSK" ]; then
-    _expect_type=EXPECT_ZRRSIG
-  fi
-
-  if [ "$(key_get KEY1 "$_expect_type")" = "yes" ] && [ "$(key_get KEY1 "$_role")" = "yes" ]; then
-    get_keys_which_signed "$_qtype" "$(key_get KEY1 ALG_NUM)" "$_file" | grep "^$(key_get KEY1 ID)$" >/dev/null || return 1
-    numsigs=$((numsigs + 1))
-  elif [ "$(key_get KEY1 EXPECT)" = "yes" ]; then
-    get_keys_which_signed "$_qtype" "$(key_get KEY1 ALG_NUM)" "$_file" | grep "^$(key_get KEY1 ID)$" >/dev/null && return 1
-  fi
-
-  if [ "$(key_get KEY2 "$_expect_type")" = "yes" ] && [ "$(key_get KEY2 "$_role")" = "yes" ]; then
-    get_keys_which_signed "$_qtype" "$(key_get KEY2 ALG_NUM)" "$_file" | grep "^$(key_get KEY2 ID)$" >/dev/null || return 1
-    numsigs=$((numsigs + 1))
-  elif [ "$(key_get KEY2 EXPECT)" = "yes" ]; then
-    get_keys_which_signed "$_qtype" "$(key_get KEY2 ALG_NUM)" "$_file" | grep "^$(key_get KEY2 ID)$" >/dev/null && return 1
-  fi
-
-  if [ "$(key_get KEY3 "$_expect_type")" = "yes" ] && [ "$(key_get KEY3 "$_role")" = "yes" ]; then
-    get_keys_which_signed "$_qtype" "$(key_get KEY3 ALG_NUM)" "$_file" | grep "^$(key_get KEY3 ID)$" >/dev/null || return 1
-    numsigs=$((numsigs + 1))
-  elif [ "$(key_get KEY3 EXPECT)" = "yes" ]; then
-    get_keys_which_signed "$_qtype" "$(key_get KEY3 ALG_NUM)" "$_file" | grep "^$(key_get KEY3 ID)$" >/dev/null && return 1
-  fi
-
-  if [ "$(key_get KEY4 "$_expect_type")" = "yes" ] && [ "$(key_get KEY4 "$_role")" = "yes" ]; then
-    get_keys_which_signed "$_qtype" "$(key_get KEY4 ALG_NUM)" "$_file" | grep "^$(key_get KEY4 ID)$" >/dev/null || return 1
-    numsigs=$((numsigs + 1))
-  elif [ "$(key_get KEY4 EXPECT)" = "yes" ]; then
-    get_keys_which_signed "$_qtype" "$(key_get KEY4 ALG_NUM)" "$_file" | grep "^$(key_get KEY4 ID)$" >/dev/null && return 1
-  fi
-
-  lines=$(get_keys_which_signed "${_qtype}" "0" "${_file}" | wc -l)
-  test "$lines" -eq "$numsigs" || echo_i "bad number of signatures for $_qtype (got $lines, expected $numsigs)"
-  test "$lines" -eq "$numsigs" || return 1
-
-  return 0
-}
-check_signatures() {
-  retry_quiet 3 _check_signatures $1 $2 $3 || _log_error "RRset $1 in zone $ZONE incorrectly signed"
-}
-
-response_has_cds_for_key() {
-  awk -v zone="${ZONE%%.}." \
-    -v ttl="${DNSKEY_TTL}" \
-    -v qtype="CDS" \
-    -v keyid="$(key_get "${2}" ID)" \
-    -v keyalg="$(key_get "${2}" ALG_NUM)" \
-    -v hashalg="$1" \
-    'BEGIN { ret=1; }
-	     $1 == zone && $2 == ttl && $4 == qtype && $5 == keyid && $6 == keyalg && $7 == hashalg { ret=0; exit; }
-	     END { exit ret; }' \
-    "$3"
-}
-
-response_has_cdnskey_for_key() (
-
-  awk -v zone="${ZONE%%.}." \
-    -v ttl="${DNSKEY_TTL}" \
-    -v qtype="CDNSKEY" \
-    -v flags="$(key_get "${1}" FLAGS)" \
-    -v keyalg="$(key_get "${1}" ALG_NUM)" \
-    'BEGIN { ret=1; }
-	     $1 == zone && $2 == ttl && $4 == qtype && $5 == flags && $7 == keyalg { ret=0; exit; }
-	     END { exit ret; }' \
-    "$2"
-)
-
-check_cds_digests() {
-  if [ "$CDS_SHA256" = "yes" ]; then
-    response_has_cds_for_key 2 $1 "${2}.cds" || _log_error "missing CDS 2 record in response for key $(key_get $1 ID)"
-  else
-    response_has_cds_for_key 2 $1 "${2}.cds" && _log_error "unexpected CDS 2 record in response for key $(key_get $1 ID)"
-  fi
-
-  if [ "$CDS_SHA384" = "yes" ]; then
-    response_has_cds_for_key 4 $1 "${2}.cds" || _log_error "missing CDS 4 record in response for key $(key_get $1 ID)"
-  else
-    response_has_cds_for_key 4 $1 "${2}.cds" && _log_error "unexpected CDS 4 record in response for key $(key_get $1 ID)"
-  fi
-
-  if [ "$CDNSKEY" = "yes" ]; then
-    response_has_cdnskey_for_key $1 "${2}.cdnskey" || _log_error "missing CDNSKEY record in response for key $(key_get $1 ID)"
-  else
-    response_has_cdnskey_for_key $1 "${2}.cdnskey" && _log_error "unexpected CDNSKEY record in response for key $(key_get $1 ID)"
-  fi
-
-  return 0
-}
-
-check_cds_digests_invert() {
-  response_has_cds_for_key 2 $1 "${2}.cds" && _log_error "unexpected CDS 2 record in response for key $(key_get $1 ID)"
-  response_has_cds_for_key 4 $1 "${2}.cds" && _log_error "unexpected CDS 4 record in response for key $(key_get $1 ID)"
-  # The key should not have an associated CDNSKEY, but there may be
-  # one for another key.  Since the CDNSKEY has no field for key
-  # id, it is hard to check what key the CDNSKEY may belong to
-  # so let's skip this check for now.
-
-  return 0
-}
-
-# Test CDS and CDNSKEY publication.
-check_cds() {
-
-  n=$((n + 1))
-  echo_i "check CDS and CDNSKEY rrset are signed correctly for zone ${ZONE} ($n)"
-  ret=0
-
-  _checksig=0
-
-  _dig_with_opts "$ZONE" "@${SERVER}" "CDS" >"dig.out.$DIR.test$n.cds" || _log_error "dig ${ZONE} CDS failed"
-  grep "status: NOERROR" "dig.out.$DIR.test$n.cds" >/dev/null || _log_error "mismatch status in DNS response"
-
-  _dig_with_opts "$ZONE" "@${SERVER}" "CDNSKEY" >"dig.out.$DIR.test$n.cdnskey" || _log_error "dig ${ZONE} CDNSKEY failed"
-  grep "status: NOERROR" "dig.out.$DIR.test$n.cdnskey" >/dev/null || _log_error "mismatch status in DNS response"
-
-  if [ "$CDS_DELETE" = "no" ]; then
-    grep "CDS.*0 0 0 00" "dig.out.$DIR.test$n.cds" >/dev/null && _log_error "unexpected CDS DELETE record in DNS response"
-    grep "CDNSKEY.*0 3 0 AA==" "dig.out.$DIR.test$n.cdnskey" >/dev/null && _log_error "unexpected CDNSKEY DELETE record in DNS response"
-  else
-    grep "CDS.*0 0 0 00" "dig.out.$DIR.test$n.cds" >/dev/null || _log_error "missing CDS DELETE record in DNS response"
-    grep "CDNSKEY.*0 3 0 AA==" "dig.out.$DIR.test$n.cdnskey" >/dev/null || _log_error "missing CDNSKEY DELETE record in DNS response"
-    _checksig=1
-  fi
-
-  if [ "$(key_get KEY1 STATE_DS)" = "rumoured" ] || [ "$(key_get KEY1 STATE_DS)" = "omnipresent" ]; then
-    check_cds_digests KEY1 "dig.out.$DIR.test$n"
-    _checksig=1
-  elif [ "$(key_get KEY1 EXPECT)" = "yes" ]; then
-    check_cds_digests_invert KEY1 "dig.out.$DIR.test$n"
-  fi
-
-  if [ "$(key_get KEY2 STATE_DS)" = "rumoured" ] || [ "$(key_get KEY2 STATE_DS)" = "omnipresent" ]; then
-    check_cds_digests KEY2 "dig.out.$DIR.test$n"
-    _checksig=1
-  elif [ "$(key_get KEY2 EXPECT)" = "yes" ]; then
-    check_cds_digests_invert KEY2 "dig.out.$DIR.test$n"
-  fi
-
-  if [ "$(key_get KEY3 STATE_DS)" = "rumoured" ] || [ "$(key_get KEY3 STATE_DS)" = "omnipresent" ]; then
-    check_cds_digests KEY3 "dig.out.$DIR.test$n"
-    _checksig=1
-  elif [ "$(key_get KEY3 EXPECT)" = "yes" ]; then
-    check_cds_digests_invert KEY3 "dig.out.$DIR.test$n"
-  fi
-
-  if [ "$(key_get KEY4 STATE_DS)" = "rumoured" ] || [ "$(key_get KEY4 STATE_DS)" = "omnipresent" ]; then
-    check_cds_digests KEY4 "dig.out.$DIR.test$n"
-    _checksig=1
-  elif [ "$(key_get KEY4 EXPECT)" = "yes" ]; then
-    check_cds_digests_invert KEY4 "dig.out.$DIR.test$n"
-  fi
-
-  test "$_checksig" -eq 0 || check_signatures "CDS" "dig.out.$DIR.test$n.cds" "KSK"
-
-  if [ "$CDNSKEY" = "yes" ]; then
-    test "$_checksig" -eq 0 || check_signatures "CDNSKEY" "dig.out.$DIR.test$n.cdnskey" "KSK"
-  fi
-
-  test "$ret" -eq 0 || echo_i "failed"
-  status=$((status + ret))
-}
-
-_find_dnskey() {
-  _owner="${ZONE}."
-  _alg="$(key_get $1 ALG_NUM)"
-  _flags="$(key_get $1 FLAGS)"
-  _key_file="$(key_get $1 BASEFILE).key"
-
-  awk '$1 == "'"$_owner"'" && $2 == "'"$KEYFILE_TTL"'" && $3 == "IN" && $4 == "DNSKEY" && $5 == "'"$_flags"'" && $6 == "3" && $7 == "'"$_alg"'" { print $8 }' <"$_key_file"
-}
-
-# Test DNSKEY query.
-_check_apex_dnskey() {
-  _dig_with_opts "$ZONE" "@${SERVER}" "DNSKEY" >"dig.out.$DIR.test$n" || return 1
-  grep "status: NOERROR" "dig.out.$DIR.test$n" >/dev/null || return 1
-
-  _checksig=0
-
-  if [ "$(key_get KEY1 STATE_DNSKEY)" = "rumoured" ] || [ "$(key_get KEY1 STATE_DNSKEY)" = "omnipresent" ]; then
-    _pubkey=$(_find_dnskey KEY1)
-    test -z "$_pubkey" && return 1
-    grep -F "$_pubkey" "dig.out.$DIR.test$n" >/dev/null || return 1
-    _checksig=1
-  elif [ "$(key_get KEY1 EXPECT)" = "yes" ]; then
-    _pubkey=$(_find_dnskey KEY1)
-    test -z "$_pubkey" && return 1
-    grep -F "$_pubkey" "dig.out.$DIR.test$n" >/dev/null && return 1
-  fi
-
-  if [ "$(key_get KEY2 STATE_DNSKEY)" = "rumoured" ] || [ "$(key_get KEY2 STATE_DNSKEY)" = "omnipresent" ]; then
-    _pubkey=$(_find_dnskey KEY2)
-    test -z "$_pubkey" && return 1
-    grep -F "$_pubkey" "dig.out.$DIR.test$n" >/dev/null || return 1
-    _checksig=1
-  elif [ "$(key_get KEY2 EXPECT)" = "yes" ]; then
-    _pubkey=$(_find_dnskey KEY2)
-    test -z "$_pubkey" && return 1
-    grep -F "$_pubkey" "dig.out.$DIR.test$n" >/dev/null && return 1
-  fi
-
-  if [ "$(key_get KEY3 STATE_DNSKEY)" = "rumoured" ] || [ "$(key_get KEY3 STATE_DNSKEY)" = "omnipresent" ]; then
-    _pubkey=$(_find_dnskey KEY3)
-    test -z "$_pubkey" && return 1
-    grep -F "$_pubkey" "dig.out.$DIR.test$n" >/dev/null || return 1
-    _checksig=1
-  elif [ "$(key_get KEY3 EXPECT)" = "yes" ]; then
-    _pubkey=$(_find_dnskey KEY3)
-    test -z "$_pubkey" && return 1
-    grep -F "$_pubkey" "dig.out.$DIR.test$n" >/dev/null && return 1
-  fi
-
-  if [ "$(key_get KEY4 STATE_DNSKEY)" = "rumoured" ] || [ "$(key_get KEY4 STATE_DNSKEY)" = "omnipresent" ]; then
-    _pubkey=$(_find_dnskey KEY4)
-    test -z "$_pubkey" && return 1
-    grep -F "$_pubkey" "dig.out.$DIR.test$n" >/dev/null || return 1
-    _checksig=1
-  elif [ "$(key_get KEY4 EXPECT)" = "yes" ]; then
-    _pubkey=$(_find_dnskey KEY4)
-    test -z "$_pubkey" && return 1
-    grep -F "$_pubkey" "dig.out.$DIR.test$n" >/dev/null && return 1
-  fi
-
-  test "$_checksig" -eq 0 && return 0
-
-  _check_signatures "DNSKEY" "dig.out.$DIR.test$n" "KSK" || return 1
-
-  return 0
-}
-
-# Test the apex of a configured zone. This checks that the SOA and DNSKEY
-# RRsets are signed correctly and with the appropriate keys.
-check_apex() {
-
-  # Test DNSKEY query.
-  n=$((n + 1))
-  echo_i "check DNSKEY rrset is signed correctly for zone ${ZONE} ($n)"
-  ret=0
-  retry_quiet 10 _check_apex_dnskey || ret=1
-  test "$ret" -eq 0 || echo_i "failed"
-  status=$((status + ret))
-
-  # We retry the DNSKEY query for at most ten seconds to avoid test
-  # failures due to timing issues. If the DNSKEY query check passes this
-  # means the zone is resigned and further apex checks (SOA, CDS, CDNSKEY)
-  # don't need to be retried quietly.
-
-  # Test SOA query.
-  n=$((n + 1))
-  echo_i "check SOA rrset is signed correctly for zone ${ZONE} ($n)"
-  ret=0
-  _dig_with_opts "$ZONE" "@${SERVER}" "SOA" >"dig.out.$DIR.test$n" || _log_error "dig ${ZONE} SOA failed"
-  grep "status: NOERROR" "dig.out.$DIR.test$n" >/dev/null || _log_error "mismatch status in DNS response"
-  grep "${ZONE}\..*${DEFAULT_TTL}.*IN.*SOA.*" "dig.out.$DIR.test$n" >/dev/null || _log_error "missing SOA record in response"
-  check_signatures "SOA" "dig.out.$DIR.test$n" "ZSK"
-  test "$ret" -eq 0 || echo_i "failed"
-  status=$((status + ret))
-
-  # Test CDS and CDNSKEY publication.
-  check_cds
-}
-
-# Test an RRset below the apex and verify it is signed correctly.
-check_subdomain() {
-  _qtype="A"
-  n=$((n + 1))
-  echo_i "check ${_qtype} a.${ZONE} rrset is signed correctly for zone ${ZONE} ($n)"
-  ret=0
-  _dig_with_opts "a.$ZONE" "@${SERVER}" $_qtype >"dig.out.$DIR.test$n" || _log_error "dig a.${ZONE} ${_qtype} failed"
-  grep "status: NOERROR" "dig.out.$DIR.test$n" >/dev/null || _log_error "mismatch status in DNS response"
-  grep "a.${ZONE}\..*${DEFAULT_TTL}.*IN.*${_qtype}.*10\.0\.0\.1" "dig.out.$DIR.test$n" >/dev/null || _log_error "missing a.${ZONE} ${_qtype} record in response"
-  lines=$(get_keys_which_signed $_qtype 0 "dig.out.$DIR.test$n" | wc -l)
-  check_signatures $_qtype "dig.out.$DIR.test$n" "ZSK"
-  test "$ret" -eq 0 || echo_i "failed"
-  status=$((status + ret))
-}
-
-# Check if "CDS/CDNSKEY Published" is logged.
-check_cdslog() {
-  _dir=$1
-  _zone=$2
-  _key=$3
-
-  _alg=$(key_get $_key ALG_STR)
-  _id=$(key_get $_key ID)
-
-  n=$((n + 1))
-  echo_i "check CDS/CDNSKEY publication is logged in ${_dir}/named.run for key ${_zone}/${_alg}/${_id} ($n)"
-  ret=0
-
-  if [ "$CDS_SHA256" = "yes" ]; then
-    grep "CDS (SHA-256) for key ${_zone}/${_alg}/${_id} is now published" "${_dir}/named.run" >/dev/null || ret=1
-  fi
-  if [ "$CDS_SHA384" = "yes" ]; then
-    grep "CDS (SHA-384) for key ${_zone}/${_alg}/${_id} is now published" "${_dir}/named.run" >/dev/null || ret=1
-  fi
-  if [ "$CDNSKEY" = "yes" ]; then
-    grep "CDNSKEY for key ${_zone}/${_alg}/${_id} is now published" "${_dir}/named.run" >/dev/null || ret=1
-  fi
-
-  test "$ret" -eq 0 || echo_i "failed"
-  status=$((status + ret))
-}
-
-# Tell named that the DS for the key in given zone has been seen in the
-# parent (this does not actually has to be true, we just issue the command
-# to make named believe it can continue with the rollover).
-rndc_checkds() {
-  _server=$1
-  _dir=$2
-  _key=$3
-  _when=$4
-  _what=$5
-  _zone=$6
-  _view=$7
-
-  _keycmd=""
-  if [ "${_key}" != "-" ]; then
-    _keyid=$(key_get $_key ID)
-    _keycmd=" -key ${_keyid}"
-  fi
-
-  _whencmd=""
-  if [ "${_when}" != "now" ]; then
-    _whencmd=" -when ${_when}"
-  fi
-
-  n=$((n + 1))
-  echo_i "calling rndc dnssec -checkds${_keycmd}${_whencmd} ${_what} zone ${_zone} in ${_view} ($n)"
-  ret=0
-
-  _rndccmd $_server dnssec -checkds $_keycmd $_whencmd $_what $_zone in $_view >rndc.dnssec.checkds.out.$_zone.$n || _log_error "rndc dnssec -checkds${_keycmd}${_whencmd} ${_what} zone ${_zone} failed"
-
-  test "$ret" -eq 0 || echo_i "failed"
-  status=$((status + ret))
-}
-
-# Tell named to schedule a key rollover.
-rndc_rollover() {
-  _server=$1
-  _dir=$2
-  _keyid=$3
-  _when=$4
-  _zone=$5
-  _view=$6
-
-  _whencmd=""
-  if [ "${_when}" != "now" ]; then
-    _whencmd="-when ${_when}"
-  fi
-
-  n=$((n + 1))
-  echo_i "calling rndc dnssec -rollover key ${_keyid} ${_whencmd} zone ${_zone} ($n)"
-  ret=0
-
-  _rndccmd $_server dnssec -rollover -key $_keyid $_whencmd $_zone in $_view >rndc.dnssec.rollover.out.$_zone.$n || _log_error "rndc dnssec -rollover (key ${_keyid} when ${_when}) zone ${_zone} failed"
-
-  test "$ret" -eq 0 || echo_i "failed"
-  status=$((status + ret))
-}

From 96cff3c001ac01de149aa7e4eed12f645e5fd30b Mon Sep 17 00:00:00 2001
From: Matthijs Mekking <matthijs@isc.org>
Date: Wed, 26 Nov 2025 08:31:45 +0100
Subject: [PATCH 7/8] Update misleading comments in multisigner test

We are not actually retrieving these records from the other provider,
they are available as key files to us and we are using those files
to send a dynamic update to the server.

(cherry picked from commit 11578aa2199d665ab70cadddccbc82fd5ce2f26a)
---
 bin/tests/system/multisigner/tests_multisigner.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/tests/system/multisigner/tests_multisigner.py b/bin/tests/system/multisigner/tests_multisigner.py
index 80288aee6d..69f644d40d 100644
--- a/bin/tests/system/multisigner/tests_multisigner.py
+++ b/bin/tests/system/multisigner/tests_multisigner.py
@@ -284,7 +284,7 @@ def check_add_cdnskey(server, zone, keys, expected, extra_keys, extra, primary=N
         f"- zone {zone} {primary.identifier}: update zone with CDNSKEY from other providers"
     )
 
-    # Retrieve CDNSKEY records from the other providers.
+    # Update the server with the CDNSKEY record from the other providers.
     update_msg = dns.update.UpdateMessage(zone)
     for ksk in extra_keys:
         dnskey = ksk.dnskey().split()
@@ -400,7 +400,7 @@ def check_add_cds(server, zone, keys, expected, extra_keys, extra, primary=None)
         f"- zone {zone} {primary.identifier}: update zone with CDS from other providers"
     )
 
-    # Retrieve CDS records from the other providers.
+    # Update the server with the CDS record from the other providers.
     update_msg = dns.update.UpdateMessage(zone)
     for ksk in extra_keys:
         ds = dsfromkey(ksk)

From d5ab331d8a41b81fcd0f6d938cd1645d3f305fc9 Mon Sep 17 00:00:00 2001
From: Matthijs Mekking <matthijs@isc.org>
Date: Fri, 28 Nov 2025 09:05:21 +0100
Subject: [PATCH 8/8] dnskey is now a property of Key class

After a rebase, the dnskey() invocations need to be adjusted to
accomodate for !11201 (0bf20f8d).

(cherry picked from commit a91f13cae81b2381f0f0b7e3c6d6a1c2e58a70e8)
---
 bin/tests/system/multisigner/tests_multisigner.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/bin/tests/system/multisigner/tests_multisigner.py b/bin/tests/system/multisigner/tests_multisigner.py
index 69f644d40d..695f2420b9 100644
--- a/bin/tests/system/multisigner/tests_multisigner.py
+++ b/bin/tests/system/multisigner/tests_multisigner.py
@@ -164,7 +164,7 @@ def check_add_zsk(server, zone, keys, expected, extra_keys, extra, primary=None)
 
     update_msg = dns.update.UpdateMessage(zone)
     for zsk in extra_keys:
-        dnskey = zsk.dnskey().split()
+        dnskey = str(zsk.dnskey).split()
         rdata = " ".join(dnskey[4:])
         update_msg.add(f"{zone}.", TTL, "DNSKEY", rdata)
     primary.nsupdate(update_msg)
@@ -205,7 +205,7 @@ def _check_remove_zsk_fail(
     )
 
     zsks = [k for k in keys if not k.is_ksk()]
-    dnskey = zsks[0].dnskey().split()
+    dnskey = str(zsks[0].dnskey).split()
     rdata = " ".join(dnskey[4:])
     update_msg = dns.update.UpdateMessage(zone)
     update_msg.delete(f"{zone}.", "DNSKEY", rdata)
@@ -251,7 +251,7 @@ def check_remove_zsk(
 
     update_msg = dns.update.UpdateMessage(zone)
     for zsk in extra_keys:
-        dnskey = zsk.dnskey().split()
+        dnskey = str(zsk.dnskey).split()
         rdata = " ".join(dnskey[4:])
         update_msg.delete(f"{zone}.", "DNSKEY", rdata)
     primary.nsupdate(update_msg)
@@ -287,7 +287,7 @@ def check_add_cdnskey(server, zone, keys, expected, extra_keys, extra, primary=N
     # Update the server with the CDNSKEY record from the other providers.
     update_msg = dns.update.UpdateMessage(zone)
     for ksk in extra_keys:
-        dnskey = ksk.dnskey().split()
+        dnskey = str(ksk.dnskey).split()
         rdata = " ".join(dnskey[4:])
         update_msg.add(f"{zone}.", TTL, "CDNSKEY", rdata)
     primary.nsupdate(update_msg)
@@ -321,7 +321,7 @@ def _check_remove_cdnskey_fail(
     )
 
     ksks = [k for k in keys if not k.is_ksk()]
-    dnskey = ksks[0].dnskey().split()
+    dnskey = str(ksks[0].dnskey).split()
     rdata = " ".join(dnskey[4:])
     update_msg = dns.update.UpdateMessage(zone)
     update_msg.delete(f"{zone}.", "CDNSKEY", rdata)
@@ -367,7 +367,7 @@ def check_remove_cdnskey(
 
     update_msg = dns.update.UpdateMessage(zone)
     for ksk in extra_keys:
-        dnskey = ksk.dnskey().split()
+        dnskey = str(ksk.dnskey).split()
         rdata = " ".join(dnskey[4:])
         update_msg.delete(f"{zone}.", "CDNSKEY", rdata)
     primary.nsupdate(update_msg)