From 04ae34cff6d112eb84916d725cbd6f1f5d33b119 Mon Sep 17 00:00:00 2001
From: Matthijs Mekking <matthijs@isc.org>
Date: Wed, 3 Jul 2024 11:32:11 +0200
Subject: [PATCH 1/7] Add test for missing key files, don't roll

In this specific case the key files are temporary unavailable, for
example because of an operator error, or a mount failure). In such
cases, BIND should not try to roll over these keys.

(cherry picked from commit a3afbd9d6f12c6a331343e91352690d0f665caf4)
---
 bin/tests/system/kasp/ns3/named-fips.conf.in |  9 +++
 bin/tests/system/kasp/ns3/setup.sh           | 16 +++++
 bin/tests/system/kasp/tests.sh               | 64 +++++++++++++++++++-
 3 files changed, 88 insertions(+), 1 deletion(-)

diff --git a/bin/tests/system/kasp/ns3/named-fips.conf.in b/bin/tests/system/kasp/ns3/named-fips.conf.in
index 793cdb5d95..33cfaa9a13 100644
--- a/bin/tests/system/kasp/ns3/named-fips.conf.in
+++ b/bin/tests/system/kasp/ns3/named-fips.conf.in
@@ -314,6 +314,15 @@ zone "unfresh-sigs.autosign" {
 	dnssec-policy "autosign";
 };
 
+/*
+ * Zone that has missing key files.
+ */
+zone "keyfiles-missing.autosign" {
+	type primary;
+	file "keyfiles-missing.autosign.db";
+	dnssec-policy "autosign";
+};
+
 /*
  * Zone that has missing private KSK.
  */
diff --git a/bin/tests/system/kasp/ns3/setup.sh b/bin/tests/system/kasp/ns3/setup.sh
index 40299c78be..4c66bdc2b2 100644
--- a/bin/tests/system/kasp/ns3/setup.sh
+++ b/bin/tests/system/kasp/ns3/setup.sh
@@ -286,6 +286,22 @@ echo "ZSK: yes" >>"${ZSK}".state
 echo "Lifetime: 31536000" >>"${ZSK}".state # PT1Y
 rm -f "${ZSK}".private
 
+# These signatures are still good, but the key files will be removed
+# before a second run of reconfiguring keys.
+setup keyfiles-missing.autosign
+T="now-6mo"
+ksktimes="-P $T -A $T -P sync $T"
+zsktimes="-P $T -A $T"
+KSK=$($KEYGEN -a $DEFAULT_ALGORITHM -L 300 -f KSK $ksktimes $zone 2>keygen.out.$zone.1)
+ZSK=$($KEYGEN -a $DEFAULT_ALGORITHM -L 300 $zsktimes $zone 2>keygen.out.$zone.2)
+$SETTIME -s -g $O -d $O $T -k $O $T -r $O $T "$KSK" >settime.out.$zone.1 2>&1
+$SETTIME -s -g $O -k $O $T -z $O $T "$ZSK" >settime.out.$zone.2 2>&1
+cat template.db.in "${KSK}.key" "${ZSK}.key" >"$infile"
+private_type_record $zone $DEFAULT_ALGORITHM_NUMBER "$KSK" >>"$infile"
+private_type_record $zone $DEFAULT_ALGORITHM_NUMBER "$ZSK" >>"$infile"
+cp $infile $zonefile
+$SIGNER -S -x -s now-1w -e now+1w -o $zone -O raw -f "${zonefile}.signed" $infile >signer.out.$zone.1 2>&1
+
 # These signatures are already expired, and the private ZSK is retired.
 setup zsk-retired.autosign
 T="now-6mo"
diff --git a/bin/tests/system/kasp/tests.sh b/bin/tests/system/kasp/tests.sh
index f2479e6368..0f55c195a9 100644
--- a/bin/tests/system/kasp/tests.sh
+++ b/bin/tests/system/kasp/tests.sh
@@ -385,7 +385,7 @@ echo_i "test that if private key files are inaccessible this doesn't trigger a r
 basefile=$(key_get KEY1 BASEFILE)
 mv "${basefile}.private" "${basefile}.offline"
 rndccmd 10.53.0.3 loadkeys "$ZONE" >/dev/null || log_error "rndc loadkeys zone ${ZONE} failed"
-wait_for_log 3 "offline, policy default" $DIR/named.run || ret=1
+wait_for_log 3 "zone $ZONE/IN (signed): zone_rekey:verify keys failed: some key files are missing" $DIR/named.run || ret=1
 mv "${basefile}.offline" "${basefile}.private"
 test "$ret" -eq 0 || echo_i "failed"
 status=$((status + ret))
@@ -1743,6 +1743,68 @@ check_apex
 check_subdomain
 dnssec_verify
 
+#
+# Zone: keyfiles-missing.autosign.
+#
+set_zone "keyfiles-missing.autosign"
+set_policy "autosign" "2" "300"
+set_server "ns3" "10.53.0.3"
+# Key properties.
+key_clear "KEY1"
+set_keyrole "KEY1" "ksk"
+set_keylifetime "KEY1" "63072000"
+set_keyalgorithm "KEY1" "$DEFAULT_ALGORITHM_NUMBER" "$DEFAULT_ALGORITHM" "$DEFAULT_BITS"
+set_keysigning "KEY1" "yes"
+set_zonesigning "KEY1" "no"
+
+key_clear "KEY2"
+set_keyrole "KEY2" "zsk"
+set_keylifetime "KEY2" "31536000"
+set_keyalgorithm "KEY2" "$DEFAULT_ALGORITHM_NUMBER" "$DEFAULT_ALGORITHM" "$DEFAULT_BITS"
+set_keysigning "KEY2" "no"
+set_zonesigning "KEY2" "yes"
+
+# Both KSK and ZSK stay OMNIPRESENT.
+set_keystate "KEY1" "GOAL" "omnipresent"
+set_keystate "KEY1" "STATE_DNSKEY" "omnipresent"
+set_keystate "KEY1" "STATE_KRRSIG" "omnipresent"
+set_keystate "KEY1" "STATE_DS" "omnipresent"
+
+set_keystate "KEY2" "GOAL" "omnipresent"
+set_keystate "KEY2" "STATE_DNSKEY" "omnipresent"
+set_keystate "KEY2" "STATE_ZRRSIG" "omnipresent"
+
+check_keys
+check_dnssecstatus "$SERVER" "$POLICY" "$ZONE"
+set_keytimes_autosign_policy
+check_keytimes
+check_apex
+check_subdomain
+dnssec_verify
+# All good, now remove key files and reload keys.
+rm_keyfiles() {
+  _basefile=$(key_get "$1" BASEFILE)
+  echo_i "remove key files $_basefile"
+  _keyfile="${_basefile}.key"
+  _privatefile="${_basefile}.private"
+  _statefile="${_basefile}.state"
+  rm -f $_keyfile
+  rm -f $_privatefile
+  rm -f $_statefile
+}
+rm_keyfiles "KEY1"
+rm_keyfiles "KEY2"
+
+rndccmd 10.53.0.3 loadkeys "$ZONE" >/dev/null || log_error "rndc loadkeys zone ${ZONE} failed"
+wait_for_log 3 "zone $ZONE/IN (signed): zone_rekey:verify keys failed: some key files are missing" $DIR/named.run || ret=1
+# Check keys again, make sure no new keys are created.
+set_policy "autosign" "0" "300"
+key_clear "KEY1"
+key_clear "KEY2"
+check_keys
+# Zone is still signed correctly.
+dnssec_verify
+
 #
 # Test dnssec-policy inheritance.
 #

From 2494275a8e42f4928e21c0c333a8b9ef94990eec Mon Sep 17 00:00:00 2001
From: Matthijs Mekking <matthijs@isc.org>
Date: Thu, 15 Aug 2024 10:36:15 +0200
Subject: [PATCH 2/7] Verify new key files before running keymgr

Prior to running the keymgr, first make sure that existing keys
are present in the new keylist. If not, treat this as an operational
error where the keys are made offline (temporarily), possibly unwanted.

(cherry picked from commit 5fdad05a8a56323a24ad8939b9eef5142388331f)
---
 bin/tests/system/kasp/tests.sh | 13 +++++-
 lib/dns/zone.c                 | 76 ++++++++++++++++++++++++++++++----
 2 files changed, 80 insertions(+), 9 deletions(-)

diff --git a/bin/tests/system/kasp/tests.sh b/bin/tests/system/kasp/tests.sh
index 0f55c195a9..15167951f3 100644
--- a/bin/tests/system/kasp/tests.sh
+++ b/bin/tests/system/kasp/tests.sh
@@ -385,7 +385,7 @@ echo_i "test that if private key files are inaccessible this doesn't trigger a r
 basefile=$(key_get KEY1 BASEFILE)
 mv "${basefile}.private" "${basefile}.offline"
 rndccmd 10.53.0.3 loadkeys "$ZONE" >/dev/null || log_error "rndc loadkeys zone ${ZONE} failed"
-wait_for_log 3 "zone $ZONE/IN (signed): zone_rekey:verify keys failed: some key files are missing" $DIR/named.run || ret=1
+wait_for_log 3 "zone $ZONE/IN (signed): zone_rekey:zone_verifykeys failed: some key files are missing" $DIR/named.run || ret=1
 mv "${basefile}.offline" "${basefile}.private"
 test "$ret" -eq 0 || echo_i "failed"
 status=$((status + ret))
@@ -1647,6 +1647,15 @@ check_subdomain
 dnssec_verify
 check_rrsig_refresh
 
+# Load again, make sure the purged key is not an issue when verifying keys.
+echo_i "load keys for $ZONE, making sure a recently purged key is not an issue when verifying keys ($n)"
+ret=0
+rndccmd 10.53.0.3 loadkeys "$ZONE" >/dev/null || log_error "rndc loadkeys zone ${ZONE} failed"
+wait_for_log 3 "keymgr: $ZONE done" $DIR/named.run
+grep "zone $ZONE/IN (signed): zone_rekey:zone_verifykeys failed: some key files are missing" $DIR/named.run && ret=1
+test "$ret" -eq 0 || echo_i "failed"
+status=$((status + ret))
+
 #
 # Zone: legacy-keys.kasp.
 #
@@ -1796,7 +1805,7 @@ rm_keyfiles "KEY1"
 rm_keyfiles "KEY2"
 
 rndccmd 10.53.0.3 loadkeys "$ZONE" >/dev/null || log_error "rndc loadkeys zone ${ZONE} failed"
-wait_for_log 3 "zone $ZONE/IN (signed): zone_rekey:verify keys failed: some key files are missing" $DIR/named.run || ret=1
+wait_for_log 3 "zone $ZONE/IN (signed): zone_rekey:zone_verifykeys failed: some key files are missing" $DIR/named.run || ret=1
 # Check keys again, make sure no new keys are created.
 set_policy "autosign" "0" "300"
 key_clear "KEY1"
diff --git a/lib/dns/zone.c b/lib/dns/zone.c
index 43c97188de..7cb0db7211 100644
--- a/lib/dns/zone.c
+++ b/lib/dns/zone.c
@@ -371,6 +371,7 @@ struct dns_zone {
 	dns_view_t *prev_view;
 	dns_kasp_t *kasp;
 	dns_kasp_t *defaultkasp;
+	dns_dnsseckeylist_t keyring;
 	dns_checkmxfunc_t checkmx;
 	dns_checksrvfunc_t checksrv;
 	dns_checknsfunc_t checkns;
@@ -1179,6 +1180,7 @@ dns_zone_create(dns_zone_t **zonep, isc_mem_t *mctx, unsigned int tid) {
 	zone->parentals = r;
 	zone->notify = r;
 	zone->defaultkasp = NULL;
+	ISC_LIST_INIT(zone->keyring);
 
 	isc_stats_create(mctx, &zone->gluecachestats,
 			 dns_gluecachestatscounter_max);
@@ -1281,6 +1283,9 @@ zone_free(dns_zone_t *zone) {
 	if (zone->defaultkasp != NULL) {
 		dns_kasp_detach(&zone->defaultkasp);
 	}
+	if (!ISC_LIST_EMPTY(zone->keyring)) {
+		clear_keylist(&zone->keyring, zone->mctx);
+	}
 	if (!ISC_LIST_EMPTY(zone->checkds_ok)) {
 		clear_keylist(&zone->checkds_ok, zone->mctx);
 	}
@@ -22006,6 +22011,47 @@ update_ttl(dns_rdataset_t *rdataset, dns_name_t *name, dns_ttl_t ttl,
 	return (ISC_R_SUCCESS);
 }
 
+static isc_result_t
+zone_verifykeys(dns_zone_t *zone, dns_dnsseckeylist_t *newkeys) {
+	dns_dnsseckey_t *key1, *key2, *next;
+
+	/*
+	 * Make sure that the existing keys are also present in the new keylist.
+	 */
+	for (key1 = ISC_LIST_HEAD(zone->keyring); key1 != NULL; key1 = next) {
+		bool found = false;
+		next = ISC_LIST_NEXT(key1, link);
+
+		if (dst_key_is_unused(key1->key)) {
+			continue;
+		}
+		if (key1->purge) {
+			continue;
+		}
+
+		for (key2 = ISC_LIST_HEAD(*newkeys); key2 != NULL;
+		     key2 = ISC_LIST_NEXT(key2, link))
+		{
+			if (dst_key_compare(key1->key, key2->key)) {
+				found = true;
+				break;
+			}
+		}
+
+		if (!found) {
+			char keystr[DST_KEY_FORMATSIZE];
+			dst_key_format(key1->key, keystr, sizeof(keystr));
+			dnssec_log(zone, ISC_LOG_DEBUG(1),
+				   "verifykeys: key %s - not available",
+				   keystr);
+			return (ISC_R_NOTFOUND);
+		}
+	}
+
+	/* All good. */
+	return (ISC_R_SUCCESS);
+}
+
 static void
 remove_rdataset(dns_zone_t *zone, dns_diff_t *diff, dns_rdataset_t *rdataset) {
 	if (!dns_rdataset_isassociated(rdataset)) {
@@ -22294,6 +22340,16 @@ zone_rekey(dns_zone_t *zone) {
 	}
 
 	if (kasp != NULL && !offlineksk) {
+		/* Verify new keys. */
+		isc_result_t ret = zone_verifykeys(zone, &keys);
+		if (ret != ISC_R_SUCCESS) {
+			dnssec_log(zone, ISC_LOG_ERROR,
+				   "zone_rekey:zone_verifykeys failed: "
+				   "some key files are missing");
+			KASP_UNLOCK(kasp);
+			goto failure;
+		}
+
 		/*
 		 * Check DS at parental agents. Clear ongoing checks.
 		 */
@@ -22303,8 +22359,8 @@ zone_rekey(dns_zone_t *zone) {
 		ISC_LIST_INIT(zone->checkds_ok);
 		UNLOCK_ZONE(zone);
 
-		isc_result_t ret = dns_zone_getdnsseckeys(zone, db, ver, now,
-							  &zone->checkds_ok);
+		ret = dns_zone_getdnsseckeys(zone, db, ver, now,
+					     &zone->checkds_ok);
 		if (ret == ISC_R_SUCCESS) {
 			zone_checkds(zone);
 		} else {
@@ -22316,7 +22372,7 @@ zone_rekey(dns_zone_t *zone) {
 				   isc_result_totext(ret));
 		}
 
-		/* Run keymgr */
+		/* Run keymgr. */
 		if (result == ISC_R_SUCCESS || result == ISC_R_NOTFOUND) {
 			dns_zone_lock_keyfiles(zone);
 			result = dns_keymgr_run(&zone->origin, zone->rdclass,
@@ -22787,10 +22843,14 @@ zone_rekey(dns_zone_t *zone) {
 	}
 	UNLOCK_ZONE(zone);
 
-	if (isc_log_wouldlog(dns_lctx, ISC_LOG_DEBUG(3))) {
-		for (key = ISC_LIST_HEAD(dnskeys); key != NULL;
-		     key = ISC_LIST_NEXT(key, link))
-		{
+	/*
+	 * Remember which keys have been used.
+	 */
+	if (!ISC_LIST_EMPTY(zone->keyring)) {
+		clear_keylist(&zone->keyring, zone->mctx);
+	}
+	while ((key = ISC_LIST_HEAD(dnskeys)) != NULL) {
+		if (isc_log_wouldlog(dns_lctx, ISC_LOG_DEBUG(3))) {
 			/* This debug log is used in the kasp system test */
 			char algbuf[DNS_SECALG_FORMATSIZE];
 			dns_secalg_format(dst_key_alg(key->key), algbuf,
@@ -22799,6 +22859,8 @@ zone_rekey(dns_zone_t *zone) {
 				   "zone_rekey done: key %d/%s",
 				   dst_key_id(key->key), algbuf);
 		}
+		ISC_LIST_UNLINK(dnskeys, key, link);
+		ISC_LIST_APPEND(zone->keyring, key, link);
 	}
 
 	result = ISC_R_SUCCESS;

From 24e8e4294a72fb2cf108851ea361ea8895d8c4e8 Mon Sep 17 00:00:00 2001
From: Matthijs Mekking <matthijs@isc.org>
Date: Wed, 14 Aug 2024 14:38:22 +0200
Subject: [PATCH 3/7] Fix some system test cases

Some test cases were working but for the wrong reasons. These started
to fail when I implemented the first approach for #4763, where the
existence of a DNSKEY together with an empty keyring is suspicious and
would prevent the keymgr from running.

These are:

1. kasp: The multisigner-model2.kasp zone has ZSKs from other providers
   in the zone, but not yet its own keys. Pregenerate signing keys and
   add them to the unsigned zone as well.

2. kasp: The dynamic-signed-inline-signing.kasp zone has a key generated
   and added in the raw version of the zone. But the key file is stored
   outside the key-directory for the given zone. Add '-K keys' to the
   dnssec-keygen command.

(cherry picked from commit d1e263ef131f4f98249fdef0e84917199ef7ce12)
---
 bin/tests/system/kasp/ns3/setup.sh | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/bin/tests/system/kasp/ns3/setup.sh b/bin/tests/system/kasp/ns3/setup.sh
index 4c66bdc2b2..b136d572d7 100644
--- a/bin/tests/system/kasp/ns3/setup.sh
+++ b/bin/tests/system/kasp/ns3/setup.sh
@@ -129,15 +129,19 @@ $KEYGEN -G -k rsasha256 -l policies/kasp.conf $zone >keygen.out.$zone.2 2>&1
 
 zone="multisigner-model2.kasp"
 echo_i "setting up zone: $zone"
+KSK=$($KEYGEN -a $DEFAULT_ALGORITHM -f KSK -L 3600 -M 32768:65535 $zone 2>keygen.out.$zone.1)
+ZSK=$($KEYGEN -a $DEFAULT_ALGORITHM -L 3600 $zone -M 32768:65535 2>keygen.out.$zone.2)
+cat "${KSK}.key" | grep -v ";.*" >>"${zone}.db"
+cat "${ZSK}.key" | grep -v ";.*" >>"${zone}.db"
 # Import the ZSK sets of the other providers into their DNSKEY RRset.
-ZSK1=$($KEYGEN -K ../ -a $DEFAULT_ALGORITHM -L 3600 -M 0:32767 $zone 2>keygen.out.$zone.1)
-ZSK2=$($KEYGEN -K ../ -a $DEFAULT_ALGORITHM -L 3600 -M 0:32767 $zone 2>keygen.out.$zone.2)
-# ZSK1 will be added to the unsigned zonefile.
+# ZSK1 is from a different provider and is added to the unsigned zonefile.
+# ZSK2 is also from a different provider and is added with a Dynamic Update.
+ZSK1=$($KEYGEN -K ../ -a $DEFAULT_ALGORITHM -L 3600 -M 0:32767 $zone 2>keygen.out.$zone.3)
+ZSK2=$($KEYGEN -K ../ -a $DEFAULT_ALGORITHM -L 3600 -M 0:32767 $zone 2>keygen.out.$zone.4)
 cat "../${ZSK1}.key" | grep -v ";.*" >>"${zone}.db"
 cat "../${ZSK1}.key" | grep -v ";.*" >"${zone}.zsk1"
-rm -f "../${ZSK1}.*"
-# ZSK2 will be used with a Dynamic Update.
 cat "../${ZSK2}.key" | grep -v ";.*" >"${zone}.zsk2"
+rm -f "../${ZSK1}.*"
 rm -f "../${ZSK2}.*"
 
 zone="rumoured.kasp"
@@ -178,11 +182,12 @@ $SIGNER -PS -x -o $zone -O raw -f "${zonefile}.signed" $infile >signer.out.$zone
 setup dynamic-signed-inline-signing.kasp
 T="now-1d"
 csktimes="-P $T -A $T -P sync $T"
-CSK=$($KEYGEN -a $DEFAULT_ALGORITHM -L 3600 -f KSK $csktimes $zone 2>keygen.out.$zone.1)
-$SETTIME -s -g $O -d $O $T -k $O $T -z $O $T -r $O $T "$CSK" >settime.out.$zone.1 2>&1
-cat template.db.in "${CSK}.key" >"$infile"
+CSK=$($KEYGEN -K keys -a $DEFAULT_ALGORITHM -L 3600 -f KSK $csktimes $zone 2>keygen.out.$zone.1)
+$SETTIME -s -g $O -d $O $T -k $O $T -z $O $T -r $O $T "keys/$CSK" >settime.out.$zone.1 2>&1
+cat template.db.in "keys/${CSK}.key" >"$infile"
+private_type_record $zone $DEFAULT_ALGORITHM_NUMBER "keys/$CSK" >>"$infile"
 cp $infile $zonefile
-$SIGNER -PS -z -x -s now-2w -e now-1mi -o $zone -f "${zonefile}.signed" $infile >signer.out.$zone.1 2>&1
+$SIGNER -PS -K keys -z -x -s now-2w -e now-1mi -o $zone -f "${zonefile}.signed" $infile >signer.out.$zone.1 2>&1
 
 # We are changing an existing single-signed zone to multi-signed
 # zone where the key tags do not match the dnssec-policy key tag range

From 80d5b3877aaccdfdc8ee3af309cc6403e83eeb57 Mon Sep 17 00:00:00 2001
From: Matthijs Mekking <matthijs@isc.org>
Date: Mon, 19 Aug 2024 09:46:56 +0200
Subject: [PATCH 4/7] Add additional test case with purged key

Test that if a key to be purged is in the keyring, it does not
prevent the keymgr from running. Normally a key that is in the keyring
should be available again on the next run, but that is not true for
a key that can be purged.

In addition, fix some wait_for_log calls, by adding the missing
'|| ret=1' parts.

(cherry picked from commit a2317425bcd511477de21140a7182f494b5080e5)
---
 bin/tests/system/kasp/ns3/setup.sh |  6 ++++++
 bin/tests/system/kasp/tests.sh     | 10 +++++-----
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/bin/tests/system/kasp/ns3/setup.sh b/bin/tests/system/kasp/ns3/setup.sh
index b136d572d7..85a88f5856 100644
--- a/bin/tests/system/kasp/ns3/setup.sh
+++ b/bin/tests/system/kasp/ns3/setup.sh
@@ -322,6 +322,12 @@ private_type_record $zone $DEFAULT_ALGORITHM_NUMBER "$ZSK" >>"$infile"
 cp $infile $zonefile
 $SIGNER -PS -x -s now-2w -e now-1mi -o $zone -O raw -f "${zonefile}.signed" $infile >signer.out.$zone.1 2>&1
 $SETTIME -s -g HIDDEN "$ZSK" >settime.out.$zone.3 2>&1
+# An old key that is being purged should not prevent keymgr to be run.
+T1="now-1y"
+T2="now-2y"
+oldtimes="-P $T2 -A $T2 -I $T1 -D $T1"
+OLD=$($KEYGEN -a $DEFAULT_ALGORITHM -L 300 $oldtimes $zone 2>keygen.out.$zone.3)
+$SETTIME -s -g $H -k $H $T1 -z $H $T1 "$OLD" >settime.out.$zone.3 2>&1
 
 #
 # The zones at enable-dnssec.autosign represent the various steps of the
diff --git a/bin/tests/system/kasp/tests.sh b/bin/tests/system/kasp/tests.sh
index 15167951f3..b114eabcf3 100644
--- a/bin/tests/system/kasp/tests.sh
+++ b/bin/tests/system/kasp/tests.sh
@@ -318,7 +318,7 @@ state_stat=$(key_get KEY1 STATE_STAT)
 
 nextpart $DIR/named.run >/dev/null
 rndccmd 10.53.0.3 loadkeys "$ZONE" >/dev/null || log_error "rndc loadkeys zone ${ZONE} failed"
-wait_for_log 3 "keymgr: $ZONE done" $DIR/named.run
+wait_for_log 3 "keymgr: $ZONE done" $DIR/named.run || ret=1
 privkey_stat2=$(key_stat "${basefile}.private")
 pubkey_stat2=$(key_stat "${basefile}.key")
 state_stat2=$(key_stat "${basefile}.state")
@@ -334,7 +334,7 @@ ret=0
 
 nextpart $DIR/named.run >/dev/null
 rndccmd 10.53.0.3 loadkeys "$ZONE" >/dev/null || log_error "rndc loadkeys zone ${ZONE} failed"
-wait_for_log 3 "keymgr: $ZONE done" $DIR/named.run
+wait_for_log 3 "keymgr: $ZONE done" $DIR/named.run || ret=1
 privkey_stat2=$(key_stat "${basefile}.private")
 pubkey_stat2=$(key_stat "${basefile}.key")
 state_stat2=$(key_stat "${basefile}.state")
@@ -1651,7 +1651,7 @@ check_rrsig_refresh
 echo_i "load keys for $ZONE, making sure a recently purged key is not an issue when verifying keys ($n)"
 ret=0
 rndccmd 10.53.0.3 loadkeys "$ZONE" >/dev/null || log_error "rndc loadkeys zone ${ZONE} failed"
-wait_for_log 3 "keymgr: $ZONE done" $DIR/named.run
+wait_for_log 3 "keymgr: $ZONE done" $DIR/named.run || ret=1
 grep "zone $ZONE/IN (signed): zone_rekey:zone_verifykeys failed: some key files are missing" $DIR/named.run && ret=1
 test "$ret" -eq 0 || echo_i "failed"
 status=$((status + ret))
@@ -5270,7 +5270,7 @@ dig_with_opts @10.53.0.6 example SOA >dig.out.ns6.test$n.soa1 || ret=1
 cp ns6/example2.db.in ns6/example.db || ret=1
 nextpart ns6/named.run >/dev/null
 rndccmd 10.53.0.6 reload || ret=1
-wait_for_log 3 "all zones loaded" ns6/named.run
+wait_for_log 3 "all zones loaded" ns6/named.run || ret=1
 # Check that the SOA SERIAL increases and check the TTLs (should be 300 as
 # defined in ns6/example2.db.in).
 retry_quiet 10 _check_soa_ttl 300 300 || ret=1
@@ -5288,7 +5288,7 @@ cp ns6/example3.db.in ns6/example.db || ret=1
 rm ns6/example.db.jnl
 nextpart ns6/named.run >/dev/null
 start_server --noclean --restart --port ${PORT} ns6
-wait_for_log 3 "all zones loaded" ns6/named.run
+wait_for_log 3 "all zones loaded" ns6/named.run || ret=1
 # Check that the SOA SERIAL increases and check the TTLs (should be changed
 # from 300 to 400 as defined in ns6/example3.db.in).
 retry_quiet 10 _check_soa_ttl 300 400 || ret=1

From 7aaa36f09f066a8aaf6bf8d7e265dae96e5badcf Mon Sep 17 00:00:00 2001
From: Matthijs Mekking <matthijs@isc.org>
Date: Mon, 19 Aug 2024 09:49:21 +0200
Subject: [PATCH 5/7] Small keymgr improvement

When a key is to be purged, don't run the key state machinery for it.

(cherry picked from commit af54e3dadc87072cd21362779a11616fb85bd797)
---
 lib/dns/keymgr.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/dns/keymgr.c b/lib/dns/keymgr.c
index 79b6880b1a..39e81e0d3d 100644
--- a/lib/dns/keymgr.c
+++ b/lib/dns/keymgr.c
@@ -1466,6 +1466,11 @@ transition:
 		char keystr[DST_KEY_FORMATSIZE];
 		dst_key_format(dkey->key, keystr, sizeof(keystr));
 
+		if (dkey->purge) {
+			/* Skip purged keys. */
+			continue;
+		}
+
 		/* For all records related to this key. */
 		for (int i = 0; i < NUM_KEYSTATES; i++) {
 			isc_result_t ret;

From 25f39228e00a2e8018dc467fba73ef756477dee2 Mon Sep 17 00:00:00 2001
From: Matthijs Mekking <matthijs@isc.org>
Date: Tue, 20 Aug 2024 14:38:24 +0200
Subject: [PATCH 6/7] Test removing DNSKEYs from other providers

In a multi-signer setup, removing DNSKEY records from the zone should
not be treated as a key that previously exists in the keyring, thus
blocking the keymgr. Add a test case to make sure.

(cherry picked from commit 5f552293d72dbfb150d6e4ab5ff660996f51baf1)
---
 bin/tests/system/kasp/tests.sh | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/bin/tests/system/kasp/tests.sh b/bin/tests/system/kasp/tests.sh
index b114eabcf3..2771c1ffdf 100644
--- a/bin/tests/system/kasp/tests.sh
+++ b/bin/tests/system/kasp/tests.sh
@@ -2201,16 +2201,23 @@ check_apex
 check_subdomain
 dnssec_verify
 
-# Check that the ZSKs from the other provider are published.
+# Check that the ZSKs from the other providers are published.
 zsks_are_published() {
+  num=$1
   dig_with_opts +short "$ZONE" "@${SERVER}" DNSKEY >"dig.out.$DIR.test$n" || return 1
   # We should have three ZSKs.
   lines=$(grep "256 3 13" dig.out.$DIR.test$n | wc -l)
-  test "$lines" -eq 3 || return 1
+  test "$lines" -eq $num || return 1
   # And one KSK.
   lines=$(grep "257 3 13" dig.out.$DIR.test$n | wc -l)
   test "$lines" -eq 1 || return 1
 }
+n=$((n + 1))
+echo_i "check initial number of ZSKs (one from us and one from another provider) for zone ${ZONE} ($n)"
+ret=0
+retry_quiet 10 zsks_are_published 2 || ret=1
+test "$ret" -eq 0 || echo_i "failed"
+status=$((status + ret))
 
 n=$((n + 1))
 echo_i "update zone with ZSK from another provider for zone ${ZONE} ($n)"
@@ -2221,7 +2228,21 @@ ret=0
   echo update add $(cat "${DIR}/${ZONE}.zsk2")
   echo send
 ) | $NSUPDATE
-retry_quiet 10 zsks_are_published || ret=1
+retry_quiet 10 zsks_are_published 3 || ret=1
+test "$ret" -eq 0 || echo_i "failed"
+status=$((status + ret))
+
+n=$((n + 1))
+echo_i "remove ZSKs from the other providers for zone ${ZONE} ($n)"
+ret=0
+(
+  echo zone ${ZONE}
+  echo server 10.53.0.3 "$PORT"
+  echo update del $(cat "${DIR}/${ZONE}.zsk1")
+  echo update del $(cat "${DIR}/${ZONE}.zsk2")
+  echo send
+) | $NSUPDATE
+retry_quiet 10 zsks_are_published 1 || ret=1
 test "$ret" -eq 0 || echo_i "failed"
 status=$((status + ret))
 

From a71e037ac4ecf8ba213c87c2a545600fa1daaf05 Mon Sep 17 00:00:00 2001
From: Matthijs Mekking <matthijs@isc.org>
Date: Fri, 11 Oct 2024 14:38:55 +0200
Subject: [PATCH 7/7] Add new behavior to the ARM

Add text to the ARM that describes what we do in case key files have
become unavailable.

(cherry picked from commit 351c066d916b0ac79070ee0c8e9879d108dfb996)
---
 doc/arm/reference.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/doc/arm/reference.rst b/doc/arm/reference.rst
index 88a44045ba..c2a6039d7f 100644
--- a/doc/arm/reference.rst
+++ b/doc/arm/reference.rst
@@ -6394,6 +6394,14 @@ zone is generated even if they have the same policy.  If multiple views
 are configured with different versions of the same zone, each separate
 version uses the same set of signing keys.
 
+If the expected key files that were previously observed have gone missing or
+are inaccessible, key management is halted. This will prevent rollovers
+from being started if there is a temporary file access issue. If his problem
+is permanent it will eventually lead to expired signatures in your zone.
+Note that if the key files are missing or inaccessible during :iscman:`named`
+startup, BIND 9 will try to generate new keys according to the DNSSEC policy,
+because it has no cached information about existing keys yet.
+
 The :any:`dnssec-policy` statement requires dynamic DNS to be set up, or
 :any:`inline-signing` to be enabled (which is the default for DNSSEC zones).