fix: usr: Skip unsupported algorithms when looking for signing key

A mix of supported and unsupported DNSSEC algorithms in the same zone could have caused validation failures. Ignore the DNSSEC keys with unsupported algorithm when looking for the signing keys.

Closes #5622

Merge branch '5622-dont-fail-on-unsupported-algorithms' into 'main'

See merge request isc-projects/bind9!11208

bin/tests/system/dnssec/ns2/example.db.in

@@ -202,3 +202,7 @@ ns3.extradsunknownoid	A	10.53.0.3
 
 extended-ds-unknown-oid	NS	ns3.extended-ds-unknown-oid
 ns3.extended-ds-unknown-oid A	10.53.0.3
+
+; A secure subdomain with extra bad key
+extrabadkey		NS	ns3.extrabadkey
+ns3.extrabadkey		A	10.53.0.3

bin/tests/system/dnssec/ns2/sign.sh

@@ -92,7 +92,7 @@ for subdomain in digest-alg-unsupported ds-unsupported secure badds \
   dnskey-nsec3-unknown managed-future future revkey \
   dname-at-apex-nsec3 occluded rsasha1 rsasha1-1024 \
   rsasha256oid rsasha512oid unknownoid extradsoid extradsunknownoid \
-  extended-ds-unknown-oid; do
+  extended-ds-unknown-oid extrabadkey; do
   cp "../ns3/dsset-$subdomain.example." .
 done
 

bin/tests/system/dnssec/ns3/named.conf.j2

@@ -141,6 +141,12 @@ zone "extrakey.example" {
 	allow-update { any; };
 };
 
+zone "extrabadkey.example" {
+	type primary;
+	file "extrabadkey.example.db.signed";
+	allow-update { any; };
+};
+
 zone "insecure.nsec3.example" {
 	type primary;
 	file "insecure.nsec3.example.db";

bin/tests/system/dnssec/ns3/sign.sh

@@ -905,3 +905,34 @@ ksk=$("$KEYGEN" -q -a "$DEFAULT_ALGORITHM" -b "$DEFAULT_BITS" -f KSK "$zone")
 zsk=$("$KEYGEN" -q -a "$DEFAULT_ALGORITHM" -b "$DEFAULT_BITS" "$zone")
 cat "$infile" "$ksk.key" "$zsk.key" >"$zonefile"
 "$SIGNER" -g -o "$zone" "$zonefile" >/dev/null 2>&1
+
+#
+#
+#
+zone=extrabadkey.example.
+infile=template.db.in
+zonefile=extrabadkey.example.db
+
+# Add KSK and ZSK that we will mangle to RSAMD5
+ksk=$("$KEYGEN" -q -a "$DEFAULT_ALGORITHM" -b "$DEFAULT_BITS" -f KSK "$zone")
+zsk=$("$KEYGEN" -q -a "$DEFAULT_ALGORITHM" -b "$DEFAULT_BITS" "$zone")
+cat "$infile" "$ksk.key" "$zsk.key" >"$zonefile"
+"$SIGNER" -g -O full -o "$zone" "$zonefile" >/dev/null 2>&1
+
+# Mangle the signatures to RSAMD5 and save them for future use
+sed -ne "s/\(IN[[:space:]]*RRSIG[[:space:]]*[A-Z]*\) $DEFAULT_ALGORITHM_NUMBER /\1 1 /p" <"$zonefile.signed" >"$zonefile.signed.rsamd5"
+
+# Now add normal KSK and ZSK to the zone file
+ksk=$("$KEYGEN" -q -a "$DEFAULT_ALGORITHM" -b "$DEFAULT_BITS" -f KSK "$zone")
+zsk=$("$KEYGEN" -q -a "$DEFAULT_ALGORITHM" -b "$DEFAULT_BITS" "$zone")
+cat "$infile" "$ksk.key" "$zsk.key" >"$zonefile"
+
+# Mangle the DNSKEY algorithm numbers and add them to the signed zone file
+cat "$ksk.key" "$zsk.key" | sed -e "s/\(IN[[:space:]]*DNSKEY[[:space:]]*[0-9]* 3\) $DEFAULT_ALGORITHM_NUMBER /\1 1 /" >>"$zonefile"
+
+# Sign normally
+"$SIGNER" -g -o "$zone" "$zonefile" >/dev/null 2>&1
+
+# Add the mangled signatures to signed zone file
+cat "$zonefile.signed.rsamd5" >>"$zonefile.signed"
+rm "$zonefile.signed.rsamd5"

bin/tests/system/dnssec/tests_validation.py

@@ -1385,3 +1385,11 @@ def test_rrsigs_for_glue():
         record.rdtype == rdatatype.RRSIG and record.covers == rdatatype.A
         for record in res.answer
     )
+
+
+def test_extra_bad_algorithm():
+    msg = isctest.query.create("a.extrabadkey.example", "A")
+    res1 = isctest.query.tcp(msg, "10.53.0.3")
+    res2 = isctest.query.tcp(msg, "10.53.0.4")
+    isctest.check.same_answer(res1, res2)
+    isctest.check.adflag(res2)

lib/dns/validator.c

@@ -1092,8 +1092,14 @@ select_signing_key(dns_validator_t *val, dns_rdataset_t *rdataset) {
 			continue;
 		}
 
-		return dns_dnssec_keyfromrdata(&siginfo->signer, &rdata,
-					       val->view->mctx, &val->key);
+		result = dns_dnssec_keyfromrdata(&siginfo->signer, &rdata,
+						 val->view->mctx, &val->key);
+		/* Don't count unsupported algorithm towards max fails */
+		if (result == DST_R_UNSUPPORTEDALG) {
+			/* Continue with the next key */
+			continue;
+		}
+		return result;
 	}
 
 	return ISC_R_NOTFOUND;