Enforce NSEC3 record consistency

NSEC3 hashes are required to fit within a single DNS label.  Since there
are 5 bits per label byte without pad characters, the maximum hash size
is floor(63*5/8) (39 bytes).

This patch enforces this maximum length for unknown algorithms, while
strictly enforcing the exact expected digest length for known algorithms
like SHA-1.

bin/tests/system/checkzone/zones/crashzone.db

@@ -47,7 +47,6 @@ FQ7RBG86KRMACA1NAAKP2KQRQALBA0C7.dyn.example.net.  7200	RRSIG	NSEC3 7 4 7200 201
 					577WZnTQemStx+diON9rEGXAGnU7C0KLjrFL
 					VyhocnBnNtxJS8eRMSWvb9XuYCMNhYKOurtt
 					Ar4qh4VW1+unmA== )
-I7A7A184GGMI35K1E3IR650LKO7NOB5R.dyn.example.net. 7200 IN NSEC3	1 0 10 76931F IMQ912BREQP1POLAH3RMONG;UED541AS A RRSIG
 IMQ912BREQP1POLAH3RMONG3UED541AS.dyn.example.net. 7200 IN NSEC3	1 0 10 76931F S3USV4M1HLVJ8F88EDSG8N9PVQRQ20N7 A RRSIG
 			7200	RRSIG	NSEC3 7 4 7200 20100227180048 (
 					20100221180048 30323 dyn.example.net.

lib/dns/include/dns/nsec3.h

@@ -27,6 +27,12 @@
 #define DNS_NSEC3_SALTSIZE	255
 #define DNS_NSEC3_MAXITERATIONS 50U
 
+/*
+ * The maximum hash that can be encoded in a single label using
+ * base32hexnp.  floor(63*5/8)
+ */
+#define NSEC3_MAX_HASH_LENGTH 39
+
 /*
  * hash = 1, flags =1, iterations = 2, salt length = 1, salt = 255 (max)
  * hash length = 1, hash = 255 (max), bitmap = 8192 + 512 (max)

lib/dns/rdata/generic/nsec3_50.c

@@ -35,6 +35,8 @@
 #include <isc/base32.h>
 #include <isc/iterated_hash.h>
 
+#include <dns/nsec3.h>
+
 #define RRTYPE_NSEC3_ATTRIBUTES DNS_RDATATYPEATTR_DNSSEC
 
 static isc_result_t
@@ -96,8 +98,17 @@ fromtext_nsec3(ARGS_FROMTEXT) {
 				      false));
 	isc_buffer_init(&b, buf, sizeof(buf));
 	RETTOK(isc_base32hexnp_decodestring(DNS_AS_STR(token), &b));
-	if (isc_buffer_usedlength(&b) > 0xffU) {
-		RETTOK(ISC_R_RANGE);
+	switch (hashalg) {
+	case dns_hash_sha1:
+		if (isc_buffer_usedlength(&b) != ISC_SHA1_DIGESTLENGTH) {
+			RETTOK(ISC_R_RANGE);
+		}
+		break;
+	default:
+		if (isc_buffer_usedlength(&b) > NSEC3_MAX_HASH_LENGTH) {
+			RETTOK(ISC_R_RANGE);
+		}
+		break;
 	}
 	RETERR(uint8_tobuffer(isc_buffer_usedlength(&b), target));
 	RETERR(mem_tobuffer(target, &buf, isc_buffer_usedlength(&b)));
@@ -184,7 +195,7 @@ totext_nsec3(ARGS_TOTEXT) {
 static isc_result_t
 fromwire_nsec3(ARGS_FROMWIRE) {
 	isc_region_t sr, rr;
-	unsigned int saltlen, hashlen;
+	unsigned int hash, saltlen, hashlen;
 
 	REQUIRE(type == dns_rdatatype_nsec3);
 
@@ -199,6 +210,7 @@ fromwire_nsec3(ARGS_FROMWIRE) {
 	if (sr.length < 5U) {
 		RETERR(DNS_R_FORMERR);
 	}
+	hash = sr.base[0];
 	saltlen = sr.base[4];
 	isc_region_consume(&sr, 5);
 
@@ -213,8 +225,19 @@ fromwire_nsec3(ARGS_FROMWIRE) {
 	hashlen = sr.base[0];
 	isc_region_consume(&sr, 1);
 
-	if (hashlen < 1 || sr.length < hashlen) {
-		RETERR(DNS_R_FORMERR);
+	switch (hash) {
+	case dns_hash_sha1:
+		if (hashlen != ISC_SHA1_DIGESTLENGTH || sr.length < hashlen) {
+			RETERR(DNS_R_FORMERR);
+		}
+		break;
+	default:
+		if (hashlen < 1 || hashlen > NSEC3_MAX_HASH_LENGTH ||
+		    sr.length < hashlen)
+		{
+			RETERR(DNS_R_FORMERR);
+		}
+		break;
 	}
 	isc_region_consume(&sr, hashlen);
 

lib/isc/include/isc/iterated_hash.h

@@ -13,18 +13,6 @@
 
 #pragma once
 
-/*
- * The maximal hash length that can be encoded in a name
- * using base32hex.  floor(255/8)*5
- */
-#define NSEC3_MAX_HASH_LENGTH 155
-
-/*
- * The maximum has that can be encoded in a single label using
- * base32hex.  floor(63/8)*5
- */
-#define NSEC3_MAX_LABEL_HASH 35
-
 int
 isc_iterated_hash(unsigned char *out, const unsigned int hashalg,
 		  const int iterations, const unsigned char *salt,

tests/bench/iterated_hash.c

@@ -23,6 +23,7 @@
 
 #include <dns/lib.h>
 #include <dns/name.h>
+#include <dns/nsec3.h>
 
 static void
 time_it(const int count, const int iterations, const unsigned char *salt,