From b5b13771f2654c476f916c81299166ccc15efe94 Mon Sep 17 00:00:00 2001
From: Mark Andrews <marka@isc.org>
Date: Tue, 11 Jul 2023 13:25:47 +1000
Subject: [PATCH] Clear OpenSSL errors on EVP_PKEY_new failures

(cherry picked from commit 6df53cdb8757b287023a8caa32b0d9757e4eeb0a)
---
 bin/tests/system/rsabigexponent/bigkey.c | 2 ++
 lib/dns/opensslecdsa_link.c              | 6 +++---
 lib/isc/hmac.c                           | 1 +
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/bin/tests/system/rsabigexponent/bigkey.c b/bin/tests/system/rsabigexponent/bigkey.c
index c1fb2c8f15..c096e547ba 100644
--- a/bin/tests/system/rsabigexponent/bigkey.c
+++ b/bin/tests/system/rsabigexponent/bigkey.c
@@ -84,6 +84,7 @@ main(int argc, char **argv) {
 	    !EVP_PKEY_set1_RSA(pkey, rsa))
 	{
 		fprintf(stderr, "fatal error: basic OpenSSL failure\n");
+		ERR_clear_error();
 		exit(1);
 	}
 
@@ -99,6 +100,7 @@ main(int argc, char **argv) {
 			"fatal error: RSA_generate_key_ex() fails "
 			"at file %s line %d\n",
 			__FILE__, __LINE__);
+		ERR_clear_error();
 		exit(1);
 	}
 
diff --git a/lib/dns/opensslecdsa_link.c b/lib/dns/opensslecdsa_link.c
index 13cbb07c68..d994387931 100644
--- a/lib/dns/opensslecdsa_link.c
+++ b/lib/dns/opensslecdsa_link.c
@@ -525,7 +525,7 @@ opensslecdsa_generate(dst_key_t *key, int unused, void (*callback)(int)) {
 
 	pkey = EVP_PKEY_new();
 	if (pkey == NULL) {
-		DST_RET(ISC_R_NOMEMORY);
+		DST_RET(dst__openssl_toresult(ISC_R_NOMEMORY));
 	}
 	if (!EVP_PKEY_set1_EC_KEY(pkey, eckey)) {
 		DST_RET(ISC_R_FAILURE);
@@ -794,7 +794,7 @@ opensslecdsa_fromdns(dst_key_t *key, isc_buffer_t *data) {
 
 	pkey = EVP_PKEY_new();
 	if (pkey == NULL) {
-		DST_RET(ISC_R_NOMEMORY);
+		DST_RET(dst__openssl_toresult(ISC_R_NOMEMORY));
 	}
 	if (!EVP_PKEY_set1_EC_KEY(pkey, eckey)) {
 		EVP_PKEY_free(pkey);
@@ -1105,7 +1105,7 @@ eckey_to_pkey(EC_KEY *eckey, EVP_PKEY **pkey) {
 
 	*pkey = EVP_PKEY_new();
 	if (*pkey == NULL) {
-		return (ISC_R_NOMEMORY);
+		return (dst__openssl_toresult(ISC_R_NOMEMORY));
 	}
 	if (!EVP_PKEY_set1_EC_KEY(*pkey, eckey)) {
 		EVP_PKEY_free(*pkey);
diff --git a/lib/isc/hmac.c b/lib/isc/hmac.c
index 8fce30bda9..15a217f218 100644
--- a/lib/isc/hmac.c
+++ b/lib/isc/hmac.c
@@ -55,6 +55,7 @@ isc_hmac_init(isc_hmac_t *hmac, const void *key, const size_t keylen,
 
 	pkey = EVP_PKEY_new_raw_private_key(EVP_PKEY_HMAC, NULL, key, keylen);
 	if (pkey == NULL) {
+		ERR_clear_error();
 		return (ISC_R_CRYPTOFAILURE);
 	}