From cfbe01c62f093b3b2b44340d63ee3f152f02f2c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Mon, 16 Jan 2023 10:54:22 +0100 Subject: [PATCH 1/6] Add microbenchmark for isc_iterated_hash() Add microbenchmark for isc_iterated_hash() to measure the speed of NSEC3 per second. --- tests/bench/.gitignore | 1 + tests/bench/Makefile.am | 1 + tests/bench/iterated_hash.c | 76 +++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 tests/bench/iterated_hash.c diff --git a/tests/bench/.gitignore b/tests/bench/.gitignore index b166d01436..afd821b425 100644 --- a/tests/bench/.gitignore +++ b/tests/bench/.gitignore @@ -1,4 +1,5 @@ /ascii /compress +/iterated_hash /dns_name_fromwire /siphash diff --git a/tests/bench/Makefile.am b/tests/bench/Makefile.am index c5be25c8ec..f99d47ed57 100644 --- a/tests/bench/Makefile.am +++ b/tests/bench/Makefile.am @@ -12,6 +12,7 @@ LDADD += \ noinst_PROGRAMS = \ ascii \ compress \ + iterated_hash \ dns_name_fromwire \ siphash diff --git a/tests/bench/iterated_hash.c b/tests/bench/iterated_hash.c new file mode 100644 index 0000000000..a8da177433 --- /dev/null +++ b/tests/bench/iterated_hash.c @@ -0,0 +1,76 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * SPDX-License-Identifier: MPL-2.0 + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, you can obtain one at https://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +#include +#include +#include +#include + +#include +#include +#include + +#include + +static void +time_it(const int count, const int iterations, const unsigned char *salt, + const int saltlen, const unsigned char *in, const int inlen) { + uint8_t out[NSEC3_MAX_HASH_LENGTH] = { 0 }; + isc_time_t start, finish; + + printf("%d iterations, %d salt length, %d input length: ", iterations, + saltlen, inlen); + fflush(stdout); + + isc_time_now_hires(&start); + + int i = 0; + while (i++ < count) { + isc_iterated_hash(out, 1, iterations, salt, saltlen, in, inlen); + } + + isc_time_now_hires(&finish); + + uint64_t microseconds = isc_time_microdiff(&finish, &start); + printf("%0.2f us per iterated_hash()\n", (double)microseconds / count); + fflush(stdout); +} + +int +main(void) { + uint8_t salt[DNS_NAME_MAXWIRE]; + uint8_t in[DNS_NAME_MAXWIRE]; + size_t saltlen = sizeof(salt); + size_t inlen = sizeof(in); + + isc_random_buf(salt, saltlen); + isc_random_buf(in, inlen); + + time_it(10000, 150, salt, saltlen, in, inlen); + time_it(10000, 15, salt, saltlen, in, inlen); + time_it(10000, 0, salt, saltlen, in, inlen); + + saltlen = 32; + inlen = 32; + + time_it(10000, 150, salt, 32, in, inlen); + time_it(10000, 15, salt, 32, in, inlen); + time_it(10000, 0, salt, saltlen, in, inlen); + + saltlen = 0; + inlen = 1; + + time_it(10000, 150, salt, 32, in, inlen); + time_it(10000, 15, salt, 32, in, inlen); + time_it(10000, 0, salt, saltlen, in, inlen); +} From e6bfb8e45626e58a1cca687c798ee00973bf26d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Mon, 16 Jan 2023 09:16:35 +0100 Subject: [PATCH 2/6] Avoid implicit algorithm fetch for OpenSSL EVP_MD family The implicit algorithm fetch causes a lock contention and significant slowdown for small input buffers. For more details, see: https://github.com/openssl/openssl/issues/19612 Instead of using EVP_DigestInit_ex() initialize empty MD_CTX objects for each algorithm and use EVP_MD_CTX_copy_ex() to initialize MD_CTX from a static copy. Additionally avoid implicit algorithm fetching by using EVP_MD_fetch() for OpenSSL 3.0. --- lib/isc/include/isc/md.h | 40 +++++++++++++++------------ lib/isc/lib.c | 3 ++ lib/isc/md.c | 60 ++++++++++++++++++++++++++++++++++------ 3 files changed, 77 insertions(+), 26 deletions(-) diff --git a/lib/isc/include/isc/md.h b/lib/isc/include/isc/md.h index f52424b3b8..2c7213db49 100644 --- a/lib/isc/include/isc/md.h +++ b/lib/isc/include/isc/md.h @@ -37,25 +37,19 @@ typedef void isc_md_t; */ typedef void isc_md_type_t; -#define ISC_MD_MD5 isc__md_md5() -#define ISC_MD_SHA1 isc__md_sha1() -#define ISC_MD_SHA224 isc__md_sha224() -#define ISC_MD_SHA256 isc__md_sha256() -#define ISC_MD_SHA384 isc__md_sha384() -#define ISC_MD_SHA512 isc__md_sha512() +extern const isc_md_type_t *isc__md_md5; +extern const isc_md_type_t *isc__md_sha1; +extern const isc_md_type_t *isc__md_sha224; +extern const isc_md_type_t *isc__md_sha256; +extern const isc_md_type_t *isc__md_sha384; +extern const isc_md_type_t *isc__md_sha512; -const isc_md_type_t * -isc__md_md5(void); -const isc_md_type_t * -isc__md_sha1(void); -const isc_md_type_t * -isc__md_sha224(void); -const isc_md_type_t * -isc__md_sha256(void); -const isc_md_type_t * -isc__md_sha384(void); -const isc_md_type_t * -isc__md_sha512(void); +#define ISC_MD_MD5 isc__md_md5 +#define ISC_MD_SHA1 isc__md_sha1 +#define ISC_MD_SHA224 isc__md_sha224 +#define ISC_MD_SHA256 isc__md_sha256 +#define ISC_MD_SHA384 isc__md_sha384 +#define ISC_MD_SHA512 isc__md_sha512 #define ISC_MD5_DIGESTLENGTH isc_md_type_get_size(ISC_MD_MD5) #define ISC_MD5_BLOCK_LENGTH isc_md_type_get_block_size(ISC_MD_MD5) @@ -202,3 +196,13 @@ isc_md_type_get_size(const isc_md_type_t *md_type); */ size_t isc_md_type_get_block_size(const isc_md_type_t *md_type); + +/** + * Private + */ + +void +isc__md_initialize(void); + +void +isc__md_shutdown(void); diff --git a/lib/isc/lib.c b/lib/isc/lib.c index a04352622c..33c874906d 100644 --- a/lib/isc/lib.c +++ b/lib/isc/lib.c @@ -14,6 +14,7 @@ /*! \file */ #include +#include #include #include #include @@ -49,11 +50,13 @@ isc__initialize(void) { isc__trampoline_initialize(); isc__uv_initialize(); isc__xml_initialize(); + isc__md_initialize(); (void)isc_os_ncpus(); } void isc__shutdown(void) { + isc__md_shutdown(); isc__xml_shutdown(); isc__uv_shutdown(); isc__trampoline_shutdown(); diff --git a/lib/isc/md.c b/lib/isc/md.c index d094cfa0ec..725fef04c7 100644 --- a/lib/isc/md.c +++ b/lib/isc/md.c @@ -164,12 +164,56 @@ end: return (res); } -#define md_register_algorithm(alg) \ - const isc_md_type_t *isc__md_##alg(void) { return (EVP_##alg()); } +#ifndef UNIT_TESTING +const isc_md_type_t *isc__md_md5 = NULL; +const isc_md_type_t *isc__md_sha1 = NULL; +const isc_md_type_t *isc__md_sha224 = NULL; +const isc_md_type_t *isc__md_sha256 = NULL; +const isc_md_type_t *isc__md_sha384 = NULL; +const isc_md_type_t *isc__md_sha512 = NULL; -md_register_algorithm(md5); -md_register_algorithm(sha1); -md_register_algorithm(sha224); -md_register_algorithm(sha256); -md_register_algorithm(sha384); -md_register_algorithm(sha512); +#if OPENSSL_VERSION_NUMBER >= 0x30000000L +#define md_register_algorithm(alg, algname) \ + { \ + REQUIRE(isc__md_##alg == NULL); \ + isc__md_##alg = EVP_MD_fetch(NULL, algname, NULL); \ + RUNTIME_CHECK(isc__md_##alg != NULL); \ + } + +#define md_unregister_algorithm(alg) \ + { \ + REQUIRE(isc__md_##alg != NULL); \ + EVP_MD_free(*(isc_md_type_t **)&isc__md_##alg); \ + isc__md_##alg = NULL; \ + } + +#else +#define md_register_algorithm(alg, algname) \ + { \ + isc__md_##alg = EVP_##alg(); \ + RUNTIME_CHECK(isc__md_##alg != NULL); \ + } +#define md_unregister_algorithm(alg) +#endif + +void +isc__md_initialize(void) { + md_register_algorithm(md5, "MD5"); + md_register_algorithm(sha1, "SHA1"); + md_register_algorithm(sha224, "SHA224"); + md_register_algorithm(sha256, "SHA256"); + md_register_algorithm(sha384, "SHA384"); + md_register_algorithm(sha512, "SHA512"); +} + +void +isc__md_shutdown(void) { + md_unregister_algorithm(sha512); + md_unregister_algorithm(sha384); + md_unregister_algorithm(sha256); + md_unregister_algorithm(sha224); + md_unregister_algorithm(sha1); + md_unregister_algorithm(md5); +} + +#endif /* UNIT_TESTING */ From 36654df732786b4feebcc0b3c1b02550a31beef9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Mon, 16 Jan 2023 11:12:06 +0100 Subject: [PATCH 3/6] Use OpenSSL EVP_MD API directly in isc_iterated_hash() Instead of going through another layer, use OpenSSL EVP_MD API directly in the isc_iterated_hash() implementation. This shaves off couple of microseconds in the microbenchmark. --- lib/isc/iterated_hash.c | 57 +++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/lib/isc/iterated_hash.c b/lib/isc/iterated_hash.c index 14c63fb7f7..5e5b67c388 100644 --- a/lib/isc/iterated_hash.c +++ b/lib/isc/iterated_hash.c @@ -13,6 +13,8 @@ #include +#include + #include #include #include @@ -22,55 +24,48 @@ isc_iterated_hash(unsigned char *out, const unsigned int hashalg, const int iterations, const unsigned char *salt, const int saltlength, const unsigned char *in, const int inlength) { - isc_md_t *md; - isc_result_t result; - int n = 0; - unsigned int outlength = 0; - size_t len; - const unsigned char *buf; - REQUIRE(out != NULL); - if (hashalg != 1) { - return (0); - } + int n = 0; + size_t len; + unsigned int outlength = 0; + const unsigned char *buf; + EVP_MD_CTX *ctx = EVP_MD_CTX_create(); - if ((md = isc_md_new()) == NULL) { + RUNTIME_CHECK(ctx != NULL); + + if (hashalg != 1) { return (0); } len = inlength; buf = in; do { - result = isc_md_init(md, ISC_MD_SHA1); - if (result != ISC_R_SUCCESS) { - goto md_fail; + if (EVP_DigestInit_ex(ctx, ISC_MD_SHA1, NULL) != 1) { + goto fail; } - result = isc_md_update(md, buf, len); - if (result != ISC_R_SUCCESS) { - goto md_fail; + + if (EVP_DigestUpdate(ctx, buf, len) != 1) { + goto fail; } - result = isc_md_update(md, salt, saltlength); - if (result != ISC_R_SUCCESS) { - goto md_fail; + + if (EVP_DigestUpdate(ctx, salt, saltlength) != 1) { + goto fail; } - result = isc_md_final(md, out, &outlength); - if (result != ISC_R_SUCCESS) { - goto md_fail; - } - result = isc_md_reset(md); - if (result != ISC_R_SUCCESS) { - goto md_fail; + + if (EVP_DigestFinal_ex(ctx, out, &outlength) != 1) { + goto fail; } + buf = out; len = outlength; } while (n++ < iterations); - isc_md_free(md); + EVP_MD_CTX_free(ctx); return (outlength); -md_fail: - isc_md_free(md); + +fail: + EVP_MD_CTX_free(ctx); return (0); } -#undef RETERR From 25db8d010337b8f62705b44a7f01aa4658ce1c6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Mon, 16 Jan 2023 12:56:53 +0100 Subject: [PATCH 4/6] Use OpenSSL 1.x SHA_CTX API in isc_iterated_hash() If the OpenSSL SHA1_{Init,Update,Final} API is still available, use it. The API has been deprecated in OpenSSL 3.0, but it is significantly faster than EVP_MD API, so make an exception here and keep using it until we can't. --- lib/isc/iterated_hash.c | 61 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/lib/isc/iterated_hash.c b/lib/isc/iterated_hash.c index 5e5b67c388..a2a6e234cf 100644 --- a/lib/isc/iterated_hash.c +++ b/lib/isc/iterated_hash.c @@ -13,12 +13,64 @@ #include -#include +#include #include -#include #include +#if OPENSSL_VERSION_NUMBER < 0x30000000L || OPENSSL_API_LEVEL < 30000 + +#include + +int +isc_iterated_hash(unsigned char *out, const unsigned int hashalg, + const int iterations, const unsigned char *salt, + const int saltlength, const unsigned char *in, + const int inlength) { + REQUIRE(out != NULL); + + int n = 0; + size_t len; + const unsigned char *buf; + SHA_CTX ctx; + + if (hashalg != 1) { + return (0); + } + + buf = in; + len = inlength; + + do { + if (SHA1_Init(&ctx) != 1) { + return (0); + } + + if (SHA1_Update(&ctx, buf, len) != 1) { + return (0); + } + + if (SHA1_Update(&ctx, salt, saltlength) != 1) { + return (0); + } + + if (SHA1_Final(out, &ctx) != 1) { + return (0); + } + + buf = out; + len = SHA_DIGEST_LENGTH; + } while (n++ < iterations); + + return (SHA_DIGEST_LENGTH); +} + +#else + +#include + +#include + int isc_iterated_hash(unsigned char *out, const unsigned int hashalg, const int iterations, const unsigned char *salt, @@ -38,8 +90,9 @@ isc_iterated_hash(unsigned char *out, const unsigned int hashalg, return (0); } - len = inlength; buf = in; + len = inlength; + do { if (EVP_DigestInit_ex(ctx, ISC_MD_SHA1, NULL) != 1) { goto fail; @@ -69,3 +122,5 @@ fail: EVP_MD_CTX_free(ctx); return (0); } + +#endif From f3753d591fbc5057997e08eaf0b41c91ef466b2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Mon, 16 Jan 2023 11:12:06 +0100 Subject: [PATCH 5/6] Use thread_local EVP_MD_CTX in isc_iterated_hash() As this code is on hot path (NSEC3) this introduces an additional optimization of the EVP_MD API - instead of calling EVP_MD_CTX_new() on every call to isc_iterated_hash(), we create two thread_local objects for each thread - a basectx and mdctx, initialize basectx once and then use EVP_MD_CTX_copy_ex() to flip the initialized state into mdctx. This saves us couple more valuable microseconds from the isc_iterated_hash() call. --- lib/isc/include/isc/iterated_hash.h | 9 ++++ lib/isc/iterated_hash.c | 66 +++++++++++++++++++++++------ lib/isc/lib.c | 3 ++ lib/isc/trampoline.c | 6 +++ 4 files changed, 71 insertions(+), 13 deletions(-) diff --git a/lib/isc/include/isc/iterated_hash.h b/lib/isc/include/isc/iterated_hash.h index b5d6ab676b..cb21761c96 100644 --- a/lib/isc/include/isc/iterated_hash.h +++ b/lib/isc/include/isc/iterated_hash.h @@ -35,4 +35,13 @@ isc_iterated_hash(unsigned char *out, const unsigned int hashalg, const int saltlength, const unsigned char *in, const int inlength); +/* + * Private + */ + +void +isc__iterated_hash_initialize(void); +void +isc__iterated_hash_shutdown(void); + ISC_LANG_ENDDECLS diff --git a/lib/isc/iterated_hash.c b/lib/isc/iterated_hash.c index a2a6e234cf..cafbb5a977 100644 --- a/lib/isc/iterated_hash.c +++ b/lib/isc/iterated_hash.c @@ -65,26 +65,39 @@ isc_iterated_hash(unsigned char *out, const unsigned int hashalg, return (SHA_DIGEST_LENGTH); } -#else +void +isc__iterated_hash_initialize(void) { + /* empty */ +} + +void +isc__iterated_hash_shutdown(void) { + /* empty */ +} + +#else /* HAVE_SHA1_INIT */ #include #include +static thread_local bool initialized = false; +static thread_local EVP_MD_CTX *mdctx = NULL; +static thread_local EVP_MD_CTX *basectx = NULL; + int isc_iterated_hash(unsigned char *out, const unsigned int hashalg, const int iterations, const unsigned char *salt, const int saltlength, const unsigned char *in, const int inlength) { REQUIRE(out != NULL); + REQUIRE(mdctx != NULL); + REQUIRE(basectx != NULL); int n = 0; size_t len; unsigned int outlength = 0; const unsigned char *buf; - EVP_MD_CTX *ctx = EVP_MD_CTX_create(); - - RUNTIME_CHECK(ctx != NULL); if (hashalg != 1) { return (0); @@ -92,21 +105,20 @@ isc_iterated_hash(unsigned char *out, const unsigned int hashalg, buf = in; len = inlength; - do { - if (EVP_DigestInit_ex(ctx, ISC_MD_SHA1, NULL) != 1) { + if (EVP_MD_CTX_copy_ex(mdctx, basectx) != 1) { goto fail; } - if (EVP_DigestUpdate(ctx, buf, len) != 1) { + if (EVP_DigestUpdate(mdctx, buf, len) != 1) { goto fail; } - if (EVP_DigestUpdate(ctx, salt, saltlength) != 1) { + if (EVP_DigestUpdate(mdctx, salt, saltlength) != 1) { goto fail; } - if (EVP_DigestFinal_ex(ctx, out, &outlength) != 1) { + if (EVP_DigestFinal_ex(mdctx, out, &outlength) != 1) { goto fail; } @@ -114,13 +126,41 @@ isc_iterated_hash(unsigned char *out, const unsigned int hashalg, len = outlength; } while (n++ < iterations); - EVP_MD_CTX_free(ctx); - return (outlength); fail: - EVP_MD_CTX_free(ctx); return (0); } -#endif +void +isc__iterated_hash_initialize(void) { + if (initialized) { + return; + } + + basectx = EVP_MD_CTX_new(); + INSIST(basectx != NULL); + mdctx = EVP_MD_CTX_new(); + INSIST(mdctx != NULL); + + RUNTIME_CHECK(EVP_DigestInit_ex(basectx, ISC_MD_SHA1, NULL) == 1); + initialized = true; +} + +void +isc__iterated_hash_shutdown(void) { + if (!initialized) { + return; + } + + REQUIRE(mdctx != NULL); + EVP_MD_CTX_free(mdctx); + mdctx = NULL; + REQUIRE(basectx != NULL); + EVP_MD_CTX_free(basectx); + basectx = NULL; + + initialized = false; +} + +#endif /* HAVE_SHA1_INIT */ diff --git a/lib/isc/lib.c b/lib/isc/lib.c index 33c874906d..daf00a5971 100644 --- a/lib/isc/lib.c +++ b/lib/isc/lib.c @@ -14,6 +14,7 @@ /*! \file */ #include +#include #include #include #include @@ -51,11 +52,13 @@ isc__initialize(void) { isc__uv_initialize(); isc__xml_initialize(); isc__md_initialize(); + isc__iterated_hash_initialize(); (void)isc_os_ncpus(); } void isc__shutdown(void) { + isc__iterated_hash_shutdown(); isc__md_shutdown(); isc__xml_shutdown(); isc__uv_shutdown(); diff --git a/lib/isc/trampoline.c b/lib/isc/trampoline.c index c862ba37b0..736c5840df 100644 --- a/lib/isc/trampoline.c +++ b/lib/isc/trampoline.c @@ -16,6 +16,8 @@ #include #include +#include +#include #include #include #include @@ -194,9 +196,13 @@ isc__trampoline_run(isc_threadarg_t arg) { isc__trampoline_attach(trampoline); + isc__iterated_hash_initialize(); + /* Run the main function */ result = (trampoline->start)(trampoline->arg); + isc__iterated_hash_shutdown(); + isc__trampoline_detach(trampoline); return (result); From 401294cf60bac9665299a8260bca0a49d9957e9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Mon, 16 Jan 2023 10:09:35 +0100 Subject: [PATCH 6/6] Add CHANGES and release note for [GL #3795] --- CHANGES | 6 ++++++ doc/notes/notes-current.rst | 3 +++ 2 files changed, 9 insertions(+) diff --git a/CHANGES b/CHANGES index 17b28f627f..f545a01b0f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,9 @@ +6072. [bug] Avoid the OpenSSL lock contention when initializing + Message Digest Contexts by using explicit algorithm + fetching, initializing static contexts for every + supported algorithms, and initializing the new context + by copying the static copy. [GL #3795] + 6071. [func] The use of "port" when configuring query-source, transfer-source, notify-source and parental-source addresses has been deprecated, along with the diff --git a/doc/notes/notes-current.rst b/doc/notes/notes-current.rst index cce6f01680..864b1f0dde 100644 --- a/doc/notes/notes-current.rst +++ b/doc/notes/notes-current.rst @@ -55,6 +55,9 @@ Bug Fixes cause increased memory consumption due to delayed cleaning of view memory. This has been fixed. :gl:`#3801` +- Improve the speed of the message digest algorithms (MD5, SHA-1, + SHA-2) and NSEC3 hashing. :gl:`#3795` + Known Issues ~~~~~~~~~~~~