diff --git a/lib/dns/resolver.c b/lib/dns/resolver.c index da2f2ab904..9c89e9f084 100644 --- a/lib/dns/resolver.c +++ b/lib/dns/resolver.c @@ -2458,7 +2458,8 @@ compute_cc(const resquery_t *query, uint8_t *cookie, const size_t len) { size_t buflen = add_serveraddr(buf, sizeof(buf), query); uint8_t digest[ISC_SIPHASH24_TAG_LENGTH] ISC_NONSTRING = { 0 }; - isc_siphash24(query->fctx->res->view->secret, buf, buflen, digest); + isc_siphash24(query->fctx->res->view->secret, buf, buflen, true, + digest); memmove(cookie, digest, CLIENT_COOKIE_SIZE); } diff --git a/lib/isc/hash.c b/lib/isc/hash.c index 8dcc788ab6..f2708e6bb5 100644 --- a/lib/isc/hash.c +++ b/lib/isc/hash.c @@ -86,14 +86,8 @@ isc_hash64(const void *data, const size_t length, const bool case_sensitive) { RUNTIME_CHECK(isc_once_do(&isc_hash_once, isc_hash_initialize) == ISC_R_SUCCESS); - if (case_sensitive) { - isc_siphash24(isc_hash_key, data, length, (uint8_t *)&hval); - } else { - uint8_t lower[1024]; - REQUIRE(length <= sizeof(lower)); - isc_ascii_lowercopy(lower, data, length); - isc_siphash24(isc_hash_key, lower, length, (uint8_t *)&hval); - } + isc_siphash24(isc_hash_key, data, length, case_sensitive, + (uint8_t *)&hval); return (hval); } @@ -107,15 +101,8 @@ isc_hash32(const void *data, const size_t length, const bool case_sensitive) { RUNTIME_CHECK(isc_once_do(&isc_hash_once, isc_hash_initialize) == ISC_R_SUCCESS); - if (case_sensitive) { - isc_halfsiphash24(isc_hash_key, data, length, (uint8_t *)&hval); - } else { - uint8_t lower[1024]; - REQUIRE(length <= sizeof(lower)); - isc_ascii_lowercopy(lower, data, length); - isc_halfsiphash24(isc_hash_key, lower, length, - (uint8_t *)&hval); - } + isc_halfsiphash24(isc_hash_key, data, length, case_sensitive, + (uint8_t *)&hval); return (hval); } diff --git a/lib/isc/include/isc/ascii.h b/lib/isc/include/isc/ascii.h index b63f461534..7ba411c4a0 100644 --- a/lib/isc/include/isc/ascii.h +++ b/lib/isc/include/isc/ascii.h @@ -68,7 +68,7 @@ isc_ascii_strtolower(char *str) { * memove() below) the balance might be different. */ static inline uint64_t -isc__ascii_tolower8(uint64_t octets) { +isc_ascii_tolower8(uint64_t octets) { /* * Multiply a single-byte constant by `all_bytes` to replicate * it to all eight bytes in a word. @@ -103,6 +103,20 @@ isc__ascii_tolower8(uint64_t octets) { return (octets | (is_upper >> 2)); } +/* + * Same, but 4 bytes at a time, used by isc_halfsiphash24() + */ +static inline uint32_t +isc_ascii_tolower4(uint32_t octets) { + uint32_t all_bytes = 0x01010101; + uint32_t heptets = octets & (0x7F * all_bytes); + uint32_t is_ascii = ~octets & (0x80 * all_bytes); + uint32_t is_gt_Z = heptets + (0x7F - 'Z') * all_bytes; + uint32_t is_ge_A = heptets + (0x80 - 'A') * all_bytes; + uint32_t is_upper = (is_ge_A ^ is_gt_Z) & is_ascii; + return (octets | (is_upper >> 2)); +} + /* * Helper function to do an unaligned load of 8 bytes in host byte order */ @@ -120,8 +134,8 @@ static inline bool isc_ascii_lowerequal(const uint8_t *a, const uint8_t *b, unsigned int len) { uint64_t a8 = 0, b8 = 0; while (len >= 8) { - a8 = isc__ascii_tolower8(isc__ascii_load8(a)); - b8 = isc__ascii_tolower8(isc__ascii_load8(b)); + a8 = isc_ascii_tolower8(isc__ascii_load8(a)); + b8 = isc_ascii_tolower8(isc__ascii_load8(b)); if (a8 != b8) { return (false); } @@ -147,8 +161,8 @@ static inline int isc_ascii_lowercmp(const uint8_t *a, const uint8_t *b, unsigned int len) { uint64_t a8 = 0, b8 = 0; while (len >= 8) { - a8 = isc__ascii_tolower8(htobe64(isc__ascii_load8(a))); - b8 = isc__ascii_tolower8(htobe64(isc__ascii_load8(b))); + a8 = isc_ascii_tolower8(htobe64(isc__ascii_load8(a))); + b8 = isc_ascii_tolower8(htobe64(isc__ascii_load8(b))); if (a8 != b8) { goto ret; } diff --git a/lib/isc/include/isc/siphash.h b/lib/isc/include/isc/siphash.h index 69d47425d7..905711f7cb 100644 --- a/lib/isc/include/isc/siphash.h +++ b/lib/isc/include/isc/siphash.h @@ -28,9 +28,9 @@ ISC_LANG_BEGINDECLS void isc_siphash24(const uint8_t *key, const uint8_t *in, const size_t inlen, - uint8_t *out); + bool case_sensitive, uint8_t *out); void isc_halfsiphash24(const uint8_t *key, const uint8_t *in, const size_t inlen, - uint8_t *out); + bool case_sensitive, uint8_t *out); ISC_LANG_ENDDECLS diff --git a/lib/isc/siphash.c b/lib/isc/siphash.c index 1a863ff8e1..8d13c61243 100644 --- a/lib/isc/siphash.c +++ b/lib/isc/siphash.c @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -76,6 +77,9 @@ (((uint32_t)((p)[0])) | ((uint32_t)((p)[1]) << 8) | \ ((uint32_t)((p)[2]) << 16) | ((uint32_t)((p)[3]) << 24)) +#define U8TO32_ONE(case_sensitive, byte) \ + (uint32_t)(case_sensitive ? byte : isc__ascii_tolower1(byte)) + #define U64TO8_LE(p, v) \ U32TO8_LE((p), (uint32_t)((v))); \ U32TO8_LE((p) + 4, (uint32_t)((v) >> 32)); @@ -86,9 +90,12 @@ ((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40) | \ ((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56)) +#define U8TO64_ONE(case_sensitive, byte) \ + (uint64_t)(case_sensitive ? byte : isc__ascii_tolower1(byte)) + void isc_siphash24(const uint8_t *k, const uint8_t *in, const size_t inlen, - uint8_t *out) { + bool case_sensitive, uint8_t *out) { REQUIRE(k != NULL); REQUIRE(out != NULL); @@ -106,7 +113,8 @@ isc_siphash24(const uint8_t *k, const uint8_t *in, const size_t inlen, const size_t left = inlen & 7; for (; in != end; in += 8) { - uint64_t m = U8TO64_LE(in); + uint64_t m = case_sensitive ? U8TO64_LE(in) + : isc_ascii_tolower8(U8TO64_LE(in)); v3 ^= m; @@ -119,25 +127,25 @@ isc_siphash24(const uint8_t *k, const uint8_t *in, const size_t inlen, switch (left) { case 7: - b |= ((uint64_t)in[6]) << 48; + b |= U8TO64_ONE(case_sensitive, in[6]) << 48; FALLTHROUGH; case 6: - b |= ((uint64_t)in[5]) << 40; + b |= U8TO64_ONE(case_sensitive, in[5]) << 40; FALLTHROUGH; case 5: - b |= ((uint64_t)in[4]) << 32; + b |= U8TO64_ONE(case_sensitive, in[4]) << 32; FALLTHROUGH; case 4: - b |= ((uint64_t)in[3]) << 24; + b |= U8TO64_ONE(case_sensitive, in[3]) << 24; FALLTHROUGH; case 3: - b |= ((uint64_t)in[2]) << 16; + b |= U8TO64_ONE(case_sensitive, in[2]) << 16; FALLTHROUGH; case 2: - b |= ((uint64_t)in[1]) << 8; + b |= U8TO64_ONE(case_sensitive, in[1]) << 8; FALLTHROUGH; case 1: - b |= ((uint64_t)in[0]); + b |= U8TO64_ONE(case_sensitive, in[0]); FALLTHROUGH; case 0: break; @@ -166,7 +174,7 @@ isc_siphash24(const uint8_t *k, const uint8_t *in, const size_t inlen, void isc_halfsiphash24(const uint8_t *k, const uint8_t *in, const size_t inlen, - uint8_t *out) { + bool case_sensitive, uint8_t *out) { REQUIRE(k != NULL); REQUIRE(out != NULL); @@ -184,7 +192,9 @@ isc_halfsiphash24(const uint8_t *k, const uint8_t *in, const size_t inlen, const int left = inlen & 3; for (; in != end; in += 4) { - uint32_t m = U8TO32_LE(in); + uint32_t m = case_sensitive ? U8TO32_LE(in) + : isc_ascii_tolower4(U8TO32_LE(in)); + v3 ^= m; for (size_t i = 0; i < cROUNDS; ++i) { @@ -196,13 +206,13 @@ isc_halfsiphash24(const uint8_t *k, const uint8_t *in, const size_t inlen, switch (left) { case 3: - b |= ((uint32_t)in[2]) << 16; + b |= U8TO32_ONE(case_sensitive, in[2]) << 16; FALLTHROUGH; case 2: - b |= ((uint32_t)in[1]) << 8; + b |= U8TO32_ONE(case_sensitive, in[1]) << 8; FALLTHROUGH; case 1: - b |= ((uint32_t)in[0]); + b |= U8TO32_ONE(case_sensitive, in[0]); FALLTHROUGH; case 0: break; diff --git a/lib/ns/client.c b/lib/ns/client.c index 699264d7cb..8b83c3908c 100644 --- a/lib/ns/client.c +++ b/lib/ns/client.c @@ -1196,7 +1196,7 @@ compute_cookie(ns_client_t *client, uint32_t when, uint32_t nonce, UNREACHABLE(); } - isc_siphash24(secret, input, inputlen, digest); + isc_siphash24(secret, input, inputlen, true, digest); isc_buffer_putmem(buf, digest, 8); break; } diff --git a/tests/bench/.gitignore b/tests/bench/.gitignore index c9c3825ad1..fb6b5a462a 100644 --- a/tests/bench/.gitignore +++ b/tests/bench/.gitignore @@ -1 +1,2 @@ ascii +siphash diff --git a/tests/bench/Makefile.am b/tests/bench/Makefile.am index 171db37b39..c53e6ea51d 100644 --- a/tests/bench/Makefile.am +++ b/tests/bench/Makefile.am @@ -7,4 +7,5 @@ LDADD += \ $(LIBISC_LIBS) noinst_PROGRAMS = \ - ascii + ascii \ + siphash diff --git a/tests/bench/siphash.c b/tests/bench/siphash.c new file mode 100644 index 0000000000..36f3ffad8b --- /dev/null +++ b/tests/bench/siphash.c @@ -0,0 +1,184 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * SPDX-License-Identifier: MPL-2.0 + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, you can obtain one at https://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include + +#define SIZE (1024 * 1024) + +int +main(void) { + static uint8_t bytes[SIZE]; + static uint8_t key[16]; + + isc_random_buf(bytes, SIZE); + isc_random_buf(key, sizeof(key)); + + for (size_t len = 256; len > 0; len = len * 4 / 5) { + isc_time_t start, finish; + uint64_t count = 0; + uint64_t sum = 0; + uint64_t us; + + isc_time_now_hires(&start); + + for (size_t end = len; end < SIZE; end += len) { + uint64_t hash; + uint8_t lower[1024]; + isc_ascii_lowercopy(lower, bytes + end - len, len); + isc_siphash24(key, lower, len, true, (void *)&hash); + sum += hash; + count++; + } + + isc_time_now_hires(&finish); + + us = isc_time_microdiff(&finish, &start); + printf("%f us wide-lower len %3zu, %7llu kh/s (%llx)\n", + (double)us / 1000000.0, len, + (unsigned long long)(count * 1000 / us), + (unsigned long long)sum); + } + + for (size_t len = 256; len > 0; len = len * 4 / 5) { + isc_time_t start, finish; + uint64_t count = 0; + uint64_t sum = 0; + uint64_t us; + + isc_time_now_hires(&start); + + for (size_t end = len; end < SIZE; end += len) { + uint64_t hash; + isc_siphash24(key, bytes + end - len, len, false, + (void *)&hash); + sum += hash; + count++; + } + + isc_time_now_hires(&finish); + + us = isc_time_microdiff(&finish, &start); + printf("%f us wide-icase len %3zu, %7llu kh/s (%llx)\n", + (double)us / 1000000.0, len, + (unsigned long long)(count * 1000 / us), + (unsigned long long)sum); + } + for (size_t len = 256; len > 0; len = len * 4 / 5) { + isc_time_t start, finish; + uint64_t count = 0; + uint64_t sum = 0; + uint64_t us; + + isc_time_now_hires(&start); + + for (size_t end = len; end < SIZE; end += len) { + uint64_t hash; + isc_siphash24(key, bytes + end - len, len, true, + (void *)&hash); + sum += hash; + count++; + } + + isc_time_now_hires(&finish); + + us = isc_time_microdiff(&finish, &start); + printf("%f us wide-bytes len %3zu, %7llu kh/s (%llx)\n", + (double)us / 1000000.0, len, + (unsigned long long)(count * 1000 / us), + (unsigned long long)sum); + } + + for (size_t len = 256; len > 0; len = len * 4 / 5) { + isc_time_t start, finish; + uint64_t count = 0; + uint64_t sum = 0; + uint64_t us; + + isc_time_now_hires(&start); + + for (size_t end = len; end < SIZE; end += len) { + uint32_t hash; + uint8_t lower[1024]; + isc_ascii_lowercopy(lower, bytes + end - len, len); + isc_halfsiphash24(key, lower, len, true, (void *)&hash); + sum += hash; + count++; + } + + isc_time_now_hires(&finish); + + us = isc_time_microdiff(&finish, &start); + printf("%f us half-lower len %3zu, %7llu kh/s (%llx)\n", + (double)us / 1000000.0, len, + (unsigned long long)(count * 1000 / us), + (unsigned long long)sum); + } + + for (size_t len = 256; len > 0; len = len * 4 / 5) { + isc_time_t start, finish; + uint64_t count = 0; + uint64_t sum = 0; + uint64_t us; + + isc_time_now_hires(&start); + + for (size_t end = len; end < SIZE; end += len) { + uint32_t hash; + isc_halfsiphash24(key, bytes + end - len, len, false, + (void *)&hash); + sum += hash; + count++; + } + + isc_time_now_hires(&finish); + + us = isc_time_microdiff(&finish, &start); + printf("%f us half-icase len %3zu, %7llu kh/s (%llx)\n", + (double)us / 1000000.0, len, + (unsigned long long)(count * 1000 / us), + (unsigned long long)sum); + } + + for (size_t len = 256; len > 0; len = len * 4 / 5) { + isc_time_t start, finish; + uint64_t count = 0; + uint64_t sum = 0; + uint64_t us; + + isc_time_now_hires(&start); + + for (size_t end = len; end < SIZE; end += len) { + uint32_t hash; + isc_halfsiphash24(key, bytes + end - len, len, true, + (void *)&hash); + sum += hash; + count++; + } + + isc_time_now_hires(&finish); + + us = isc_time_microdiff(&finish, &start); + printf("%f us half-bytes len %3zu, %7llu kh/s (%llx)\n", + (double)us / 1000000.0, len, + (unsigned long long)(count * 1000 / us), + (unsigned long long)sum); + } +} diff --git a/tests/isc/ascii_test.c b/tests/isc/ascii_test.c index 239e77b6c7..078a98ce27 100644 --- a/tests/isc/ascii_test.c +++ b/tests/isc/ascii_test.c @@ -145,14 +145,14 @@ ISC_RUN_TEST_IMPL(exhaustive) { uint64_t abi = isc_ascii_tolower(a) << 8 | isc_ascii_tolower(b); uint64_t ab1 = isc__ascii_tolower1(a) << 8 | isc__ascii_tolower1(b); - uint64_t ab8 = isc__ascii_tolower8(ab); + uint64_t ab8 = isc_ascii_tolower8(ab); /* each byte individually matches ctype.h */ assert_int_equal(tolower(a), isc_ascii_tolower(a)); assert_int_equal(tolower(a), isc__ascii_tolower1(a)); - assert_int_equal(tolower(a), isc__ascii_tolower8(a)); + assert_int_equal(tolower(a), isc_ascii_tolower8(a)); assert_int_equal(tolower(b), isc_ascii_tolower(b)); assert_int_equal(tolower(b), isc__ascii_tolower1(b)); - assert_int_equal(tolower(b), isc__ascii_tolower8(b)); + assert_int_equal(tolower(b), isc_ascii_tolower8(b)); /* two lanes of SWAR match other implementations */ assert_int_equal(ab8, abc); assert_int_equal(ab8, abi); @@ -160,12 +160,12 @@ ISC_RUN_TEST_IMPL(exhaustive) { /* check lack of overflow */ assert_int_equal(ab8 >> 16, 0); /* all lanes of SWAR work */ - assert_int_equal(isc__ascii_tolower8(ab << 8), abc << 8); - assert_int_equal(isc__ascii_tolower8(ab << 16), abc << 16); - assert_int_equal(isc__ascii_tolower8(ab << 24), abc << 24); - assert_int_equal(isc__ascii_tolower8(ab << 32), abc << 32); - assert_int_equal(isc__ascii_tolower8(ab << 40), abc << 40); - assert_int_equal(isc__ascii_tolower8(ab << 48), abc << 48); + assert_int_equal(isc_ascii_tolower8(ab << 8), abc << 8); + assert_int_equal(isc_ascii_tolower8(ab << 16), abc << 16); + assert_int_equal(isc_ascii_tolower8(ab << 24), abc << 24); + assert_int_equal(isc_ascii_tolower8(ab << 32), abc << 32); + assert_int_equal(isc_ascii_tolower8(ab << 40), abc << 40); + assert_int_equal(isc_ascii_tolower8(ab << 48), abc << 48); } } diff --git a/tests/isc/siphash_test.c b/tests/isc/siphash_test.c index 5480afd791..f78c1d1e72 100644 --- a/tests/isc/siphash_test.c +++ b/tests/isc/siphash_test.c @@ -139,7 +139,7 @@ ISC_RUN_TEST_IMPL(isc_siphash24) { for (size_t i = 0; i < ARRAY_SIZE(in); i++) { in[i] = i; - isc_siphash24(key, in, i, out); + isc_siphash24(key, in, i, true, out); assert_memory_equal(out, vectors_sip64[i], 8); } } @@ -154,7 +154,7 @@ ISC_RUN_TEST_IMPL(isc_halfsiphash24) { for (size_t i = 0; i < ARRAY_SIZE(in); i++) { in[i] = i; - isc_halfsiphash24(key, in, i, out); + isc_halfsiphash24(key, in, i, true, out); assert_memory_equal(out, vectors_hsip32[i], 4); } }