From a301dc364c336bee58950f88e040f825df840b60 Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Fri, 31 May 2024 15:11:00 -0700 Subject: [PATCH 01/42] ZTS: Fix redacted_send failures on FreeBSD We're seeing failures for redacted_deleted and redacted_mount on FreeBSD 13-15: 09:58:34.74 diff: /dev/fd/3: No such file or directory 09:58:34.74 ERROR: diff /dev/fd/3 /dev/fd/4 exited 2 The test was trying to diff the file listings between two directories to see if they are the same. The workaround is to do a string comparison of the directory listings instead of using `diff`. Reviewed-by: Brian Behlendorf Signed-off-by: Tony Hutter Closes #16224 --- .../tests/functional/redacted_send/redacted_deleted.ksh | 2 +- .../tests/functional/redacted_send/redacted_mounts.ksh | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh index 3e2aeb73354..ec11610742f 100755 --- a/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh @@ -96,7 +96,7 @@ log_must zfs destroy -R $clone2 log_must eval "zfs send -i $sendfs#book2 --redact book3 $sendfs@snap2 >$stream" log_must eval "zfs recv $recvfs <$stream" log_must mount_redacted -f $recvfs -log_must diff <(ls $send_mnt) <(ls $recv_mnt) +log_must [ "$(ls $send_mnt)" == "$(ls $recv_mnt)" ] log_must zfs destroy -R $recvfs log_must zfs rollback -R $sendfs@snap diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_mounts.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_mounts.ksh index 0bc4bf46174..c041469163c 100755 --- a/tests/zfs-tests/tests/functional/redacted_send/redacted_mounts.ksh +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_mounts.ksh @@ -71,8 +71,7 @@ log_must ismounted $recvfs # deleted. contents=$(log_must find $recv_mnt) contents_orig=$(log_must find $send_mnt) -log_must diff <(echo ${contents//$recv_mnt/}) \ - <(echo ${contents_orig//$send_mnt/}) +log_must [ "${contents//$recv_mnt/}" == "${contents_orig//$send_mnt/}" ] log_must zfs redact $sendvol@snap book2 $clonevol@snap log_must eval "zfs send --redact book2 $sendvol@snap >$stream" log_must eval "zfs receive $recvvol <$stream" @@ -103,7 +102,6 @@ log_must mount_redacted -f $recvfs log_must ismounted $recvfs contents=$(log_must find $recv_mnt) contents_orig=$(log_must find $send_mnt) -log_must diff <(echo ${contents//$recv_mnt/}) \ - <(echo ${contents_orig//$send_mnt/}) +log_must [ "${contents//$recv_mnt/}" == "${contents_orig//$send_mnt/}" ] log_pass "Received redacted streams can be mounted." From 4185179190f3dd8e4ee467452a54f1eb2ee360b9 Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Sat, 18 May 2024 21:05:20 +1000 Subject: [PATCH 02/42] icp: remove unused blowfish_ctx and des_ctx Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Reviewed-by: Brian Behlendorf Signed-off-by: Rob Norris Closes #16209 --- module/icp/include/modes/modes.h | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/module/icp/include/modes/modes.h b/module/icp/include/modes/modes.h index 23bf46ab51a..950c1115f3e 100644 --- a/module/icp/include/modes/modes.h +++ b/module/icp/include/modes/modes.h @@ -269,34 +269,6 @@ typedef struct aes_ctx { #define ac_processed_data_len acu.acu_ccm.ccm_processed_data_len #define ac_tag_len acu.acu_gcm.gcm_tag_len -typedef struct blowfish_ctx { - union { - ecb_ctx_t bcu_ecb; - cbc_ctx_t bcu_cbc; - } bcu; -} blowfish_ctx_t; - -#define bc_flags bcu.bcu_ecb.ecb_common.cc_flags -#define bc_remainder_len bcu.bcu_ecb.ecb_common.cc_remainder_len -#define bc_keysched bcu.bcu_ecb.ecb_common.cc_keysched -#define bc_keysched_len bcu.bcu_ecb.ecb_common.cc_keysched_len -#define bc_iv bcu.bcu_ecb.ecb_common.cc_iv -#define bc_lastp bcu.bcu_ecb.ecb_common.cc_lastp - -typedef struct des_ctx { - union { - ecb_ctx_t dcu_ecb; - cbc_ctx_t dcu_cbc; - } dcu; -} des_ctx_t; - -#define dc_flags dcu.dcu_ecb.ecb_common.cc_flags -#define dc_remainder_len dcu.dcu_ecb.ecb_common.cc_remainder_len -#define dc_keysched dcu.dcu_ecb.ecb_common.cc_keysched -#define dc_keysched_len dcu.dcu_ecb.ecb_common.cc_keysched_len -#define dc_iv dcu.dcu_ecb.ecb_common.cc_iv -#define dc_lastp dcu.dcu_ecb.ecb_common.cc_lastp - extern int ecb_cipher_contiguous_blocks(ecb_ctx_t *, char *, size_t, crypto_data_t *, size_t, int (*cipher)(const void *, const uint8_t *, uint8_t *)); From 57249bcddc0d743d64777d79344d03d22c1233d4 Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Sat, 18 May 2024 21:57:36 +1000 Subject: [PATCH 03/42] icp: brutally remove unused AES modes Still retaining the struture, for now. Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Reviewed-by: Brian Behlendorf Signed-off-by: Rob Norris Closes #16209 --- include/sys/crypto/common.h | 17 -- lib/libicp/Makefile.am | 3 - module/Kbuild.in | 3 - module/icp/algs/aes/aes_modes.c | 37 +-- module/icp/algs/modes/cbc.c | 264 -------------------- module/icp/algs/modes/ctr.c | 227 ------------------ module/icp/algs/modes/ecb.c | 127 ---------- module/icp/algs/modes/gcm.c | 79 +----- module/icp/algs/modes/modes.c | 20 +- module/icp/include/aes/aes_impl.h | 5 - module/icp/include/modes/modes.h | 114 +-------- module/icp/io/aes.c | 384 ++---------------------------- 12 files changed, 57 insertions(+), 1223 deletions(-) delete mode 100644 module/icp/algs/modes/cbc.c delete mode 100644 module/icp/algs/modes/ctr.c delete mode 100644 module/icp/algs/modes/ecb.c diff --git a/include/sys/crypto/common.h b/include/sys/crypto/common.h index 261e88eceee..74380565005 100644 --- a/include/sys/crypto/common.h +++ b/include/sys/crypto/common.h @@ -51,12 +51,6 @@ typedef struct crypto_mechanism { size_t cm_param_len; /* mech. parameter len */ } crypto_mechanism_t; -/* CK_AES_CTR_PARAMS provides parameters to the CKM_AES_CTR mechanism */ -typedef struct CK_AES_CTR_PARAMS { - ulong_t ulCounterBits; - uint8_t cb[16]; -} CK_AES_CTR_PARAMS; - /* CK_AES_CCM_PARAMS provides parameters to the CKM_AES_CCM mechanism */ typedef struct CK_AES_CCM_PARAMS { ulong_t ulMACSize; @@ -77,13 +71,6 @@ typedef struct CK_AES_GCM_PARAMS { ulong_t ulTagBits; } CK_AES_GCM_PARAMS; -/* CK_AES_GMAC_PARAMS provides parameters to the CKM_AES_GMAC mechanism */ -typedef struct CK_AES_GMAC_PARAMS { - uchar_t *pIv; - uchar_t *pAAD; - ulong_t ulAADLen; -} CK_AES_GMAC_PARAMS; - /* * The measurement unit bit flag for a mechanism's minimum or maximum key size. * The unit are mechanism dependent. It can be in bits or in bytes. @@ -103,12 +90,8 @@ typedef uint32_t crypto_keysize_unit_t; #define SUN_CKM_SHA512_HMAC_GENERAL "CKM_SHA512_HMAC_GENERAL" #define SUN_CKM_SHA512_224 "CKM_SHA512_224" #define SUN_CKM_SHA512_256 "CKM_SHA512_256" -#define SUN_CKM_AES_CBC "CKM_AES_CBC" -#define SUN_CKM_AES_ECB "CKM_AES_ECB" -#define SUN_CKM_AES_CTR "CKM_AES_CTR" #define SUN_CKM_AES_CCM "CKM_AES_CCM" #define SUN_CKM_AES_GCM "CKM_AES_GCM" -#define SUN_CKM_AES_GMAC "CKM_AES_GMAC" /* Data arguments of cryptographic operations */ diff --git a/lib/libicp/Makefile.am b/lib/libicp/Makefile.am index 4ba55b2158b..f40512bec9c 100644 --- a/lib/libicp/Makefile.am +++ b/lib/libicp/Makefile.am @@ -18,13 +18,10 @@ nodist_libicp_la_SOURCES = \ module/icp/algs/blake3/blake3_impl.c \ module/icp/algs/edonr/edonr.c \ module/icp/algs/modes/modes.c \ - module/icp/algs/modes/cbc.c \ module/icp/algs/modes/gcm_generic.c \ module/icp/algs/modes/gcm_pclmulqdq.c \ module/icp/algs/modes/gcm.c \ - module/icp/algs/modes/ctr.c \ module/icp/algs/modes/ccm.c \ - module/icp/algs/modes/ecb.c \ module/icp/algs/sha2/sha2_generic.c \ module/icp/algs/sha2/sha256_impl.c \ module/icp/algs/sha2/sha512_impl.c \ diff --git a/module/Kbuild.in b/module/Kbuild.in index 7e08374fa2b..6e2eab22588 100644 --- a/module/Kbuild.in +++ b/module/Kbuild.in @@ -101,10 +101,7 @@ ICP_OBJS := \ algs/blake3/blake3_generic.o \ algs/blake3/blake3_impl.o \ algs/edonr/edonr.o \ - algs/modes/cbc.o \ algs/modes/ccm.o \ - algs/modes/ctr.o \ - algs/modes/ecb.o \ algs/modes/gcm.o \ algs/modes/gcm_generic.o \ algs/modes/modes.o \ diff --git a/module/icp/algs/aes/aes_modes.c b/module/icp/algs/aes/aes_modes.c index 6a25496d050..631e92f3542 100644 --- a/module/icp/algs/aes/aes_modes.c +++ b/module/icp/algs/aes/aes_modes.c @@ -75,25 +75,17 @@ aes_encrypt_contiguous_blocks(void *ctx, char *data, size_t length, aes_ctx_t *aes_ctx = ctx; int rv; - if (aes_ctx->ac_flags & CTR_MODE) { - rv = ctr_mode_contiguous_blocks(ctx, data, length, out, - AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); - } else if (aes_ctx->ac_flags & CCM_MODE) { + if (aes_ctx->ac_flags & CCM_MODE) { rv = ccm_mode_encrypt_contiguous_blocks(ctx, data, length, out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, aes_xor_block); - } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) { + } else if (aes_ctx->ac_flags & GCM_MODE) { rv = gcm_mode_encrypt_contiguous_blocks(ctx, data, length, out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, aes_xor_block); - } else if (aes_ctx->ac_flags & CBC_MODE) { - rv = cbc_encrypt_contiguous_blocks(ctx, - data, length, out, AES_BLOCK_LEN, aes_encrypt_block, - aes_copy_block, aes_xor_block); - } else { - rv = ecb_cipher_contiguous_blocks(ctx, data, length, out, - AES_BLOCK_LEN, aes_encrypt_block); } + else + __builtin_unreachable(); return (rv); } @@ -108,28 +100,15 @@ aes_decrypt_contiguous_blocks(void *ctx, char *data, size_t length, aes_ctx_t *aes_ctx = ctx; int rv; - if (aes_ctx->ac_flags & CTR_MODE) { - rv = ctr_mode_contiguous_blocks(ctx, data, length, out, - AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); - if (rv == CRYPTO_DATA_LEN_RANGE) - rv = CRYPTO_ENCRYPTED_DATA_LEN_RANGE; - } else if (aes_ctx->ac_flags & CCM_MODE) { + if (aes_ctx->ac_flags & CCM_MODE) { rv = ccm_mode_decrypt_contiguous_blocks(ctx, data, length, out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, aes_xor_block); - } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) { + } else if (aes_ctx->ac_flags & GCM_MODE) { rv = gcm_mode_decrypt_contiguous_blocks(ctx, data, length, out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, aes_xor_block); - } else if (aes_ctx->ac_flags & CBC_MODE) { - rv = cbc_decrypt_contiguous_blocks(ctx, data, length, out, - AES_BLOCK_LEN, aes_decrypt_block, aes_copy_block, - aes_xor_block); - } else { - rv = ecb_cipher_contiguous_blocks(ctx, data, length, out, - AES_BLOCK_LEN, aes_decrypt_block); - if (rv == CRYPTO_DATA_LEN_RANGE) - rv = CRYPTO_ENCRYPTED_DATA_LEN_RANGE; - } + } else + __builtin_unreachable(); return (rv); } diff --git a/module/icp/algs/modes/cbc.c b/module/icp/algs/modes/cbc.c deleted file mode 100644 index d0219fb24c4..00000000000 --- a/module/icp/algs/modes/cbc.c +++ /dev/null @@ -1,264 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or https://opensource.org/licenses/CDDL-1.0. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#include -#include -#include -#include - -/* - * Algorithm independent CBC functions. - */ -int -cbc_encrypt_contiguous_blocks(cbc_ctx_t *ctx, char *data, size_t length, - crypto_data_t *out, size_t block_size, - int (*encrypt)(const void *, const uint8_t *, uint8_t *), - void (*copy_block)(uint8_t *, uint8_t *), - void (*xor_block)(uint8_t *, uint8_t *)) -{ - size_t remainder = length; - size_t need = 0; - uint8_t *datap = (uint8_t *)data; - uint8_t *blockp; - uint8_t *lastp; - void *iov_or_mp; - offset_t offset; - uint8_t *out_data_1; - uint8_t *out_data_2; - size_t out_data_1_len; - - if (length + ctx->cbc_remainder_len < block_size) { - /* accumulate bytes here and return */ - memcpy((uint8_t *)ctx->cbc_remainder + ctx->cbc_remainder_len, - datap, - length); - ctx->cbc_remainder_len += length; - ctx->cbc_copy_to = datap; - return (CRYPTO_SUCCESS); - } - - lastp = (uint8_t *)ctx->cbc_iv; - crypto_init_ptrs(out, &iov_or_mp, &offset); - - do { - /* Unprocessed data from last call. */ - if (ctx->cbc_remainder_len > 0) { - need = block_size - ctx->cbc_remainder_len; - - if (need > remainder) - return (CRYPTO_DATA_LEN_RANGE); - - memcpy(&((uint8_t *)ctx->cbc_remainder) - [ctx->cbc_remainder_len], datap, need); - - blockp = (uint8_t *)ctx->cbc_remainder; - } else { - blockp = datap; - } - - /* - * XOR the previous cipher block or IV with the - * current clear block. - */ - xor_block(blockp, lastp); - encrypt(ctx->cbc_keysched, lastp, lastp); - crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1, - &out_data_1_len, &out_data_2, block_size); - - /* copy block to where it belongs */ - if (out_data_1_len == block_size) { - copy_block(lastp, out_data_1); - } else { - memcpy(out_data_1, lastp, out_data_1_len); - if (out_data_2 != NULL) { - memcpy(out_data_2, - lastp + out_data_1_len, - block_size - out_data_1_len); - } - } - /* update offset */ - out->cd_offset += block_size; - - /* Update pointer to next block of data to be processed. */ - if (ctx->cbc_remainder_len != 0) { - datap += need; - ctx->cbc_remainder_len = 0; - } else { - datap += block_size; - } - - remainder = (size_t)&data[length] - (size_t)datap; - - /* Incomplete last block. */ - if (remainder > 0 && remainder < block_size) { - memcpy(ctx->cbc_remainder, datap, remainder); - ctx->cbc_remainder_len = remainder; - ctx->cbc_copy_to = datap; - goto out; - } - ctx->cbc_copy_to = NULL; - - } while (remainder > 0); - -out: - /* - * Save the last encrypted block in the context. - */ - if (ctx->cbc_lastp != NULL) { - copy_block((uint8_t *)ctx->cbc_lastp, (uint8_t *)ctx->cbc_iv); - ctx->cbc_lastp = (uint8_t *)ctx->cbc_iv; - } - - return (CRYPTO_SUCCESS); -} - -#define OTHER(a, ctx) \ - (((a) == (ctx)->cbc_lastblock) ? (ctx)->cbc_iv : (ctx)->cbc_lastblock) - -int -cbc_decrypt_contiguous_blocks(cbc_ctx_t *ctx, char *data, size_t length, - crypto_data_t *out, size_t block_size, - int (*decrypt)(const void *, const uint8_t *, uint8_t *), - void (*copy_block)(uint8_t *, uint8_t *), - void (*xor_block)(uint8_t *, uint8_t *)) -{ - size_t remainder = length; - size_t need = 0; - uint8_t *datap = (uint8_t *)data; - uint8_t *blockp; - uint8_t *lastp; - void *iov_or_mp; - offset_t offset; - uint8_t *out_data_1; - uint8_t *out_data_2; - size_t out_data_1_len; - - if (length + ctx->cbc_remainder_len < block_size) { - /* accumulate bytes here and return */ - memcpy((uint8_t *)ctx->cbc_remainder + ctx->cbc_remainder_len, - datap, - length); - ctx->cbc_remainder_len += length; - ctx->cbc_copy_to = datap; - return (CRYPTO_SUCCESS); - } - - lastp = ctx->cbc_lastp; - crypto_init_ptrs(out, &iov_or_mp, &offset); - - do { - /* Unprocessed data from last call. */ - if (ctx->cbc_remainder_len > 0) { - need = block_size - ctx->cbc_remainder_len; - - if (need > remainder) - return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE); - - memcpy(&((uint8_t *)ctx->cbc_remainder) - [ctx->cbc_remainder_len], datap, need); - - blockp = (uint8_t *)ctx->cbc_remainder; - } else { - blockp = datap; - } - - /* LINTED: pointer alignment */ - copy_block(blockp, (uint8_t *)OTHER((uint64_t *)lastp, ctx)); - - decrypt(ctx->cbc_keysched, blockp, - (uint8_t *)ctx->cbc_remainder); - blockp = (uint8_t *)ctx->cbc_remainder; - - /* - * XOR the previous cipher block or IV with the - * currently decrypted block. - */ - xor_block(lastp, blockp); - - /* LINTED: pointer alignment */ - lastp = (uint8_t *)OTHER((uint64_t *)lastp, ctx); - - crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1, - &out_data_1_len, &out_data_2, block_size); - - memcpy(out_data_1, blockp, out_data_1_len); - if (out_data_2 != NULL) { - memcpy(out_data_2, blockp + out_data_1_len, - block_size - out_data_1_len); - } - - /* update offset */ - out->cd_offset += block_size; - - /* Update pointer to next block of data to be processed. */ - if (ctx->cbc_remainder_len != 0) { - datap += need; - ctx->cbc_remainder_len = 0; - } else { - datap += block_size; - } - - remainder = (size_t)&data[length] - (size_t)datap; - - /* Incomplete last block. */ - if (remainder > 0 && remainder < block_size) { - memcpy(ctx->cbc_remainder, datap, remainder); - ctx->cbc_remainder_len = remainder; - ctx->cbc_lastp = lastp; - ctx->cbc_copy_to = datap; - return (CRYPTO_SUCCESS); - } - ctx->cbc_copy_to = NULL; - - } while (remainder > 0); - - ctx->cbc_lastp = lastp; - return (CRYPTO_SUCCESS); -} - -int -cbc_init_ctx(cbc_ctx_t *cbc_ctx, char *param, size_t param_len, - size_t block_size, void (*copy_block)(uint8_t *, uint64_t *)) -{ - /* Copy IV into context. */ - ASSERT3P(param, !=, NULL); - ASSERT3U(param_len, ==, block_size); - - copy_block((uchar_t *)param, cbc_ctx->cbc_iv); - - return (CRYPTO_SUCCESS); -} - -void * -cbc_alloc_ctx(int kmflag) -{ - cbc_ctx_t *cbc_ctx; - - if ((cbc_ctx = kmem_zalloc(sizeof (cbc_ctx_t), kmflag)) == NULL) - return (NULL); - - cbc_ctx->cbc_flags = CBC_MODE; - return (cbc_ctx); -} diff --git a/module/icp/algs/modes/ctr.c b/module/icp/algs/modes/ctr.c deleted file mode 100644 index db6b1c71d5c..00000000000 --- a/module/icp/algs/modes/ctr.c +++ /dev/null @@ -1,227 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or https://opensource.org/licenses/CDDL-1.0. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#include -#include -#include -#include -#include - -/* - * Encrypt and decrypt multiple blocks of data in counter mode. - */ -int -ctr_mode_contiguous_blocks(ctr_ctx_t *ctx, char *data, size_t length, - crypto_data_t *out, size_t block_size, - int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct), - void (*xor_block)(uint8_t *, uint8_t *)) -{ - size_t remainder = length; - size_t need = 0; - uint8_t *datap = (uint8_t *)data; - uint8_t *blockp; - uint8_t *lastp; - void *iov_or_mp; - offset_t offset; - uint8_t *out_data_1; - uint8_t *out_data_2; - size_t out_data_1_len; - uint64_t lower_counter, upper_counter; - - if (length + ctx->ctr_remainder_len < block_size) { - /* accumulate bytes here and return */ - memcpy((uint8_t *)ctx->ctr_remainder + ctx->ctr_remainder_len, - datap, - length); - ctx->ctr_remainder_len += length; - ctx->ctr_copy_to = datap; - return (CRYPTO_SUCCESS); - } - - crypto_init_ptrs(out, &iov_or_mp, &offset); - - do { - /* Unprocessed data from last call. */ - if (ctx->ctr_remainder_len > 0) { - need = block_size - ctx->ctr_remainder_len; - - if (need > remainder) - return (CRYPTO_DATA_LEN_RANGE); - - memcpy(&((uint8_t *)ctx->ctr_remainder) - [ctx->ctr_remainder_len], datap, need); - - blockp = (uint8_t *)ctx->ctr_remainder; - } else { - blockp = datap; - } - - /* ctr_cb is the counter block */ - cipher(ctx->ctr_keysched, (uint8_t *)ctx->ctr_cb, - (uint8_t *)ctx->ctr_tmp); - - lastp = (uint8_t *)ctx->ctr_tmp; - - /* - * Increment Counter. - */ - lower_counter = ntohll(ctx->ctr_cb[1] & ctx->ctr_lower_mask); - lower_counter = htonll(lower_counter + 1); - lower_counter &= ctx->ctr_lower_mask; - ctx->ctr_cb[1] = (ctx->ctr_cb[1] & ~(ctx->ctr_lower_mask)) | - lower_counter; - - /* wrap around */ - if (lower_counter == 0) { - upper_counter = - ntohll(ctx->ctr_cb[0] & ctx->ctr_upper_mask); - upper_counter = htonll(upper_counter + 1); - upper_counter &= ctx->ctr_upper_mask; - ctx->ctr_cb[0] = - (ctx->ctr_cb[0] & ~(ctx->ctr_upper_mask)) | - upper_counter; - } - - /* - * XOR encrypted counter block with the current clear block. - */ - xor_block(blockp, lastp); - - crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1, - &out_data_1_len, &out_data_2, block_size); - - /* copy block to where it belongs */ - memcpy(out_data_1, lastp, out_data_1_len); - if (out_data_2 != NULL) { - memcpy(out_data_2, lastp + out_data_1_len, - block_size - out_data_1_len); - } - /* update offset */ - out->cd_offset += block_size; - - /* Update pointer to next block of data to be processed. */ - if (ctx->ctr_remainder_len != 0) { - datap += need; - ctx->ctr_remainder_len = 0; - } else { - datap += block_size; - } - - remainder = (size_t)&data[length] - (size_t)datap; - - /* Incomplete last block. */ - if (remainder > 0 && remainder < block_size) { - memcpy(ctx->ctr_remainder, datap, remainder); - ctx->ctr_remainder_len = remainder; - ctx->ctr_copy_to = datap; - goto out; - } - ctx->ctr_copy_to = NULL; - - } while (remainder > 0); - -out: - return (CRYPTO_SUCCESS); -} - -int -ctr_mode_final(ctr_ctx_t *ctx, crypto_data_t *out, - int (*encrypt_block)(const void *, const uint8_t *, uint8_t *)) -{ - uint8_t *lastp; - void *iov_or_mp; - offset_t offset; - uint8_t *out_data_1; - uint8_t *out_data_2; - size_t out_data_1_len; - uint8_t *p; - int i; - - if (out->cd_length < ctx->ctr_remainder_len) - return (CRYPTO_DATA_LEN_RANGE); - - encrypt_block(ctx->ctr_keysched, (uint8_t *)ctx->ctr_cb, - (uint8_t *)ctx->ctr_tmp); - - lastp = (uint8_t *)ctx->ctr_tmp; - p = (uint8_t *)ctx->ctr_remainder; - for (i = 0; i < ctx->ctr_remainder_len; i++) { - p[i] ^= lastp[i]; - } - - crypto_init_ptrs(out, &iov_or_mp, &offset); - crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1, - &out_data_1_len, &out_data_2, ctx->ctr_remainder_len); - - memcpy(out_data_1, p, out_data_1_len); - if (out_data_2 != NULL) { - memcpy(out_data_2, - (uint8_t *)p + out_data_1_len, - ctx->ctr_remainder_len - out_data_1_len); - } - out->cd_offset += ctx->ctr_remainder_len; - ctx->ctr_remainder_len = 0; - return (CRYPTO_SUCCESS); -} - -int -ctr_init_ctx(ctr_ctx_t *ctr_ctx, ulong_t count, uint8_t *cb, - void (*copy_block)(uint8_t *, uint8_t *)) -{ - uint64_t upper_mask = 0; - uint64_t lower_mask = 0; - - if (count == 0 || count > 128) { - return (CRYPTO_MECHANISM_PARAM_INVALID); - } - /* upper 64 bits of the mask */ - if (count >= 64) { - count -= 64; - upper_mask = (count == 64) ? UINT64_MAX : (1ULL << count) - 1; - lower_mask = UINT64_MAX; - } else { - /* now the lower 63 bits */ - lower_mask = (1ULL << count) - 1; - } - ctr_ctx->ctr_lower_mask = htonll(lower_mask); - ctr_ctx->ctr_upper_mask = htonll(upper_mask); - - copy_block(cb, (uchar_t *)ctr_ctx->ctr_cb); - ctr_ctx->ctr_lastp = (uint8_t *)&ctr_ctx->ctr_cb[0]; - ctr_ctx->ctr_flags |= CTR_MODE; - return (CRYPTO_SUCCESS); -} - -void * -ctr_alloc_ctx(int kmflag) -{ - ctr_ctx_t *ctr_ctx; - - if ((ctr_ctx = kmem_zalloc(sizeof (ctr_ctx_t), kmflag)) == NULL) - return (NULL); - - ctr_ctx->ctr_flags = CTR_MODE; - return (ctr_ctx); -} diff --git a/module/icp/algs/modes/ecb.c b/module/icp/algs/modes/ecb.c deleted file mode 100644 index e2d8e71c161..00000000000 --- a/module/icp/algs/modes/ecb.c +++ /dev/null @@ -1,127 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or https://opensource.org/licenses/CDDL-1.0. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#include -#include -#include -#include - -/* - * Algorithm independent ECB functions. - */ -int -ecb_cipher_contiguous_blocks(ecb_ctx_t *ctx, char *data, size_t length, - crypto_data_t *out, size_t block_size, - int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct)) -{ - size_t remainder = length; - size_t need = 0; - uint8_t *datap = (uint8_t *)data; - uint8_t *blockp; - uint8_t *lastp; - void *iov_or_mp; - offset_t offset; - uint8_t *out_data_1; - uint8_t *out_data_2; - size_t out_data_1_len; - - if (length + ctx->ecb_remainder_len < block_size) { - /* accumulate bytes here and return */ - memcpy((uint8_t *)ctx->ecb_remainder + ctx->ecb_remainder_len, - datap, - length); - ctx->ecb_remainder_len += length; - ctx->ecb_copy_to = datap; - return (CRYPTO_SUCCESS); - } - - lastp = (uint8_t *)ctx->ecb_iv; - crypto_init_ptrs(out, &iov_or_mp, &offset); - - do { - /* Unprocessed data from last call. */ - if (ctx->ecb_remainder_len > 0) { - need = block_size - ctx->ecb_remainder_len; - - if (need > remainder) - return (CRYPTO_DATA_LEN_RANGE); - - memcpy(&((uint8_t *)ctx->ecb_remainder) - [ctx->ecb_remainder_len], datap, need); - - blockp = (uint8_t *)ctx->ecb_remainder; - } else { - blockp = datap; - } - - cipher(ctx->ecb_keysched, blockp, lastp); - crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1, - &out_data_1_len, &out_data_2, block_size); - - /* copy block to where it belongs */ - memcpy(out_data_1, lastp, out_data_1_len); - if (out_data_2 != NULL) { - memcpy(out_data_2, lastp + out_data_1_len, - block_size - out_data_1_len); - } - /* update offset */ - out->cd_offset += block_size; - - /* Update pointer to next block of data to be processed. */ - if (ctx->ecb_remainder_len != 0) { - datap += need; - ctx->ecb_remainder_len = 0; - } else { - datap += block_size; - } - - remainder = (size_t)&data[length] - (size_t)datap; - - /* Incomplete last block. */ - if (remainder > 0 && remainder < block_size) { - memcpy(ctx->ecb_remainder, datap, remainder); - ctx->ecb_remainder_len = remainder; - ctx->ecb_copy_to = datap; - goto out; - } - ctx->ecb_copy_to = NULL; - - } while (remainder > 0); - -out: - return (CRYPTO_SUCCESS); -} - -void * -ecb_alloc_ctx(int kmflag) -{ - ecb_ctx_t *ecb_ctx; - - if ((ecb_ctx = kmem_zalloc(sizeof (ecb_ctx_t), kmflag)) == NULL) - return (NULL); - - ecb_ctx->ecb_flags = ECB_MODE; - return (ecb_ctx); -} diff --git a/module/icp/algs/modes/gcm.c b/module/icp/algs/modes/gcm.c index dd8db6f9746..21f4301d584 100644 --- a/module/icp/algs/modes/gcm.c +++ b/module/icp/algs/modes/gcm.c @@ -50,11 +50,6 @@ static uint32_t icp_gcm_impl = IMPL_FASTEST; static uint32_t user_sel_impl = IMPL_FASTEST; -static inline int gcm_init_ctx_impl(boolean_t, gcm_ctx_t *, char *, size_t, - int (*)(const void *, const uint8_t *, uint8_t *), - void (*)(uint8_t *, uint8_t *), - void (*)(uint8_t *, uint8_t *)); - #ifdef CAN_USE_GCM_ASM /* Does the architecture we run on support the MOVBE instruction? */ boolean_t gcm_avx_can_use_movbe = B_FALSE; @@ -590,40 +585,11 @@ gcm_init(gcm_ctx_t *ctx, const uint8_t *iv, size_t iv_len, return (CRYPTO_SUCCESS); } -/* - * The following function is called at encrypt or decrypt init time - * for AES GCM mode. - */ -int -gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size, - int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), - void (*copy_block)(uint8_t *, uint8_t *), - void (*xor_block)(uint8_t *, uint8_t *)) -{ - return (gcm_init_ctx_impl(B_FALSE, gcm_ctx, param, block_size, - encrypt_block, copy_block, xor_block)); -} - -/* - * The following function is called at encrypt or decrypt init time - * for AES GMAC mode. - */ -int -gmac_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size, - int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), - void (*copy_block)(uint8_t *, uint8_t *), - void (*xor_block)(uint8_t *, uint8_t *)) -{ - return (gcm_init_ctx_impl(B_TRUE, gcm_ctx, param, block_size, - encrypt_block, copy_block, xor_block)); -} - /* * Init the GCM context struct. Handle the cycle and avx implementations here. - * Initialization of a GMAC context differs slightly from a GCM context. */ -static inline int -gcm_init_ctx_impl(boolean_t gmac_mode, gcm_ctx_t *gcm_ctx, char *param, +int +gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size, int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), void (*copy_block)(uint8_t *, uint8_t *), void (*xor_block)(uint8_t *, uint8_t *)) @@ -635,22 +601,16 @@ gcm_init_ctx_impl(boolean_t gmac_mode, gcm_ctx_t *gcm_ctx, char *param, if (param != NULL) { gcm_param = (CK_AES_GCM_PARAMS *)(void *)param; - if (gmac_mode == B_FALSE) { - /* GCM mode. */ - if ((rv = gcm_validate_args(gcm_param)) != 0) { - return (rv); - } - gcm_ctx->gcm_flags |= GCM_MODE; - - size_t tbits = gcm_param->ulTagBits; - tag_len = CRYPTO_BITS2BYTES(tbits); - iv_len = gcm_param->ulIvLen; - } else { - /* GMAC mode. */ - gcm_ctx->gcm_flags |= GMAC_MODE; - tag_len = CRYPTO_BITS2BYTES(AES_GMAC_TAG_BITS); - iv_len = AES_GMAC_IV_LEN; + /* GCM mode. */ + if ((rv = gcm_validate_args(gcm_param)) != 0) { + return (rv); } + gcm_ctx->gcm_flags |= GCM_MODE; + + size_t tbits = gcm_param->ulTagBits; + tag_len = CRYPTO_BITS2BYTES(tbits); + iv_len = gcm_param->ulIvLen; + gcm_ctx->gcm_tag_len = tag_len; gcm_ctx->gcm_processed_data_len = 0; @@ -684,10 +644,9 @@ gcm_init_ctx_impl(boolean_t gmac_mode, gcm_ctx_t *gcm_ctx, char *param, } /* * If this is a GCM context, use the MOVBE and the BSWAP - * variants alternately. GMAC contexts code paths do not - * use the MOVBE instruction. + * variants alternately. */ - if (gcm_ctx->gcm_use_avx == B_TRUE && gmac_mode == B_FALSE && + if (gcm_ctx->gcm_use_avx == B_TRUE && zfs_movbe_available() == B_TRUE) { (void) atomic_toggle_boolean_nv( (volatile boolean_t *)&gcm_avx_can_use_movbe); @@ -758,18 +717,6 @@ gcm_alloc_ctx(int kmflag) return (gcm_ctx); } -void * -gmac_alloc_ctx(int kmflag) -{ - gcm_ctx_t *gcm_ctx; - - if ((gcm_ctx = kmem_zalloc(sizeof (gcm_ctx_t), kmflag)) == NULL) - return (NULL); - - gcm_ctx->gcm_flags = GMAC_MODE; - return (gcm_ctx); -} - /* GCM implementation that contains the fastest methods */ static gcm_impl_ops_t gcm_fastest_impl = { .name = "fastest" diff --git a/module/icp/algs/modes/modes.c b/module/icp/algs/modes/modes.c index 6f6649b3b58..786a89f10c9 100644 --- a/module/icp/algs/modes/modes.c +++ b/module/icp/algs/modes/modes.c @@ -126,20 +126,7 @@ crypto_free_mode_ctx(void *ctx) { common_ctx_t *common_ctx = (common_ctx_t *)ctx; - switch (common_ctx->cc_flags & - (ECB_MODE|CBC_MODE|CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) { - case ECB_MODE: - kmem_free(common_ctx, sizeof (ecb_ctx_t)); - break; - - case CBC_MODE: - kmem_free(common_ctx, sizeof (cbc_ctx_t)); - break; - - case CTR_MODE: - kmem_free(common_ctx, sizeof (ctr_ctx_t)); - break; - + switch (common_ctx->cc_flags & (CCM_MODE|GCM_MODE)) { case CCM_MODE: if (((ccm_ctx_t *)ctx)->ccm_pt_buf != NULL) vmem_free(((ccm_ctx_t *)ctx)->ccm_pt_buf, @@ -149,9 +136,12 @@ crypto_free_mode_ctx(void *ctx) break; case GCM_MODE: - case GMAC_MODE: gcm_clear_ctx((gcm_ctx_t *)ctx); kmem_free(ctx, sizeof (gcm_ctx_t)); + break; + + default: + __builtin_unreachable(); } } diff --git a/module/icp/include/aes/aes_impl.h b/module/icp/include/aes/aes_impl.h index 66eb4a6c8fb..d26ced58ff1 100644 --- a/module/icp/include/aes/aes_impl.h +++ b/module/icp/include/aes/aes_impl.h @@ -149,13 +149,8 @@ extern int aes_decrypt_contiguous_blocks(void *ctx, char *data, size_t length, #ifdef _AES_IMPL typedef enum aes_mech_type { - AES_ECB_MECH_INFO_TYPE, /* SUN_CKM_AES_ECB */ - AES_CBC_MECH_INFO_TYPE, /* SUN_CKM_AES_CBC */ - AES_CBC_PAD_MECH_INFO_TYPE, /* SUN_CKM_AES_CBC_PAD */ - AES_CTR_MECH_INFO_TYPE, /* SUN_CKM_AES_CTR */ AES_CCM_MECH_INFO_TYPE, /* SUN_CKM_AES_CCM */ AES_GCM_MECH_INFO_TYPE, /* SUN_CKM_AES_GCM */ - AES_GMAC_MECH_INFO_TYPE /* SUN_CKM_AES_GMAC */ } aes_mech_type_t; #endif /* _AES_IMPL */ diff --git a/module/icp/include/modes/modes.h b/module/icp/include/modes/modes.h index 950c1115f3e..daa0335b5c3 100644 --- a/module/icp/include/modes/modes.h +++ b/module/icp/include/modes/modes.h @@ -45,12 +45,8 @@ extern "C" { extern boolean_t gcm_avx_can_use_movbe; #endif -#define ECB_MODE 0x00000002 -#define CBC_MODE 0x00000004 -#define CTR_MODE 0x00000008 #define CCM_MODE 0x00000010 #define GCM_MODE 0x00000020 -#define GMAC_MODE 0x00000040 /* * cc_keysched: Pointer to key schedule. @@ -76,7 +72,7 @@ extern boolean_t gcm_avx_can_use_movbe; * by the caller, or internally, e.g. an init routine. * If allocated by the latter, then it needs to be freed. * - * ECB_MODE, CBC_MODE, CTR_MODE, or CCM_MODE + * CCM_MODE */ struct common_ctx { void *cc_keysched; @@ -91,57 +87,6 @@ struct common_ctx { typedef struct common_ctx common_ctx_t; -typedef struct ecb_ctx { - struct common_ctx ecb_common; - uint64_t ecb_lastblock[2]; -} ecb_ctx_t; - -#define ecb_keysched ecb_common.cc_keysched -#define ecb_keysched_len ecb_common.cc_keysched_len -#define ecb_iv ecb_common.cc_iv -#define ecb_remainder ecb_common.cc_remainder -#define ecb_remainder_len ecb_common.cc_remainder_len -#define ecb_lastp ecb_common.cc_lastp -#define ecb_copy_to ecb_common.cc_copy_to -#define ecb_flags ecb_common.cc_flags - -typedef struct cbc_ctx { - struct common_ctx cbc_common; - uint64_t cbc_lastblock[2]; -} cbc_ctx_t; - -#define cbc_keysched cbc_common.cc_keysched -#define cbc_keysched_len cbc_common.cc_keysched_len -#define cbc_iv cbc_common.cc_iv -#define cbc_remainder cbc_common.cc_remainder -#define cbc_remainder_len cbc_common.cc_remainder_len -#define cbc_lastp cbc_common.cc_lastp -#define cbc_copy_to cbc_common.cc_copy_to -#define cbc_flags cbc_common.cc_flags - -/* - * ctr_lower_mask Bit-mask for lower 8 bytes of counter block. - * ctr_upper_mask Bit-mask for upper 8 bytes of counter block. - */ -typedef struct ctr_ctx { - struct common_ctx ctr_common; - uint64_t ctr_lower_mask; - uint64_t ctr_upper_mask; - uint32_t ctr_tmp[4]; -} ctr_ctx_t; - -/* - * ctr_cb Counter block. - */ -#define ctr_keysched ctr_common.cc_keysched -#define ctr_keysched_len ctr_common.cc_keysched_len -#define ctr_cb ctr_common.cc_iv -#define ctr_remainder ctr_common.cc_remainder -#define ctr_remainder_len ctr_common.cc_remainder_len -#define ctr_lastp ctr_common.cc_lastp -#define ctr_copy_to ctr_common.cc_copy_to -#define ctr_flags ctr_common.cc_flags - /* * * ccm_mac_len: Stores length of the MAC in CCM mode. @@ -241,27 +186,21 @@ typedef struct gcm_ctx { #define gcm_copy_to gcm_common.cc_copy_to #define gcm_flags gcm_common.cc_flags -#define AES_GMAC_IV_LEN 12 -#define AES_GMAC_TAG_BITS 128 - void gcm_clear_ctx(gcm_ctx_t *ctx); typedef struct aes_ctx { union { - ecb_ctx_t acu_ecb; - cbc_ctx_t acu_cbc; - ctr_ctx_t acu_ctr; ccm_ctx_t acu_ccm; gcm_ctx_t acu_gcm; } acu; } aes_ctx_t; -#define ac_flags acu.acu_ecb.ecb_common.cc_flags -#define ac_remainder_len acu.acu_ecb.ecb_common.cc_remainder_len -#define ac_keysched acu.acu_ecb.ecb_common.cc_keysched -#define ac_keysched_len acu.acu_ecb.ecb_common.cc_keysched_len -#define ac_iv acu.acu_ecb.ecb_common.cc_iv -#define ac_lastp acu.acu_ecb.ecb_common.cc_lastp +#define ac_flags acu.acu_ccm.ccm_common.cc_flags +#define ac_remainder_len acu.acu_ccm.ccm_common.cc_remainder_len +#define ac_keysched acu.acu_ccm.ccm_common.cc_keysched +#define ac_keysched_len acu.acu_ccm.ccm_common.cc_keysched_len +#define ac_iv acu.acu_ccm.ccm_common.cc_iv +#define ac_lastp acu.acu_ccm.ccm_common.cc_lastp #define ac_pt_buf acu.acu_ccm.ccm_pt_buf #define ac_mac_len acu.acu_ccm.ccm_mac_len #define ac_data_len acu.acu_ccm.ccm_data_len @@ -269,27 +208,6 @@ typedef struct aes_ctx { #define ac_processed_data_len acu.acu_ccm.ccm_processed_data_len #define ac_tag_len acu.acu_gcm.gcm_tag_len -extern int ecb_cipher_contiguous_blocks(ecb_ctx_t *, char *, size_t, - crypto_data_t *, size_t, int (*cipher)(const void *, const uint8_t *, - uint8_t *)); - -extern int cbc_encrypt_contiguous_blocks(cbc_ctx_t *, char *, size_t, - crypto_data_t *, size_t, - int (*encrypt)(const void *, const uint8_t *, uint8_t *), - void (*copy_block)(uint8_t *, uint8_t *), - void (*xor_block)(uint8_t *, uint8_t *)); - -extern int cbc_decrypt_contiguous_blocks(cbc_ctx_t *, char *, size_t, - crypto_data_t *, size_t, - int (*decrypt)(const void *, const uint8_t *, uint8_t *), - void (*copy_block)(uint8_t *, uint8_t *), - void (*xor_block)(uint8_t *, uint8_t *)); - -extern int ctr_mode_contiguous_blocks(ctr_ctx_t *, char *, size_t, - crypto_data_t *, size_t, - int (*cipher)(const void *, const uint8_t *, uint8_t *), - void (*xor_block)(uint8_t *, uint8_t *)); - extern int ccm_mode_encrypt_contiguous_blocks(ccm_ctx_t *, char *, size_t, crypto_data_t *, size_t, int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), @@ -332,15 +250,6 @@ extern int gcm_decrypt_final(gcm_ctx_t *, crypto_data_t *, size_t, int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), void (*xor_block)(uint8_t *, uint8_t *)); -extern int ctr_mode_final(ctr_ctx_t *, crypto_data_t *, - int (*encrypt_block)(const void *, const uint8_t *, uint8_t *)); - -extern int cbc_init_ctx(cbc_ctx_t *, char *, size_t, size_t, - void (*copy_block)(uint8_t *, uint64_t *)); - -extern int ctr_init_ctx(ctr_ctx_t *, ulong_t, uint8_t *, - void (*copy_block)(uint8_t *, uint8_t *)); - extern int ccm_init_ctx(ccm_ctx_t *, char *, int, boolean_t, size_t, int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), void (*xor_block)(uint8_t *, uint8_t *)); @@ -350,11 +259,6 @@ extern int gcm_init_ctx(gcm_ctx_t *, char *, size_t, void (*copy_block)(uint8_t *, uint8_t *), void (*xor_block)(uint8_t *, uint8_t *)); -extern int gmac_init_ctx(gcm_ctx_t *, char *, size_t, - int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), - void (*copy_block)(uint8_t *, uint8_t *), - void (*xor_block)(uint8_t *, uint8_t *)); - extern void calculate_ccm_mac(ccm_ctx_t *, uint8_t *, int (*encrypt_block)(const void *, const uint8_t *, uint8_t *)); @@ -364,12 +268,8 @@ extern void crypto_init_ptrs(crypto_data_t *, void **, offset_t *); extern void crypto_get_ptrs(crypto_data_t *, void **, offset_t *, uint8_t **, size_t *, uint8_t **, size_t); -extern void *ecb_alloc_ctx(int); -extern void *cbc_alloc_ctx(int); -extern void *ctr_alloc_ctx(int); extern void *ccm_alloc_ctx(int); extern void *gcm_alloc_ctx(int); -extern void *gmac_alloc_ctx(int); extern void crypto_free_mode_ctx(void *); #ifdef __cplusplus diff --git a/module/icp/io/aes.c b/module/icp/io/aes.c index 522c436497b..a4ef1716710 100644 --- a/module/icp/io/aes.c +++ b/module/icp/io/aes.c @@ -40,18 +40,6 @@ * Mechanism info structure passed to KCF during registration. */ static const crypto_mech_info_t aes_mech_info_tab[] = { - /* AES_ECB */ - {SUN_CKM_AES_ECB, AES_ECB_MECH_INFO_TYPE, - CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | - CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC}, - /* AES_CBC */ - {SUN_CKM_AES_CBC, AES_CBC_MECH_INFO_TYPE, - CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | - CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC}, - /* AES_CTR */ - {SUN_CKM_AES_CTR, AES_CTR_MECH_INFO_TYPE, - CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | - CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC}, /* AES_CCM */ {SUN_CKM_AES_CCM, AES_CCM_MECH_INFO_TYPE, CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | @@ -60,11 +48,6 @@ static const crypto_mech_info_t aes_mech_info_tab[] = { {SUN_CKM_AES_GCM, AES_GCM_MECH_INFO_TYPE, CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC}, - /* AES_GMAC */ - {SUN_CKM_AES_GMAC, AES_GMAC_MECH_INFO_TYPE, - CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | - CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC | - CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, }; static int aes_encrypt_init(crypto_ctx_t *, crypto_mechanism_t *, @@ -103,20 +86,6 @@ static const crypto_cipher_ops_t aes_cipher_ops = { .decrypt_atomic = aes_decrypt_atomic }; -static int aes_mac_atomic(crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, - crypto_data_t *, crypto_spi_ctx_template_t); -static int aes_mac_verify_atomic(crypto_mechanism_t *, crypto_key_t *, - crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t); - -static const crypto_mac_ops_t aes_mac_ops = { - .mac_init = NULL, - .mac = NULL, - .mac_update = NULL, - .mac_final = NULL, - .mac_atomic = aes_mac_atomic, - .mac_verify_atomic = aes_mac_verify_atomic -}; - static int aes_create_ctx_template(crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t *, size_t *); static int aes_free_context(crypto_ctx_t *); @@ -129,7 +98,7 @@ static const crypto_ctx_ops_t aes_ctx_ops = { static const crypto_ops_t aes_crypto_ops = { NULL, &aes_cipher_ops, - &aes_mac_ops, + NULL, &aes_ctx_ops, }; @@ -141,7 +110,6 @@ static const crypto_provider_info_t aes_prov_info = { }; static crypto_kcf_provider_handle_t aes_prov_handle = 0; -static crypto_data_t null_crypto_data = { CRYPTO_DATA_RAW }; int aes_mod_init(void) @@ -181,18 +149,6 @@ aes_check_mech_param(crypto_mechanism_t *mechanism, aes_ctx_t **ctx) int rv = CRYPTO_SUCCESS; switch (mechanism->cm_type) { - case AES_ECB_MECH_INFO_TYPE: - param_required = B_FALSE; - alloc_fun = ecb_alloc_ctx; - break; - case AES_CBC_MECH_INFO_TYPE: - param_len = AES_BLOCK_LEN; - alloc_fun = cbc_alloc_ctx; - break; - case AES_CTR_MECH_INFO_TYPE: - param_len = sizeof (CK_AES_CTR_PARAMS); - alloc_fun = ctr_alloc_ctx; - break; case AES_CCM_MECH_INFO_TYPE: param_len = sizeof (CK_AES_CCM_PARAMS); alloc_fun = ccm_alloc_ctx; @@ -201,13 +157,8 @@ aes_check_mech_param(crypto_mechanism_t *mechanism, aes_ctx_t **ctx) param_len = sizeof (CK_AES_GCM_PARAMS); alloc_fun = gcm_alloc_ctx; break; - case AES_GMAC_MECH_INFO_TYPE: - param_len = sizeof (CK_AES_GMAC_PARAMS); - alloc_fun = gmac_alloc_ctx; - break; default: - rv = CRYPTO_MECHANISM_INVALID; - return (rv); + __builtin_unreachable(); } if (param_required && mechanism->cm_param != NULL && mechanism->cm_param_len != param_len) { @@ -282,22 +233,6 @@ aes_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, return (CRYPTO_SUCCESS); } -static void -aes_copy_block64(uint8_t *in, uint64_t *out) -{ - if (IS_P2ALIGNED(in, sizeof (uint64_t))) { - /* LINTED: pointer alignment */ - out[0] = *(uint64_t *)&in[0]; - /* LINTED: pointer alignment */ - out[1] = *(uint64_t *)&in[8]; - } else { - uint8_t *iv8 = (uint8_t *)&out[0]; - - AES_COPY_BLOCK(in, iv8); - } -} - - static int aes_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext, crypto_data_t *ciphertext) @@ -310,35 +245,21 @@ aes_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext, ASSERT(ctx->cc_provider_private != NULL); aes_ctx = ctx->cc_provider_private; - /* - * For block ciphers, plaintext must be a multiple of AES block size. - * This test is only valid for ciphers whose blocksize is a power of 2. - */ - if (((aes_ctx->ac_flags & (CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) - == 0) && (plaintext->cd_length & (AES_BLOCK_LEN - 1)) != 0) - return (CRYPTO_DATA_LEN_RANGE); - ASSERT(ciphertext != NULL); /* * We need to just return the length needed to store the output. * We should not destroy the context for the following case. */ - switch (aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|GMAC_MODE)) { + switch (aes_ctx->ac_flags & (CCM_MODE|GCM_MODE)) { case CCM_MODE: length_needed = plaintext->cd_length + aes_ctx->ac_mac_len; break; case GCM_MODE: length_needed = plaintext->cd_length + aes_ctx->ac_tag_len; break; - case GMAC_MODE: - if (plaintext->cd_length != 0) - return (CRYPTO_ARGUMENTS_BAD); - - length_needed = aes_ctx->ac_tag_len; - break; default: - length_needed = plaintext->cd_length; + __builtin_unreachable(); } if (ciphertext->cd_length < length_needed) { @@ -382,7 +303,7 @@ aes_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext, ciphertext->cd_offset - saved_offset; } ciphertext->cd_offset = saved_offset; - } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) { + } else if (aes_ctx->ac_flags & GCM_MODE) { /* * gcm_encrypt_final() will compute the MAC and append * it to existing ciphertext. So, need to adjust the left over @@ -426,15 +347,6 @@ aes_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext, ASSERT(ctx->cc_provider_private != NULL); aes_ctx = ctx->cc_provider_private; - /* - * For block ciphers, plaintext must be a multiple of AES block size. - * This test is only valid for ciphers whose blocksize is a power of 2. - */ - if (((aes_ctx->ac_flags & (CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) - == 0) && (ciphertext->cd_length & (AES_BLOCK_LEN - 1)) != 0) { - return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE); - } - ASSERT(plaintext != NULL); /* @@ -443,23 +355,16 @@ aes_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext, * * CCM: plaintext is MAC len smaller than cipher text * GCM: plaintext is TAG len smaller than cipher text - * GMAC: plaintext length must be zero */ - switch (aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|GMAC_MODE)) { + switch (aes_ctx->ac_flags & (CCM_MODE|GCM_MODE)) { case CCM_MODE: length_needed = aes_ctx->ac_processed_data_len; break; case GCM_MODE: length_needed = ciphertext->cd_length - aes_ctx->ac_tag_len; break; - case GMAC_MODE: - if (plaintext->cd_length != 0) - return (CRYPTO_ARGUMENTS_BAD); - - length_needed = 0; - break; default: - length_needed = ciphertext->cd_length; + __builtin_unreachable(); } if (plaintext->cd_length < length_needed) { @@ -499,7 +404,7 @@ aes_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext, } plaintext->cd_offset = saved_offset; - } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) { + } else if (aes_ctx->ac_flags & GCM_MODE) { /* order of following 2 lines MUST not be reversed */ plaintext->cd_offset = plaintext->cd_length; plaintext->cd_length = saved_length - plaintext->cd_length; @@ -571,17 +476,6 @@ aes_encrypt_update(crypto_ctx_t *ctx, crypto_data_t *plaintext, ret = CRYPTO_ARGUMENTS_BAD; } - /* - * Since AES counter mode is a stream cipher, we call - * ctr_mode_final() to pick up any remaining bytes. - * It is an internal function that does not destroy - * the context like *normal* final routines. - */ - if ((aes_ctx->ac_flags & CTR_MODE) && (aes_ctx->ac_remainder_len > 0)) { - ret = ctr_mode_final((ctr_ctx_t *)aes_ctx, - ciphertext, aes_encrypt_block); - } - if (ret == CRYPTO_SUCCESS) { if (plaintext != ciphertext) ciphertext->cd_length = @@ -600,32 +494,13 @@ aes_decrypt_update(crypto_ctx_t *ctx, crypto_data_t *ciphertext, crypto_data_t *plaintext) { off_t saved_offset; - size_t saved_length, out_len; + size_t saved_length; int ret = CRYPTO_SUCCESS; - aes_ctx_t *aes_ctx; ASSERT(ctx->cc_provider_private != NULL); - aes_ctx = ctx->cc_provider_private; ASSERT(plaintext != NULL); - /* - * Compute number of bytes that will hold the plaintext. - * This is not necessary for CCM, GCM, and GMAC since these - * mechanisms never return plaintext for update operations. - */ - if ((aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) { - out_len = aes_ctx->ac_remainder_len; - out_len += ciphertext->cd_length; - out_len &= ~(AES_BLOCK_LEN - 1); - - /* return length needed to store the output */ - if (plaintext->cd_length < out_len) { - plaintext->cd_length = out_len; - return (CRYPTO_BUFFER_TOO_SMALL); - } - } - saved_offset = plaintext->cd_offset; saved_length = plaintext->cd_length; @@ -645,19 +520,6 @@ aes_decrypt_update(crypto_ctx_t *ctx, crypto_data_t *ciphertext, ret = CRYPTO_ARGUMENTS_BAD; } - /* - * Since AES counter mode is a stream cipher, we call - * ctr_mode_final() to pick up any remaining bytes. - * It is an internal function that does not destroy - * the context like *normal* final routines. - */ - if ((aes_ctx->ac_flags & CTR_MODE) && (aes_ctx->ac_remainder_len > 0)) { - ret = ctr_mode_final((ctr_ctx_t *)aes_ctx, plaintext, - aes_encrypt_block); - if (ret == CRYPTO_DATA_LEN_RANGE) - ret = CRYPTO_ENCRYPTED_DATA_LEN_RANGE; - } - if (ret == CRYPTO_SUCCESS) { if (ciphertext != plaintext) plaintext->cd_length = @@ -685,20 +547,13 @@ aes_encrypt_final(crypto_ctx_t *ctx, crypto_data_t *data) return (CRYPTO_ARGUMENTS_BAD); } - if (aes_ctx->ac_flags & CTR_MODE) { - if (aes_ctx->ac_remainder_len > 0) { - ret = ctr_mode_final((ctr_ctx_t *)aes_ctx, data, - aes_encrypt_block); - if (ret != CRYPTO_SUCCESS) - return (ret); - } - } else if (aes_ctx->ac_flags & CCM_MODE) { + if (aes_ctx->ac_flags & CCM_MODE) { ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, data, AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); if (ret != CRYPTO_SUCCESS) { return (ret); } - } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) { + } else if (aes_ctx->ac_flags & GCM_MODE) { size_t saved_offset = data->cd_offset; ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, data, @@ -709,16 +564,6 @@ aes_encrypt_final(crypto_ctx_t *ctx, crypto_data_t *data) } data->cd_length = data->cd_offset - saved_offset; data->cd_offset = saved_offset; - } else { - /* - * There must be no unprocessed plaintext. - * This happens if the length of the last data is - * not a multiple of the AES block length. - */ - if (aes_ctx->ac_remainder_len > 0) { - return (CRYPTO_DATA_LEN_RANGE); - } - data->cd_length = 0; } (void) aes_free_context(ctx); @@ -747,18 +592,8 @@ aes_decrypt_final(crypto_ctx_t *ctx, crypto_data_t *data) * This happens if the length of the last ciphertext is * not a multiple of the AES block length. */ - if (aes_ctx->ac_remainder_len > 0) { - if ((aes_ctx->ac_flags & CTR_MODE) == 0) - return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE); - else { - ret = ctr_mode_final((ctr_ctx_t *)aes_ctx, data, - aes_encrypt_block); - if (ret == CRYPTO_DATA_LEN_RANGE) - ret = CRYPTO_ENCRYPTED_DATA_LEN_RANGE; - if (ret != CRYPTO_SUCCESS) - return (ret); - } - } + if (aes_ctx->ac_remainder_len > 0) + return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE); if (aes_ctx->ac_flags & CCM_MODE) { /* @@ -788,7 +623,7 @@ aes_decrypt_final(crypto_ctx_t *ctx, crypto_data_t *data) if (ret != CRYPTO_SUCCESS) { return (ret); } - } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) { + } else if (aes_ctx->ac_flags & GCM_MODE) { /* * This is where all the plaintext is returned, make sure * the plaintext buffer is big enough @@ -818,10 +653,6 @@ aes_decrypt_final(crypto_ctx_t *ctx, crypto_data_t *data) } - if ((aes_ctx->ac_flags & (CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) { - data->cd_length = 0; - } - (void) aes_free_context(ctx); return (CRYPTO_SUCCESS); @@ -842,21 +673,6 @@ aes_encrypt_atomic(crypto_mechanism_t *mechanism, ASSERT(ciphertext != NULL); - /* - * CTR, CCM, GCM, and GMAC modes do not require that plaintext - * be a multiple of AES block size. - */ - switch (mechanism->cm_type) { - case AES_CTR_MECH_INFO_TYPE: - case AES_CCM_MECH_INFO_TYPE: - case AES_GCM_MECH_INFO_TYPE: - case AES_GMAC_MECH_INFO_TYPE: - break; - default: - if ((plaintext->cd_length & (AES_BLOCK_LEN - 1)) != 0) - return (CRYPTO_DATA_LEN_RANGE); - } - if ((ret = aes_check_mech_param(mechanism, NULL)) != CRYPTO_SUCCESS) return (ret); @@ -869,15 +685,11 @@ aes_encrypt_atomic(crypto_mechanism_t *mechanism, case AES_CCM_MECH_INFO_TYPE: length_needed = plaintext->cd_length + aes_ctx.ac_mac_len; break; - case AES_GMAC_MECH_INFO_TYPE: - if (plaintext->cd_length != 0) - return (CRYPTO_ARGUMENTS_BAD); - zfs_fallthrough; case AES_GCM_MECH_INFO_TYPE: length_needed = plaintext->cd_length + aes_ctx.ac_tag_len; break; default: - length_needed = plaintext->cd_length; + __builtin_unreachable(); } /* return size of buffer needed to store output */ @@ -914,21 +726,13 @@ aes_encrypt_atomic(crypto_mechanism_t *mechanism, if (ret != CRYPTO_SUCCESS) goto out; ASSERT(aes_ctx.ac_remainder_len == 0); - } else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE || - mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) { + } else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE) { ret = gcm_encrypt_final((gcm_ctx_t *)&aes_ctx, ciphertext, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, aes_xor_block); if (ret != CRYPTO_SUCCESS) goto out; ASSERT(aes_ctx.ac_remainder_len == 0); - } else if (mechanism->cm_type == AES_CTR_MECH_INFO_TYPE) { - if (aes_ctx.ac_remainder_len > 0) { - ret = ctr_mode_final((ctr_ctx_t *)&aes_ctx, - ciphertext, aes_encrypt_block); - if (ret != CRYPTO_SUCCESS) - goto out; - } } else { ASSERT(aes_ctx.ac_remainder_len == 0); } @@ -947,7 +751,7 @@ out: memset(aes_ctx.ac_keysched, 0, aes_ctx.ac_keysched_len); kmem_free(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len); } - if (aes_ctx.ac_flags & (GCM_MODE|GMAC_MODE)) { + if (aes_ctx.ac_flags & GCM_MODE) { gcm_clear_ctx((gcm_ctx_t *)&aes_ctx); } return (ret); @@ -968,21 +772,6 @@ aes_decrypt_atomic(crypto_mechanism_t *mechanism, ASSERT(plaintext != NULL); - /* - * CCM, GCM, CTR, and GMAC modes do not require that ciphertext - * be a multiple of AES block size. - */ - switch (mechanism->cm_type) { - case AES_CTR_MECH_INFO_TYPE: - case AES_CCM_MECH_INFO_TYPE: - case AES_GCM_MECH_INFO_TYPE: - case AES_GMAC_MECH_INFO_TYPE: - break; - default: - if ((ciphertext->cd_length & (AES_BLOCK_LEN - 1)) != 0) - return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE); - } - if ((ret = aes_check_mech_param(mechanism, NULL)) != CRYPTO_SUCCESS) return (ret); @@ -998,13 +787,8 @@ aes_decrypt_atomic(crypto_mechanism_t *mechanism, case AES_GCM_MECH_INFO_TYPE: length_needed = ciphertext->cd_length - aes_ctx.ac_tag_len; break; - case AES_GMAC_MECH_INFO_TYPE: - if (plaintext->cd_length != 0) - return (CRYPTO_ARGUMENTS_BAD); - length_needed = 0; - break; default: - length_needed = ciphertext->cd_length; + __builtin_unreachable(); } /* return size of buffer needed to store output */ @@ -1050,8 +834,7 @@ aes_decrypt_atomic(crypto_mechanism_t *mechanism, } else { plaintext->cd_length = saved_length; } - } else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE || - mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) { + } else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE) { ret = gcm_decrypt_final((gcm_ctx_t *)&aes_ctx, plaintext, AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); @@ -1063,24 +846,8 @@ aes_decrypt_atomic(crypto_mechanism_t *mechanism, } else { plaintext->cd_length = saved_length; } - } else if (mechanism->cm_type != AES_CTR_MECH_INFO_TYPE) { - ASSERT(aes_ctx.ac_remainder_len == 0); - if (ciphertext != plaintext) - plaintext->cd_length = - plaintext->cd_offset - saved_offset; - } else { - if (aes_ctx.ac_remainder_len > 0) { - ret = ctr_mode_final((ctr_ctx_t *)&aes_ctx, - plaintext, aes_encrypt_block); - if (ret == CRYPTO_DATA_LEN_RANGE) - ret = CRYPTO_ENCRYPTED_DATA_LEN_RANGE; - if (ret != CRYPTO_SUCCESS) - goto out; - } - if (ciphertext != plaintext) - plaintext->cd_length = - plaintext->cd_offset - saved_offset; - } + } else + __builtin_unreachable(); } else { plaintext->cd_length = saved_length; } @@ -1096,7 +863,7 @@ out: if (aes_ctx.ac_pt_buf != NULL) { vmem_free(aes_ctx.ac_pt_buf, aes_ctx.ac_data_len); } - } else if (aes_ctx.ac_flags & (GCM_MODE|GMAC_MODE)) { + } else if (aes_ctx.ac_flags & GCM_MODE) { gcm_clear_ctx((gcm_ctx_t *)&aes_ctx); } @@ -1114,12 +881,8 @@ aes_create_ctx_template(crypto_mechanism_t *mechanism, crypto_key_t *key, size_t size; int rv; - if (mechanism->cm_type != AES_ECB_MECH_INFO_TYPE && - mechanism->cm_type != AES_CBC_MECH_INFO_TYPE && - mechanism->cm_type != AES_CTR_MECH_INFO_TYPE && - mechanism->cm_type != AES_CCM_MECH_INFO_TYPE && - mechanism->cm_type != AES_GCM_MECH_INFO_TYPE && - mechanism->cm_type != AES_GMAC_MECH_INFO_TYPE) + if (mechanism->cm_type != AES_CCM_MECH_INFO_TYPE && + mechanism->cm_type != AES_GCM_MECH_INFO_TYPE) return (CRYPTO_MECHANISM_INVALID); if ((keysched = aes_alloc_keysched(&size, KM_SLEEP)) == NULL) { @@ -1193,22 +956,6 @@ aes_common_init_ctx(aes_ctx_t *aes_ctx, crypto_spi_ctx_template_t *template, aes_ctx->ac_keysched = keysched; switch (mechanism->cm_type) { - case AES_CBC_MECH_INFO_TYPE: - rv = cbc_init_ctx((cbc_ctx_t *)aes_ctx, mechanism->cm_param, - mechanism->cm_param_len, AES_BLOCK_LEN, aes_copy_block64); - break; - case AES_CTR_MECH_INFO_TYPE: { - CK_AES_CTR_PARAMS *pp; - - if (mechanism->cm_param == NULL || - mechanism->cm_param_len != sizeof (CK_AES_CTR_PARAMS)) { - return (CRYPTO_MECHANISM_PARAM_INVALID); - } - pp = (CK_AES_CTR_PARAMS *)(void *)mechanism->cm_param; - rv = ctr_init_ctx((ctr_ctx_t *)aes_ctx, pp->ulCounterBits, - pp->cb, aes_copy_block); - break; - } case AES_CCM_MECH_INFO_TYPE: if (mechanism->cm_param == NULL || mechanism->cm_param_len != sizeof (CK_AES_CCM_PARAMS)) { @@ -1227,17 +974,6 @@ aes_common_init_ctx(aes_ctx_t *aes_ctx, crypto_spi_ctx_template_t *template, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, aes_xor_block); break; - case AES_GMAC_MECH_INFO_TYPE: - if (mechanism->cm_param == NULL || - mechanism->cm_param_len != sizeof (CK_AES_GMAC_PARAMS)) { - return (CRYPTO_MECHANISM_PARAM_INVALID); - } - rv = gmac_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param, - AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, - aes_xor_block); - break; - case AES_ECB_MECH_INFO_TYPE: - aes_ctx->ac_flags |= ECB_MODE; } if (rv != CRYPTO_SUCCESS) { @@ -1249,75 +985,3 @@ aes_common_init_ctx(aes_ctx_t *aes_ctx, crypto_spi_ctx_template_t *template, return (rv); } - -static int -process_gmac_mech(crypto_mechanism_t *mech, crypto_data_t *data, - CK_AES_GCM_PARAMS *gcm_params) -{ - /* LINTED: pointer alignment */ - CK_AES_GMAC_PARAMS *params = (CK_AES_GMAC_PARAMS *)mech->cm_param; - - if (mech->cm_type != AES_GMAC_MECH_INFO_TYPE) - return (CRYPTO_MECHANISM_INVALID); - - if (mech->cm_param_len != sizeof (CK_AES_GMAC_PARAMS)) - return (CRYPTO_MECHANISM_PARAM_INVALID); - - if (params->pIv == NULL) - return (CRYPTO_MECHANISM_PARAM_INVALID); - - gcm_params->pIv = params->pIv; - gcm_params->ulIvLen = AES_GMAC_IV_LEN; - gcm_params->ulTagBits = AES_GMAC_TAG_BITS; - - if (data == NULL) - return (CRYPTO_SUCCESS); - - if (data->cd_format != CRYPTO_DATA_RAW) - return (CRYPTO_ARGUMENTS_BAD); - - gcm_params->pAAD = (uchar_t *)data->cd_raw.iov_base; - gcm_params->ulAADLen = data->cd_length; - return (CRYPTO_SUCCESS); -} - -static int -aes_mac_atomic(crypto_mechanism_t *mechanism, - crypto_key_t *key, crypto_data_t *data, crypto_data_t *mac, - crypto_spi_ctx_template_t template) -{ - CK_AES_GCM_PARAMS gcm_params; - crypto_mechanism_t gcm_mech; - int rv; - - if ((rv = process_gmac_mech(mechanism, data, &gcm_params)) - != CRYPTO_SUCCESS) - return (rv); - - gcm_mech.cm_type = AES_GCM_MECH_INFO_TYPE; - gcm_mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS); - gcm_mech.cm_param = (char *)&gcm_params; - - return (aes_encrypt_atomic(&gcm_mech, - key, &null_crypto_data, mac, template)); -} - -static int -aes_mac_verify_atomic(crypto_mechanism_t *mechanism, crypto_key_t *key, - crypto_data_t *data, crypto_data_t *mac, crypto_spi_ctx_template_t template) -{ - CK_AES_GCM_PARAMS gcm_params; - crypto_mechanism_t gcm_mech; - int rv; - - if ((rv = process_gmac_mech(mechanism, data, &gcm_params)) - != CRYPTO_SUCCESS) - return (rv); - - gcm_mech.cm_type = AES_GCM_MECH_INFO_TYPE; - gcm_mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS); - gcm_mech.cm_param = (char *)&gcm_params; - - return (aes_decrypt_atomic(&gcm_mech, - key, mac, &null_crypto_data, template)); -} From 4ed91dc26e63cd18817f7bd91d1590dd6514394b Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Sat, 18 May 2024 22:17:36 +1000 Subject: [PATCH 04/42] icp: remove unusued incremental cipher methods Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Reviewed-by: Brian Behlendorf Signed-off-by: Rob Norris Closes #16209 --- module/icp/core/kcf_mech_tabs.c | 3 +- module/icp/include/sys/crypto/spi.h | 21 -- module/icp/io/aes.c | 493 +--------------------------- 3 files changed, 3 insertions(+), 514 deletions(-) diff --git a/module/icp/core/kcf_mech_tabs.c b/module/icp/core/kcf_mech_tabs.c index 41705e84bc4..b6e69376980 100644 --- a/module/icp/core/kcf_mech_tabs.c +++ b/module/icp/core/kcf_mech_tabs.c @@ -222,8 +222,7 @@ kcf_add_mech_provider(short mech_indx, if (fg & CRYPTO_FG_DIGEST || fg & CRYPTO_FG_DIGEST_ATOMIC) class = KCF_DIGEST_CLASS; - else if (fg & CRYPTO_FG_ENCRYPT || fg & CRYPTO_FG_DECRYPT || - fg & CRYPTO_FG_ENCRYPT_ATOMIC || + else if (fg & CRYPTO_FG_ENCRYPT_ATOMIC || fg & CRYPTO_FG_DECRYPT_ATOMIC) class = KCF_CIPHER_CLASS; else if (fg & CRYPTO_FG_MAC || fg & CRYPTO_FG_MAC_ATOMIC) diff --git a/module/icp/include/sys/crypto/spi.h b/module/icp/include/sys/crypto/spi.h index 63dfce7957a..9bcb62ac529 100644 --- a/module/icp/include/sys/crypto/spi.h +++ b/module/icp/include/sys/crypto/spi.h @@ -89,27 +89,8 @@ typedef struct crypto_digest_ops { * with the kernel using crypto_register_provider(9F). */ typedef struct crypto_cipher_ops { - int (*encrypt_init)(crypto_ctx_t *, - crypto_mechanism_t *, crypto_key_t *, - crypto_spi_ctx_template_t); - int (*encrypt)(crypto_ctx_t *, - crypto_data_t *, crypto_data_t *); - int (*encrypt_update)(crypto_ctx_t *, - crypto_data_t *, crypto_data_t *); - int (*encrypt_final)(crypto_ctx_t *, - crypto_data_t *); int (*encrypt_atomic)(crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t); - - int (*decrypt_init)(crypto_ctx_t *, - crypto_mechanism_t *, crypto_key_t *, - crypto_spi_ctx_template_t); - int (*decrypt)(crypto_ctx_t *, - crypto_data_t *, crypto_data_t *); - int (*decrypt_update)(crypto_ctx_t *, - crypto_data_t *, crypto_data_t *); - int (*decrypt_final)(crypto_ctx_t *, - crypto_data_t *); int (*decrypt_atomic)(crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t); } __no_const crypto_cipher_ops_t; @@ -172,8 +153,6 @@ typedef struct crypto_ops { typedef uint32_t crypto_func_group_t; -#define CRYPTO_FG_ENCRYPT 0x00000001 /* encrypt_init() */ -#define CRYPTO_FG_DECRYPT 0x00000002 /* decrypt_init() */ #define CRYPTO_FG_DIGEST 0x00000004 /* digest_init() */ #define CRYPTO_FG_MAC 0x00001000 /* mac_init() */ #define CRYPTO_FG_ENCRYPT_ATOMIC 0x00008000 /* encrypt_atomic() */ diff --git a/module/icp/io/aes.c b/module/icp/io/aes.c index a4ef1716710..a68a878b6ab 100644 --- a/module/icp/io/aes.c +++ b/module/icp/io/aes.c @@ -42,47 +42,23 @@ static const crypto_mech_info_t aes_mech_info_tab[] = { /* AES_CCM */ {SUN_CKM_AES_CCM, AES_CCM_MECH_INFO_TYPE, - CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | - CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC}, + CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC}, /* AES_GCM */ {SUN_CKM_AES_GCM, AES_GCM_MECH_INFO_TYPE, - CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | - CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC}, + CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC}, }; -static int aes_encrypt_init(crypto_ctx_t *, crypto_mechanism_t *, - crypto_key_t *, crypto_spi_ctx_template_t); -static int aes_decrypt_init(crypto_ctx_t *, crypto_mechanism_t *, - crypto_key_t *, crypto_spi_ctx_template_t); -static int aes_common_init(crypto_ctx_t *, crypto_mechanism_t *, - crypto_key_t *, crypto_spi_ctx_template_t, boolean_t); static int aes_common_init_ctx(aes_ctx_t *, crypto_spi_ctx_template_t *, crypto_mechanism_t *, crypto_key_t *, int, boolean_t); -static int aes_encrypt_final(crypto_ctx_t *, crypto_data_t *); -static int aes_decrypt_final(crypto_ctx_t *, crypto_data_t *); -static int aes_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *); -static int aes_encrypt_update(crypto_ctx_t *, crypto_data_t *, - crypto_data_t *); static int aes_encrypt_atomic(crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t); -static int aes_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *); -static int aes_decrypt_update(crypto_ctx_t *, crypto_data_t *, - crypto_data_t *); static int aes_decrypt_atomic(crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t); static const crypto_cipher_ops_t aes_cipher_ops = { - .encrypt_init = aes_encrypt_init, - .encrypt = aes_encrypt, - .encrypt_update = aes_encrypt_update, - .encrypt_final = aes_encrypt_final, .encrypt_atomic = aes_encrypt_atomic, - .decrypt_init = aes_decrypt_init, - .decrypt = aes_decrypt, - .decrypt_update = aes_decrypt_update, - .decrypt_final = aes_decrypt_final, .decrypt_atomic = aes_decrypt_atomic }; @@ -190,474 +166,9 @@ init_keysched(crypto_key_t *key, void *newbie) return (CRYPTO_SUCCESS); } -static int -aes_encrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, - crypto_key_t *key, crypto_spi_ctx_template_t template) -{ - return (aes_common_init(ctx, mechanism, key, template, B_TRUE)); -} - -static int -aes_decrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, - crypto_key_t *key, crypto_spi_ctx_template_t template) -{ - return (aes_common_init(ctx, mechanism, key, template, B_FALSE)); -} - - - /* * KCF software provider encrypt entry points. */ -static int -aes_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, - crypto_key_t *key, crypto_spi_ctx_template_t template, - boolean_t is_encrypt_init) -{ - aes_ctx_t *aes_ctx; - int rv; - - if ((rv = aes_check_mech_param(mechanism, &aes_ctx)) - != CRYPTO_SUCCESS) - return (rv); - - rv = aes_common_init_ctx(aes_ctx, template, mechanism, key, KM_SLEEP, - is_encrypt_init); - if (rv != CRYPTO_SUCCESS) { - crypto_free_mode_ctx(aes_ctx); - return (rv); - } - - ctx->cc_provider_private = aes_ctx; - - return (CRYPTO_SUCCESS); -} - -static int -aes_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext, - crypto_data_t *ciphertext) -{ - int ret = CRYPTO_FAILED; - - aes_ctx_t *aes_ctx; - size_t saved_length, saved_offset, length_needed; - - ASSERT(ctx->cc_provider_private != NULL); - aes_ctx = ctx->cc_provider_private; - - ASSERT(ciphertext != NULL); - - /* - * We need to just return the length needed to store the output. - * We should not destroy the context for the following case. - */ - switch (aes_ctx->ac_flags & (CCM_MODE|GCM_MODE)) { - case CCM_MODE: - length_needed = plaintext->cd_length + aes_ctx->ac_mac_len; - break; - case GCM_MODE: - length_needed = plaintext->cd_length + aes_ctx->ac_tag_len; - break; - default: - __builtin_unreachable(); - } - - if (ciphertext->cd_length < length_needed) { - ciphertext->cd_length = length_needed; - return (CRYPTO_BUFFER_TOO_SMALL); - } - - saved_length = ciphertext->cd_length; - saved_offset = ciphertext->cd_offset; - - /* - * Do an update on the specified input data. - */ - ret = aes_encrypt_update(ctx, plaintext, ciphertext); - if (ret != CRYPTO_SUCCESS) { - return (ret); - } - - /* - * For CCM mode, aes_ccm_encrypt_final() will take care of any - * left-over unprocessed data, and compute the MAC - */ - if (aes_ctx->ac_flags & CCM_MODE) { - /* - * ccm_encrypt_final() will compute the MAC and append - * it to existing ciphertext. So, need to adjust the left over - * length value accordingly - */ - - /* order of following 2 lines MUST not be reversed */ - ciphertext->cd_offset = ciphertext->cd_length; - ciphertext->cd_length = saved_length - ciphertext->cd_length; - ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, ciphertext, - AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); - if (ret != CRYPTO_SUCCESS) { - return (ret); - } - - if (plaintext != ciphertext) { - ciphertext->cd_length = - ciphertext->cd_offset - saved_offset; - } - ciphertext->cd_offset = saved_offset; - } else if (aes_ctx->ac_flags & GCM_MODE) { - /* - * gcm_encrypt_final() will compute the MAC and append - * it to existing ciphertext. So, need to adjust the left over - * length value accordingly - */ - - /* order of following 2 lines MUST not be reversed */ - ciphertext->cd_offset = ciphertext->cd_length; - ciphertext->cd_length = saved_length - ciphertext->cd_length; - ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, ciphertext, - AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, - aes_xor_block); - if (ret != CRYPTO_SUCCESS) { - return (ret); - } - - if (plaintext != ciphertext) { - ciphertext->cd_length = - ciphertext->cd_offset - saved_offset; - } - ciphertext->cd_offset = saved_offset; - } - - ASSERT(aes_ctx->ac_remainder_len == 0); - (void) aes_free_context(ctx); - - return (ret); -} - - -static int -aes_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext, - crypto_data_t *plaintext) -{ - int ret = CRYPTO_FAILED; - - aes_ctx_t *aes_ctx; - off_t saved_offset; - size_t saved_length, length_needed; - - ASSERT(ctx->cc_provider_private != NULL); - aes_ctx = ctx->cc_provider_private; - - ASSERT(plaintext != NULL); - - /* - * Return length needed to store the output. - * Do not destroy context when plaintext buffer is too small. - * - * CCM: plaintext is MAC len smaller than cipher text - * GCM: plaintext is TAG len smaller than cipher text - */ - switch (aes_ctx->ac_flags & (CCM_MODE|GCM_MODE)) { - case CCM_MODE: - length_needed = aes_ctx->ac_processed_data_len; - break; - case GCM_MODE: - length_needed = ciphertext->cd_length - aes_ctx->ac_tag_len; - break; - default: - __builtin_unreachable(); - } - - if (plaintext->cd_length < length_needed) { - plaintext->cd_length = length_needed; - return (CRYPTO_BUFFER_TOO_SMALL); - } - - saved_offset = plaintext->cd_offset; - saved_length = plaintext->cd_length; - - /* - * Do an update on the specified input data. - */ - ret = aes_decrypt_update(ctx, ciphertext, plaintext); - if (ret != CRYPTO_SUCCESS) { - goto cleanup; - } - - if (aes_ctx->ac_flags & CCM_MODE) { - ASSERT(aes_ctx->ac_processed_data_len == aes_ctx->ac_data_len); - ASSERT(aes_ctx->ac_processed_mac_len == aes_ctx->ac_mac_len); - - /* order of following 2 lines MUST not be reversed */ - plaintext->cd_offset = plaintext->cd_length; - plaintext->cd_length = saved_length - plaintext->cd_length; - - ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, plaintext, - AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, - aes_xor_block); - if (ret == CRYPTO_SUCCESS) { - if (plaintext != ciphertext) { - plaintext->cd_length = - plaintext->cd_offset - saved_offset; - } - } else { - plaintext->cd_length = saved_length; - } - - plaintext->cd_offset = saved_offset; - } else if (aes_ctx->ac_flags & GCM_MODE) { - /* order of following 2 lines MUST not be reversed */ - plaintext->cd_offset = plaintext->cd_length; - plaintext->cd_length = saved_length - plaintext->cd_length; - - ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, plaintext, - AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); - if (ret == CRYPTO_SUCCESS) { - if (plaintext != ciphertext) { - plaintext->cd_length = - plaintext->cd_offset - saved_offset; - } - } else { - plaintext->cd_length = saved_length; - } - - plaintext->cd_offset = saved_offset; - } - - ASSERT(aes_ctx->ac_remainder_len == 0); - -cleanup: - (void) aes_free_context(ctx); - - return (ret); -} - - -static int -aes_encrypt_update(crypto_ctx_t *ctx, crypto_data_t *plaintext, - crypto_data_t *ciphertext) -{ - off_t saved_offset; - size_t saved_length, out_len; - int ret = CRYPTO_SUCCESS; - aes_ctx_t *aes_ctx; - - ASSERT(ctx->cc_provider_private != NULL); - aes_ctx = ctx->cc_provider_private; - - ASSERT(ciphertext != NULL); - - /* compute number of bytes that will hold the ciphertext */ - out_len = aes_ctx->ac_remainder_len; - out_len += plaintext->cd_length; - out_len &= ~(AES_BLOCK_LEN - 1); - - /* return length needed to store the output */ - if (ciphertext->cd_length < out_len) { - ciphertext->cd_length = out_len; - return (CRYPTO_BUFFER_TOO_SMALL); - } - - saved_offset = ciphertext->cd_offset; - saved_length = ciphertext->cd_length; - - /* - * Do the AES update on the specified input data. - */ - switch (plaintext->cd_format) { - case CRYPTO_DATA_RAW: - ret = crypto_update_iov(ctx->cc_provider_private, - plaintext, ciphertext, aes_encrypt_contiguous_blocks); - break; - case CRYPTO_DATA_UIO: - ret = crypto_update_uio(ctx->cc_provider_private, - plaintext, ciphertext, aes_encrypt_contiguous_blocks); - break; - default: - ret = CRYPTO_ARGUMENTS_BAD; - } - - if (ret == CRYPTO_SUCCESS) { - if (plaintext != ciphertext) - ciphertext->cd_length = - ciphertext->cd_offset - saved_offset; - } else { - ciphertext->cd_length = saved_length; - } - ciphertext->cd_offset = saved_offset; - - return (ret); -} - - -static int -aes_decrypt_update(crypto_ctx_t *ctx, crypto_data_t *ciphertext, - crypto_data_t *plaintext) -{ - off_t saved_offset; - size_t saved_length; - int ret = CRYPTO_SUCCESS; - - ASSERT(ctx->cc_provider_private != NULL); - - ASSERT(plaintext != NULL); - - saved_offset = plaintext->cd_offset; - saved_length = plaintext->cd_length; - - /* - * Do the AES update on the specified input data. - */ - switch (ciphertext->cd_format) { - case CRYPTO_DATA_RAW: - ret = crypto_update_iov(ctx->cc_provider_private, - ciphertext, plaintext, aes_decrypt_contiguous_blocks); - break; - case CRYPTO_DATA_UIO: - ret = crypto_update_uio(ctx->cc_provider_private, - ciphertext, plaintext, aes_decrypt_contiguous_blocks); - break; - default: - ret = CRYPTO_ARGUMENTS_BAD; - } - - if (ret == CRYPTO_SUCCESS) { - if (ciphertext != plaintext) - plaintext->cd_length = - plaintext->cd_offset - saved_offset; - } else { - plaintext->cd_length = saved_length; - } - plaintext->cd_offset = saved_offset; - - - return (ret); -} - -static int -aes_encrypt_final(crypto_ctx_t *ctx, crypto_data_t *data) -{ - aes_ctx_t *aes_ctx; - int ret; - - ASSERT(ctx->cc_provider_private != NULL); - aes_ctx = ctx->cc_provider_private; - - if (data->cd_format != CRYPTO_DATA_RAW && - data->cd_format != CRYPTO_DATA_UIO) { - return (CRYPTO_ARGUMENTS_BAD); - } - - if (aes_ctx->ac_flags & CCM_MODE) { - ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, data, - AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); - if (ret != CRYPTO_SUCCESS) { - return (ret); - } - } else if (aes_ctx->ac_flags & GCM_MODE) { - size_t saved_offset = data->cd_offset; - - ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, data, - AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, - aes_xor_block); - if (ret != CRYPTO_SUCCESS) { - return (ret); - } - data->cd_length = data->cd_offset - saved_offset; - data->cd_offset = saved_offset; - } - - (void) aes_free_context(ctx); - - return (CRYPTO_SUCCESS); -} - -static int -aes_decrypt_final(crypto_ctx_t *ctx, crypto_data_t *data) -{ - aes_ctx_t *aes_ctx; - int ret; - off_t saved_offset; - size_t saved_length; - - ASSERT(ctx->cc_provider_private != NULL); - aes_ctx = ctx->cc_provider_private; - - if (data->cd_format != CRYPTO_DATA_RAW && - data->cd_format != CRYPTO_DATA_UIO) { - return (CRYPTO_ARGUMENTS_BAD); - } - - /* - * There must be no unprocessed ciphertext. - * This happens if the length of the last ciphertext is - * not a multiple of the AES block length. - */ - if (aes_ctx->ac_remainder_len > 0) - return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE); - - if (aes_ctx->ac_flags & CCM_MODE) { - /* - * This is where all the plaintext is returned, make sure - * the plaintext buffer is big enough - */ - size_t pt_len = aes_ctx->ac_data_len; - if (data->cd_length < pt_len) { - data->cd_length = pt_len; - return (CRYPTO_BUFFER_TOO_SMALL); - } - - ASSERT(aes_ctx->ac_processed_data_len == pt_len); - ASSERT(aes_ctx->ac_processed_mac_len == aes_ctx->ac_mac_len); - saved_offset = data->cd_offset; - saved_length = data->cd_length; - ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, data, - AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, - aes_xor_block); - if (ret == CRYPTO_SUCCESS) { - data->cd_length = data->cd_offset - saved_offset; - } else { - data->cd_length = saved_length; - } - - data->cd_offset = saved_offset; - if (ret != CRYPTO_SUCCESS) { - return (ret); - } - } else if (aes_ctx->ac_flags & GCM_MODE) { - /* - * This is where all the plaintext is returned, make sure - * the plaintext buffer is big enough - */ - gcm_ctx_t *ctx = (gcm_ctx_t *)aes_ctx; - size_t pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len; - - if (data->cd_length < pt_len) { - data->cd_length = pt_len; - return (CRYPTO_BUFFER_TOO_SMALL); - } - - saved_offset = data->cd_offset; - saved_length = data->cd_length; - ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, data, - AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); - if (ret == CRYPTO_SUCCESS) { - data->cd_length = data->cd_offset - saved_offset; - } else { - data->cd_length = saved_length; - } - - data->cd_offset = saved_offset; - if (ret != CRYPTO_SUCCESS) { - return (ret); - } - } - - - (void) aes_free_context(ctx); - - return (CRYPTO_SUCCESS); -} - static int aes_encrypt_atomic(crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext, From 94f1e56e412909cf76b9acf799f5154a08d50a2f Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Sun, 19 May 2024 12:24:35 +1000 Subject: [PATCH 05/42] icp: remove unused KCF_ macros Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Reviewed-by: Brian Behlendorf Signed-off-by: Rob Norris Closes #16209 --- module/icp/include/sys/crypto/impl.h | 37 ---------------------------- 1 file changed, 37 deletions(-) diff --git a/module/icp/include/sys/crypto/impl.h b/module/icp/include/sys/crypto/impl.h index 4d17221ea9a..f6b2e519f0a 100644 --- a/module/icp/include/sys/crypto/impl.h +++ b/module/icp/include/sys/crypto/impl.h @@ -187,28 +187,6 @@ typedef struct kcf_mech_entry { avl_node_t me_node; } kcf_mech_entry_t; -/* - * If a component has a reference to a kcf_policy_desc_t, - * it REFHOLD()s. A new policy descriptor which is referenced only - * by the policy table has a reference count of one. - */ -#define KCF_POLICY_REFHOLD(desc) { \ - int newval = atomic_add_32_nv(&(desc)->pd_refcnt, 1); \ - ASSERT(newval != 0); \ -} - -/* - * Releases a reference to a policy descriptor. When the last - * reference is released, the descriptor is freed. - */ -#define KCF_POLICY_REFRELE(desc) { \ - membar_producer(); \ - int newval = atomic_add_32_nv(&(desc)->pd_refcnt, -1); \ - ASSERT(newval != -1); \ - if (newval == 0) \ - kcf_policy_free_desc(desc); \ -} - /* * Global tables. The sizes are from the predefined PKCS#11 v2.20 mechanisms, * with a margin of few extra empty entry points @@ -275,29 +253,14 @@ extern const kcf_mech_entry_tab_t kcf_mech_tabs_tab[]; * of type kcf_prov_desc_t. */ -#define KCF_PROV_DIGEST_OPS(pd) ((pd)->pd_ops_vector->co_digest_ops) #define KCF_PROV_CIPHER_OPS(pd) ((pd)->pd_ops_vector->co_cipher_ops) #define KCF_PROV_MAC_OPS(pd) ((pd)->pd_ops_vector->co_mac_ops) #define KCF_PROV_CTX_OPS(pd) ((pd)->pd_ops_vector->co_ctx_ops) -/* - * Wrappers for crypto_digest_ops(9S) entry points. - */ - -#define KCF_PROV_DIGEST_INIT(pd, ctx, mech) ( \ - (KCF_PROV_DIGEST_OPS(pd) && KCF_PROV_DIGEST_OPS(pd)->digest_init) ? \ - KCF_PROV_DIGEST_OPS(pd)->digest_init(ctx, mech) : \ - CRYPTO_NOT_SUPPORTED) - /* * Wrappers for crypto_cipher_ops(9S) entry points. */ -#define KCF_PROV_ENCRYPT_INIT(pd, ctx, mech, key, template) ( \ - (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->encrypt_init) ? \ - KCF_PROV_CIPHER_OPS(pd)->encrypt_init(ctx, mech, key, template) : \ - CRYPTO_NOT_SUPPORTED) - #define KCF_PROV_ENCRYPT_ATOMIC(pd, mech, key, plaintext, ciphertext, \ template) ( \ (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->encrypt_atomic) ? \ From 1291c46ea4baf3f8807cf533edbdbd4999f6759e Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Sun, 19 May 2024 12:58:56 +1000 Subject: [PATCH 06/42] icp: remove digest entry points For whatever reason, we call digest mechanisms directly, not through the KCF digest provider. So we can remove those entry points entirely. Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Reviewed-by: Brian Behlendorf Signed-off-by: Rob Norris Closes #16209 --- include/sys/skein.h | 9 - module/icp/core/kcf_mech_tabs.c | 7 +- module/icp/include/sys/crypto/impl.h | 7 +- module/icp/include/sys/crypto/spi.h | 19 -- module/icp/io/aes.c | 1 - module/icp/io/sha2_mod.c | 286 +-------------------------- module/icp/io/skein_mod.c | 143 +------------- 7 files changed, 8 insertions(+), 464 deletions(-) diff --git a/include/sys/skein.h b/include/sys/skein.h index 2f649d6b269..3359d48af79 100644 --- a/include/sys/skein.h +++ b/include/sys/skein.h @@ -152,25 +152,16 @@ typedef struct skein_param { /* Module definitions */ #ifdef SKEIN_MODULE_IMPL -#define CKM_SKEIN_256 "CKM_SKEIN_256" -#define CKM_SKEIN_512 "CKM_SKEIN_512" -#define CKM_SKEIN1024 "CKM_SKEIN1024" #define CKM_SKEIN_256_MAC "CKM_SKEIN_256_MAC" #define CKM_SKEIN_512_MAC "CKM_SKEIN_512_MAC" #define CKM_SKEIN1024_MAC "CKM_SKEIN1024_MAC" typedef enum skein_mech_type { - SKEIN_256_MECH_INFO_TYPE, - SKEIN_512_MECH_INFO_TYPE, - SKEIN1024_MECH_INFO_TYPE, SKEIN_256_MAC_MECH_INFO_TYPE, SKEIN_512_MAC_MECH_INFO_TYPE, SKEIN1024_MAC_MECH_INFO_TYPE } skein_mech_type_t; -#define VALID_SKEIN_DIGEST_MECH(__mech) \ - ((int)(__mech) >= SKEIN_256_MECH_INFO_TYPE && \ - (__mech) <= SKEIN1024_MECH_INFO_TYPE) #define VALID_SKEIN_MAC_MECH(__mech) \ ((int)(__mech) >= SKEIN_256_MAC_MECH_INFO_TYPE && \ (__mech) <= SKEIN1024_MAC_MECH_INFO_TYPE) diff --git a/module/icp/core/kcf_mech_tabs.c b/module/icp/core/kcf_mech_tabs.c index b6e69376980..a1e95847d06 100644 --- a/module/icp/core/kcf_mech_tabs.c +++ b/module/icp/core/kcf_mech_tabs.c @@ -41,7 +41,6 @@ * mech_index is the index for that mechanism in the table. * A mechanism belongs to exactly 1 table. * The tables are: - * . digest_mechs_tab[] for the msg digest mechs. * . cipher_mechs_tab[] for encrypt/decrypt and wrap/unwrap mechs. * . mac_mechs_tab[] for MAC mechs. * . sign_mechs_tab[] for sign & verify mechs. @@ -75,13 +74,11 @@ /* RFE 4687834 Will deal with the extensibility of these tables later */ -static kcf_mech_entry_t kcf_digest_mechs_tab[KCF_MAXDIGEST]; static kcf_mech_entry_t kcf_cipher_mechs_tab[KCF_MAXCIPHER]; static kcf_mech_entry_t kcf_mac_mechs_tab[KCF_MAXMAC]; const kcf_mech_entry_tab_t kcf_mech_tabs_tab[KCF_LAST_OPSCLASS + 1] = { {0, NULL}, /* No class zero */ - {KCF_MAXDIGEST, kcf_digest_mechs_tab}, {KCF_MAXCIPHER, kcf_cipher_mechs_tab}, {KCF_MAXMAC, kcf_mac_mechs_tab}, }; @@ -220,9 +217,7 @@ kcf_add_mech_provider(short mech_indx, crypto_func_group_t fg = mech_info->cm_func_group_mask; kcf_ops_class_t class; - if (fg & CRYPTO_FG_DIGEST || fg & CRYPTO_FG_DIGEST_ATOMIC) - class = KCF_DIGEST_CLASS; - else if (fg & CRYPTO_FG_ENCRYPT_ATOMIC || + if (fg & CRYPTO_FG_ENCRYPT_ATOMIC || fg & CRYPTO_FG_DECRYPT_ATOMIC) class = KCF_CIPHER_CLASS; else if (fg & CRYPTO_FG_MAC || fg & CRYPTO_FG_MAC_ATOMIC) diff --git a/module/icp/include/sys/crypto/impl.h b/module/icp/include/sys/crypto/impl.h index f6b2e519f0a..0f5ef58ac00 100644 --- a/module/icp/include/sys/crypto/impl.h +++ b/module/icp/include/sys/crypto/impl.h @@ -55,7 +55,7 @@ extern "C" { * When impl.h is broken up (bug# 4703218), this will be done. For now, * we hardcode these values. */ -#define KCF_OPS_CLASSSIZE 4 +#define KCF_OPS_CLASSSIZE 3 #define KCF_MAXMECHTAB 32 /* @@ -200,12 +200,11 @@ _Static_assert(KCF_MAXCIPHER == KCF_MAXMECHTAB, "KCF_MAXCIPHER != KCF_MAXMECHTAB"); /* See KCF_MAXMECHTAB comment */ typedef enum { - KCF_DIGEST_CLASS = 1, - KCF_CIPHER_CLASS, + KCF_CIPHER_CLASS = 1, KCF_MAC_CLASS, } kcf_ops_class_t; -#define KCF_FIRST_OPSCLASS KCF_DIGEST_CLASS +#define KCF_FIRST_OPSCLASS KCF_CIPHER_CLASS #define KCF_LAST_OPSCLASS KCF_MAC_CLASS _Static_assert( KCF_OPS_CLASSSIZE == (KCF_LAST_OPSCLASS - KCF_FIRST_OPSCLASS + 2), diff --git a/module/icp/include/sys/crypto/spi.h b/module/icp/include/sys/crypto/spi.h index 9bcb62ac529..e9be7e0c54d 100644 --- a/module/icp/include/sys/crypto/spi.h +++ b/module/icp/include/sys/crypto/spi.h @@ -66,22 +66,6 @@ typedef struct crypto_ctx { void *cc_framework_private; /* owned by framework */ } crypto_ctx_t; -/* - * The crypto_digest_ops structure contains pointers to digest - * operations for cryptographic providers. It is passed through - * the crypto_ops(9S) structure when providers register with the - * kernel using crypto_register_provider(9F). - */ -typedef struct crypto_digest_ops { - int (*digest_init)(crypto_ctx_t *, crypto_mechanism_t *); - int (*digest)(crypto_ctx_t *, crypto_data_t *, crypto_data_t *); - int (*digest_update)(crypto_ctx_t *, crypto_data_t *); - int (*digest_key)(crypto_ctx_t *, crypto_key_t *); - int (*digest_final)(crypto_ctx_t *, crypto_data_t *); - int (*digest_atomic)(crypto_mechanism_t *, crypto_data_t *, - crypto_data_t *); -} __no_const crypto_digest_ops_t; - /* * The crypto_cipher_ops structure contains pointers to encryption * and decryption operations for cryptographic providers. It is @@ -137,7 +121,6 @@ typedef struct crypto_ctx_ops { * by calling crypto_register_provider(9F). */ typedef struct crypto_ops { - const crypto_digest_ops_t *co_digest_ops; const crypto_cipher_ops_t *co_cipher_ops; const crypto_mac_ops_t *co_mac_ops; const crypto_ctx_ops_t *co_ctx_ops; @@ -153,12 +136,10 @@ typedef struct crypto_ops { typedef uint32_t crypto_func_group_t; -#define CRYPTO_FG_DIGEST 0x00000004 /* digest_init() */ #define CRYPTO_FG_MAC 0x00001000 /* mac_init() */ #define CRYPTO_FG_ENCRYPT_ATOMIC 0x00008000 /* encrypt_atomic() */ #define CRYPTO_FG_DECRYPT_ATOMIC 0x00010000 /* decrypt_atomic() */ #define CRYPTO_FG_MAC_ATOMIC 0x00020000 /* mac_atomic() */ -#define CRYPTO_FG_DIGEST_ATOMIC 0x00040000 /* digest_atomic() */ /* * Maximum length of the pi_provider_description field of the diff --git a/module/icp/io/aes.c b/module/icp/io/aes.c index a68a878b6ab..8ee2d036c1e 100644 --- a/module/icp/io/aes.c +++ b/module/icp/io/aes.c @@ -72,7 +72,6 @@ static const crypto_ctx_ops_t aes_ctx_ops = { }; static const crypto_ops_t aes_crypto_ops = { - NULL, &aes_cipher_ops, NULL, &aes_ctx_ops, diff --git a/module/icp/io/sha2_mod.c b/module/icp/io/sha2_mod.c index f068951b07f..c8e3b4fccdd 100644 --- a/module/icp/io/sha2_mod.c +++ b/module/icp/io/sha2_mod.c @@ -61,8 +61,7 @@ */ static const crypto_mech_info_t sha2_mech_info_tab[] = { /* SHA256 */ - {SUN_CKM_SHA256, SHA256_MECH_INFO_TYPE, - CRYPTO_FG_DIGEST | CRYPTO_FG_DIGEST_ATOMIC}, + {SUN_CKM_SHA256, SHA256_MECH_INFO_TYPE, 0}, /* SHA256-HMAC */ {SUN_CKM_SHA256_HMAC, SHA256_HMAC_MECH_INFO_TYPE, CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, @@ -70,8 +69,7 @@ static const crypto_mech_info_t sha2_mech_info_tab[] = { {SUN_CKM_SHA256_HMAC_GENERAL, SHA256_HMAC_GEN_MECH_INFO_TYPE, CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, /* SHA384 */ - {SUN_CKM_SHA384, SHA384_MECH_INFO_TYPE, - CRYPTO_FG_DIGEST | CRYPTO_FG_DIGEST_ATOMIC}, + {SUN_CKM_SHA384, SHA384_MECH_INFO_TYPE, 0}, /* SHA384-HMAC */ {SUN_CKM_SHA384_HMAC, SHA384_HMAC_MECH_INFO_TYPE, CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, @@ -79,8 +77,7 @@ static const crypto_mech_info_t sha2_mech_info_tab[] = { {SUN_CKM_SHA384_HMAC_GENERAL, SHA384_HMAC_GEN_MECH_INFO_TYPE, CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, /* SHA512 */ - {SUN_CKM_SHA512, SHA512_MECH_INFO_TYPE, - CRYPTO_FG_DIGEST | CRYPTO_FG_DIGEST_ATOMIC}, + {SUN_CKM_SHA512, SHA512_MECH_INFO_TYPE, 0}, /* SHA512-HMAC */ {SUN_CKM_SHA512_HMAC, SHA512_HMAC_MECH_INFO_TYPE, CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, @@ -89,21 +86,6 @@ static const crypto_mech_info_t sha2_mech_info_tab[] = { CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, }; -static int sha2_digest_init(crypto_ctx_t *, crypto_mechanism_t *); -static int sha2_digest(crypto_ctx_t *, crypto_data_t *, crypto_data_t *); -static int sha2_digest_update(crypto_ctx_t *, crypto_data_t *); -static int sha2_digest_final(crypto_ctx_t *, crypto_data_t *); -static int sha2_digest_atomic(crypto_mechanism_t *, crypto_data_t *, - crypto_data_t *); - -static const crypto_digest_ops_t sha2_digest_ops = { - .digest_init = sha2_digest_init, - .digest = sha2_digest, - .digest_update = sha2_digest_update, - .digest_final = sha2_digest_final, - .digest_atomic = sha2_digest_atomic -}; - static int sha2_mac_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t); static int sha2_mac_update(crypto_ctx_t *, crypto_data_t *); @@ -132,7 +114,6 @@ static const crypto_ctx_ops_t sha2_ctx_ops = { }; static const crypto_ops_t sha2_crypto_ops = { - &sha2_digest_ops, NULL, &sha2_mac_ops, &sha2_ctx_ops, @@ -184,27 +165,6 @@ sha2_mod_fini(void) return (ret); } -/* - * KCF software provider digest entry points. - */ - -static int -sha2_digest_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism) -{ - - /* - * Allocate and initialize SHA2 context. - */ - ctx->cc_provider_private = kmem_alloc(sizeof (sha2_ctx_t), KM_SLEEP); - if (ctx->cc_provider_private == NULL) - return (CRYPTO_HOST_MEMORY); - - PROV_SHA2_CTX(ctx)->sc_mech_type = mechanism->cm_type; - SHA2Init(mechanism->cm_type, &PROV_SHA2_CTX(ctx)->sc_sha2_ctx); - - return (CRYPTO_SUCCESS); -} - /* * Helper SHA2 digest update function for uio data. */ @@ -360,246 +320,6 @@ sha2_digest_final_uio(SHA2_CTX *sha2_ctx, crypto_data_t *digest, return (CRYPTO_SUCCESS); } -static int -sha2_digest(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *digest) -{ - int ret = CRYPTO_SUCCESS; - uint_t sha_digest_len; - - ASSERT(ctx->cc_provider_private != NULL); - - switch (PROV_SHA2_CTX(ctx)->sc_mech_type) { - case SHA256_MECH_INFO_TYPE: - sha_digest_len = SHA256_DIGEST_LENGTH; - break; - case SHA384_MECH_INFO_TYPE: - sha_digest_len = SHA384_DIGEST_LENGTH; - break; - case SHA512_MECH_INFO_TYPE: - sha_digest_len = SHA512_DIGEST_LENGTH; - break; - default: - return (CRYPTO_MECHANISM_INVALID); - } - - /* - * We need to just return the length needed to store the output. - * We should not destroy the context for the following cases. - */ - if ((digest->cd_length == 0) || - (digest->cd_length < sha_digest_len)) { - digest->cd_length = sha_digest_len; - return (CRYPTO_BUFFER_TOO_SMALL); - } - - /* - * Do the SHA2 update on the specified input data. - */ - switch (data->cd_format) { - case CRYPTO_DATA_RAW: - SHA2Update(&PROV_SHA2_CTX(ctx)->sc_sha2_ctx, - (uint8_t *)data->cd_raw.iov_base + data->cd_offset, - data->cd_length); - break; - case CRYPTO_DATA_UIO: - ret = sha2_digest_update_uio(&PROV_SHA2_CTX(ctx)->sc_sha2_ctx, - data); - break; - default: - ret = CRYPTO_ARGUMENTS_BAD; - } - - if (ret != CRYPTO_SUCCESS) { - /* the update failed, free context and bail */ - kmem_free(ctx->cc_provider_private, sizeof (sha2_ctx_t)); - ctx->cc_provider_private = NULL; - digest->cd_length = 0; - return (ret); - } - - /* - * Do a SHA2 final, must be done separately since the digest - * type can be different than the input data type. - */ - switch (digest->cd_format) { - case CRYPTO_DATA_RAW: - SHA2Final((unsigned char *)digest->cd_raw.iov_base + - digest->cd_offset, &PROV_SHA2_CTX(ctx)->sc_sha2_ctx); - break; - case CRYPTO_DATA_UIO: - ret = sha2_digest_final_uio(&PROV_SHA2_CTX(ctx)->sc_sha2_ctx, - digest, sha_digest_len, NULL); - break; - default: - ret = CRYPTO_ARGUMENTS_BAD; - } - - /* all done, free context and return */ - - if (ret == CRYPTO_SUCCESS) - digest->cd_length = sha_digest_len; - else - digest->cd_length = 0; - - kmem_free(ctx->cc_provider_private, sizeof (sha2_ctx_t)); - ctx->cc_provider_private = NULL; - return (ret); -} - -static int -sha2_digest_update(crypto_ctx_t *ctx, crypto_data_t *data) -{ - int ret = CRYPTO_SUCCESS; - - ASSERT(ctx->cc_provider_private != NULL); - - /* - * Do the SHA2 update on the specified input data. - */ - switch (data->cd_format) { - case CRYPTO_DATA_RAW: - SHA2Update(&PROV_SHA2_CTX(ctx)->sc_sha2_ctx, - (uint8_t *)data->cd_raw.iov_base + data->cd_offset, - data->cd_length); - break; - case CRYPTO_DATA_UIO: - ret = sha2_digest_update_uio(&PROV_SHA2_CTX(ctx)->sc_sha2_ctx, - data); - break; - default: - ret = CRYPTO_ARGUMENTS_BAD; - } - - return (ret); -} - -static int -sha2_digest_final(crypto_ctx_t *ctx, crypto_data_t *digest) -{ - int ret = CRYPTO_SUCCESS; - uint_t sha_digest_len; - - ASSERT(ctx->cc_provider_private != NULL); - - switch (PROV_SHA2_CTX(ctx)->sc_mech_type) { - case SHA256_MECH_INFO_TYPE: - sha_digest_len = SHA256_DIGEST_LENGTH; - break; - case SHA384_MECH_INFO_TYPE: - sha_digest_len = SHA384_DIGEST_LENGTH; - break; - case SHA512_MECH_INFO_TYPE: - sha_digest_len = SHA512_DIGEST_LENGTH; - break; - default: - return (CRYPTO_MECHANISM_INVALID); - } - - /* - * We need to just return the length needed to store the output. - * We should not destroy the context for the following cases. - */ - if ((digest->cd_length == 0) || - (digest->cd_length < sha_digest_len)) { - digest->cd_length = sha_digest_len; - return (CRYPTO_BUFFER_TOO_SMALL); - } - - /* - * Do a SHA2 final. - */ - switch (digest->cd_format) { - case CRYPTO_DATA_RAW: - SHA2Final((unsigned char *)digest->cd_raw.iov_base + - digest->cd_offset, &PROV_SHA2_CTX(ctx)->sc_sha2_ctx); - break; - case CRYPTO_DATA_UIO: - ret = sha2_digest_final_uio(&PROV_SHA2_CTX(ctx)->sc_sha2_ctx, - digest, sha_digest_len, NULL); - break; - default: - ret = CRYPTO_ARGUMENTS_BAD; - } - - /* all done, free context and return */ - - if (ret == CRYPTO_SUCCESS) - digest->cd_length = sha_digest_len; - else - digest->cd_length = 0; - - kmem_free(ctx->cc_provider_private, sizeof (sha2_ctx_t)); - ctx->cc_provider_private = NULL; - - return (ret); -} - -static int -sha2_digest_atomic(crypto_mechanism_t *mechanism, crypto_data_t *data, - crypto_data_t *digest) -{ - int ret = CRYPTO_SUCCESS; - SHA2_CTX sha2_ctx; - uint32_t sha_digest_len; - - /* - * Do the SHA inits. - */ - - SHA2Init(mechanism->cm_type, &sha2_ctx); - - switch (data->cd_format) { - case CRYPTO_DATA_RAW: - SHA2Update(&sha2_ctx, (uint8_t *)data-> - cd_raw.iov_base + data->cd_offset, data->cd_length); - break; - case CRYPTO_DATA_UIO: - ret = sha2_digest_update_uio(&sha2_ctx, data); - break; - default: - ret = CRYPTO_ARGUMENTS_BAD; - } - - /* - * Do the SHA updates on the specified input data. - */ - - if (ret != CRYPTO_SUCCESS) { - /* the update failed, bail */ - digest->cd_length = 0; - return (ret); - } - - if (mechanism->cm_type <= SHA256_HMAC_GEN_MECH_INFO_TYPE) - sha_digest_len = SHA256_DIGEST_LENGTH; - else - sha_digest_len = SHA512_DIGEST_LENGTH; - - /* - * Do a SHA2 final, must be done separately since the digest - * type can be different than the input data type. - */ - switch (digest->cd_format) { - case CRYPTO_DATA_RAW: - SHA2Final((unsigned char *)digest->cd_raw.iov_base + - digest->cd_offset, &sha2_ctx); - break; - case CRYPTO_DATA_UIO: - ret = sha2_digest_final_uio(&sha2_ctx, digest, - sha_digest_len, NULL); - break; - default: - ret = CRYPTO_ARGUMENTS_BAD; - } - - if (ret == CRYPTO_SUCCESS) - digest->cd_length = sha_digest_len; - else - digest->cd_length = 0; - - return (ret); -} - /* * KCF software provider mac entry points. * diff --git a/module/icp/io/skein_mod.c b/module/icp/io/skein_mod.c index 221e1debd45..3e969513be6 100644 --- a/module/icp/io/skein_mod.c +++ b/module/icp/io/skein_mod.c @@ -31,34 +31,16 @@ #include static const crypto_mech_info_t skein_mech_info_tab[] = { - {CKM_SKEIN_256, SKEIN_256_MECH_INFO_TYPE, - CRYPTO_FG_DIGEST | CRYPTO_FG_DIGEST_ATOMIC}, {CKM_SKEIN_256_MAC, SKEIN_256_MAC_MECH_INFO_TYPE, CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, - {CKM_SKEIN_512, SKEIN_512_MECH_INFO_TYPE, - CRYPTO_FG_DIGEST | CRYPTO_FG_DIGEST_ATOMIC}, {CKM_SKEIN_512_MAC, SKEIN_512_MAC_MECH_INFO_TYPE, CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, - {CKM_SKEIN1024, SKEIN1024_MECH_INFO_TYPE, - CRYPTO_FG_DIGEST | CRYPTO_FG_DIGEST_ATOMIC}, {CKM_SKEIN1024_MAC, SKEIN1024_MAC_MECH_INFO_TYPE, CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, }; -static int skein_digest_init(crypto_ctx_t *, crypto_mechanism_t *); -static int skein_digest(crypto_ctx_t *, crypto_data_t *, crypto_data_t *); static int skein_update(crypto_ctx_t *, crypto_data_t *); static int skein_final(crypto_ctx_t *, crypto_data_t *); -static int skein_digest_atomic(crypto_mechanism_t *, crypto_data_t *, - crypto_data_t *); - -static const crypto_digest_ops_t skein_digest_ops = { - .digest_init = skein_digest_init, - .digest = skein_digest, - .digest_update = skein_update, - .digest_final = skein_final, - .digest_atomic = skein_digest_atomic -}; static int skein_mac_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t); @@ -84,7 +66,6 @@ static const crypto_ctx_ops_t skein_ctx_ops = { }; static const crypto_ops_t skein_crypto_ops = { - &skein_digest_ops, NULL, &skein_mac_ops, &skein_ctx_ops, @@ -115,15 +96,12 @@ typedef struct skein_ctx { do { \ skein_ctx_t *sc = (_skein_ctx); \ switch (sc->sc_mech_type) { \ - case SKEIN_256_MECH_INFO_TYPE: \ case SKEIN_256_MAC_MECH_INFO_TYPE: \ (void) Skein_256_ ## _op(&sc->sc_256, __VA_ARGS__);\ break; \ - case SKEIN_512_MECH_INFO_TYPE: \ case SKEIN_512_MAC_MECH_INFO_TYPE: \ (void) Skein_512_ ## _op(&sc->sc_512, __VA_ARGS__);\ break; \ - case SKEIN1024_MECH_INFO_TYPE: \ case SKEIN1024_MAC_MECH_INFO_TYPE: \ (void) Skein1024_ ## _op(&sc->sc_1024, __VA_ARGS__);\ break; \ @@ -143,19 +121,7 @@ skein_get_digest_bitlen(const crypto_mechanism_t *mechanism, size_t *result) } *result = param->sp_digest_bitlen; } else { - switch (mechanism->cm_type) { - case SKEIN_256_MECH_INFO_TYPE: - *result = 256; - break; - case SKEIN_512_MECH_INFO_TYPE: - *result = 512; - break; - case SKEIN1024_MECH_INFO_TYPE: - *result = 1024; - break; - default: - return (CRYPTO_MECHANISM_INVALID); - } + return (CRYPTO_MECHANISM_INVALID); } return (CRYPTO_SUCCESS); } @@ -320,73 +286,6 @@ skein_digest_final_uio(skein_ctx_t *ctx, crypto_data_t *digest) * KCF software provider digest entry points. */ -/* - * Initializes a skein digest context to the configuration in `mechanism'. - * The mechanism cm_type must be one of SKEIN_*_MECH_INFO_TYPE. The cm_param - * field may contain a skein_param_t structure indicating the length of the - * digest the algorithm should produce. Otherwise the default output lengths - * are applied (32 bytes for Skein-256, 64 bytes for Skein-512 and 128 bytes - * for Skein-1024). - */ -static int -skein_digest_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism) -{ - int error = CRYPTO_SUCCESS; - - if (!VALID_SKEIN_DIGEST_MECH(mechanism->cm_type)) - return (CRYPTO_MECHANISM_INVALID); - - SKEIN_CTX_LVALUE(ctx) = kmem_alloc(sizeof (*SKEIN_CTX(ctx)), KM_SLEEP); - if (SKEIN_CTX(ctx) == NULL) - return (CRYPTO_HOST_MEMORY); - - SKEIN_CTX(ctx)->sc_mech_type = mechanism->cm_type; - error = skein_get_digest_bitlen(mechanism, - &SKEIN_CTX(ctx)->sc_digest_bitlen); - if (error != CRYPTO_SUCCESS) - goto errout; - SKEIN_OP(SKEIN_CTX(ctx), Init, SKEIN_CTX(ctx)->sc_digest_bitlen); - - return (CRYPTO_SUCCESS); -errout: - memset(SKEIN_CTX(ctx), 0, sizeof (*SKEIN_CTX(ctx))); - kmem_free(SKEIN_CTX(ctx), sizeof (*SKEIN_CTX(ctx))); - SKEIN_CTX_LVALUE(ctx) = NULL; - return (error); -} - -/* - * Executes a skein_update and skein_digest on a pre-initialized crypto - * context in a single step. See the documentation to these functions to - * see what to pass here. - */ -static int -skein_digest(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *digest) -{ - int error = CRYPTO_SUCCESS; - - ASSERT(SKEIN_CTX(ctx) != NULL); - - if (digest->cd_length < - CRYPTO_BITS2BYTES(SKEIN_CTX(ctx)->sc_digest_bitlen)) { - digest->cd_length = - CRYPTO_BITS2BYTES(SKEIN_CTX(ctx)->sc_digest_bitlen); - return (CRYPTO_BUFFER_TOO_SMALL); - } - - error = skein_update(ctx, data); - if (error != CRYPTO_SUCCESS) { - memset(SKEIN_CTX(ctx), 0, sizeof (*SKEIN_CTX(ctx))); - kmem_free(SKEIN_CTX(ctx), sizeof (*SKEIN_CTX(ctx))); - SKEIN_CTX_LVALUE(ctx) = NULL; - digest->cd_length = 0; - return (error); - } - error = skein_final(ctx, digest); - - return (error); -} - /* * Performs a skein Update with the input message in `data' (successive calls * can push more data). This is used both for digest and MAC operation. @@ -470,46 +369,6 @@ skein_final(crypto_ctx_t *ctx, crypto_data_t *digest) return (error); } -/* - * Performs a full skein digest computation in a single call, configuring the - * algorithm according to `mechanism', reading the input to be digested from - * `data' and writing the output to `digest'. - * Supported input/output formats are raw, uio and mblk. - */ -static int -skein_digest_atomic(crypto_mechanism_t *mechanism, crypto_data_t *data, - crypto_data_t *digest) -{ - int error; - skein_ctx_t skein_ctx; - crypto_ctx_t ctx; - SKEIN_CTX_LVALUE(&ctx) = &skein_ctx; - - /* Init */ - if (!VALID_SKEIN_DIGEST_MECH(mechanism->cm_type)) - return (CRYPTO_MECHANISM_INVALID); - skein_ctx.sc_mech_type = mechanism->cm_type; - error = skein_get_digest_bitlen(mechanism, &skein_ctx.sc_digest_bitlen); - if (error != CRYPTO_SUCCESS) - goto out; - SKEIN_OP(&skein_ctx, Init, skein_ctx.sc_digest_bitlen); - - if ((error = skein_update(&ctx, data)) != CRYPTO_SUCCESS) - goto out; - if ((error = skein_final_nofree(&ctx, data)) != CRYPTO_SUCCESS) - goto out; - -out: - if (error == CRYPTO_SUCCESS) - digest->cd_length = - CRYPTO_BITS2BYTES(skein_ctx.sc_digest_bitlen); - else - digest->cd_length = 0; - memset(&skein_ctx, 0, sizeof (skein_ctx)); - - return (error); -} - /* * Helper function that builds a Skein MAC context from the provided * mechanism and key. From 10de12e9ed2fee85adc2b9b4efac64f8655062bc Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Sun, 19 May 2024 13:18:42 +1000 Subject: [PATCH 07/42] icp: reorganise SHA2 digest mechanisms sha2_mech_type_t serves double-duty, as the list of MAC providers and also the algo type for direct callers to SHA2Init. Until we disentangle that, reorganise it to make the separation more clear. While we're there, remove the digest mechs we don't use. Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Reviewed-by: Brian Behlendorf Signed-off-by: Rob Norris Closes #16209 --- include/sys/crypto/common.h | 5 -- include/sys/sha2.h | 22 ++------ module/icp/algs/sha2/sha2_generic.c | 44 ++++----------- module/icp/io/sha2_mod.c | 6 -- tests/zfs-tests/cmd/checksum/sha2_test.c | 70 ++++-------------------- 5 files changed, 26 insertions(+), 121 deletions(-) diff --git a/include/sys/crypto/common.h b/include/sys/crypto/common.h index 74380565005..a73804f916b 100644 --- a/include/sys/crypto/common.h +++ b/include/sys/crypto/common.h @@ -79,17 +79,12 @@ typedef uint32_t crypto_keysize_unit_t; /* Mechanisms supported out-of-the-box */ -#define SUN_CKM_SHA256 "CKM_SHA256" #define SUN_CKM_SHA256_HMAC "CKM_SHA256_HMAC" #define SUN_CKM_SHA256_HMAC_GENERAL "CKM_SHA256_HMAC_GENERAL" -#define SUN_CKM_SHA384 "CKM_SHA384" #define SUN_CKM_SHA384_HMAC "CKM_SHA384_HMAC" #define SUN_CKM_SHA384_HMAC_GENERAL "CKM_SHA384_HMAC_GENERAL" -#define SUN_CKM_SHA512 "CKM_SHA512" #define SUN_CKM_SHA512_HMAC "CKM_SHA512_HMAC" #define SUN_CKM_SHA512_HMAC_GENERAL "CKM_SHA512_HMAC_GENERAL" -#define SUN_CKM_SHA512_224 "CKM_SHA512_224" -#define SUN_CKM_SHA512_256 "CKM_SHA512_256" #define SUN_CKM_AES_CCM "CKM_AES_CCM" #define SUN_CKM_AES_GCM "CKM_AES_GCM" diff --git a/include/sys/sha2.h b/include/sys/sha2.h index 81dfbbb8cea..2d38885bd96 100644 --- a/include/sys/sha2.h +++ b/include/sys/sha2.h @@ -86,30 +86,18 @@ typedef struct { /* SHA2 algorithm types */ typedef enum sha2_mech_type { - SHA256_MECH_INFO_TYPE, /* SUN_CKM_SHA256 */ SHA256_HMAC_MECH_INFO_TYPE, /* SUN_CKM_SHA256_HMAC */ SHA256_HMAC_GEN_MECH_INFO_TYPE, /* SUN_CKM_SHA256_HMAC_GENERAL */ - SHA384_MECH_INFO_TYPE, /* SUN_CKM_SHA384 */ SHA384_HMAC_MECH_INFO_TYPE, /* SUN_CKM_SHA384_HMAC */ SHA384_HMAC_GEN_MECH_INFO_TYPE, /* SUN_CKM_SHA384_HMAC_GENERAL */ - SHA512_MECH_INFO_TYPE, /* SUN_CKM_SHA512 */ SHA512_HMAC_MECH_INFO_TYPE, /* SUN_CKM_SHA512_HMAC */ SHA512_HMAC_GEN_MECH_INFO_TYPE, /* SUN_CKM_SHA512_HMAC_GENERAL */ - SHA512_224_MECH_INFO_TYPE, /* SUN_CKM_SHA512_224 */ - SHA512_256_MECH_INFO_TYPE /* SUN_CKM_SHA512_256 */ -} sha2_mech_type_t; -#define SHA256 0 -#define SHA256_HMAC 1 -#define SHA256_HMAC_GEN 2 -#define SHA384 3 -#define SHA384_HMAC 4 -#define SHA384_HMAC_GEN 5 -#define SHA512 6 -#define SHA512_HMAC 7 -#define SHA512_HMAC_GEN 8 -#define SHA512_224 9 -#define SHA512_256 10 + /* Not true KCF mech types; used by direct callers to SHA2Init */ + SHA256, + SHA512, + SHA512_256, +} sha2_mech_type_t; /* SHA2 Init function */ extern void SHA2Init(int algotype, SHA2_CTX *ctx); diff --git a/module/icp/algs/sha2/sha2_generic.c b/module/icp/algs/sha2/sha2_generic.c index 60d7ad9a1df..ab361b9d59f 100644 --- a/module/icp/algs/sha2/sha2_generic.c +++ b/module/icp/algs/sha2/sha2_generic.c @@ -400,13 +400,13 @@ SHA2Init(int algotype, SHA2_CTX *ctx) sha256_ctx *ctx256 = &ctx->sha256; sha512_ctx *ctx512 = &ctx->sha512; - ASSERT3S(algotype, >=, SHA256_MECH_INFO_TYPE); - ASSERT3S(algotype, <=, SHA512_256_MECH_INFO_TYPE); + ASSERT3S(algotype, >=, SHA256_HMAC_MECH_INFO_TYPE); + ASSERT3S(algotype, <=, SHA512_256); memset(ctx, 0, sizeof (*ctx)); ctx->algotype = algotype; switch (ctx->algotype) { - case SHA256_MECH_INFO_TYPE: + case SHA256: case SHA256_HMAC_MECH_INFO_TYPE: case SHA256_HMAC_GEN_MECH_INFO_TYPE: ctx256->state[0] = 0x6a09e667; @@ -420,7 +420,6 @@ SHA2Init(int algotype, SHA2_CTX *ctx) ctx256->count[0] = 0; ctx256->ops = sha256_get_ops(); break; - case SHA384_MECH_INFO_TYPE: case SHA384_HMAC_MECH_INFO_TYPE: case SHA384_HMAC_GEN_MECH_INFO_TYPE: ctx512->state[0] = 0xcbbb9d5dc1059ed8ULL; @@ -435,7 +434,7 @@ SHA2Init(int algotype, SHA2_CTX *ctx) ctx512->count[1] = 0; ctx512->ops = sha512_get_ops(); break; - case SHA512_MECH_INFO_TYPE: + case SHA512: case SHA512_HMAC_MECH_INFO_TYPE: case SHA512_HMAC_GEN_MECH_INFO_TYPE: ctx512->state[0] = 0x6a09e667f3bcc908ULL; @@ -450,20 +449,7 @@ SHA2Init(int algotype, SHA2_CTX *ctx) ctx512->count[1] = 0; ctx512->ops = sha512_get_ops(); break; - case SHA512_224_MECH_INFO_TYPE: - ctx512->state[0] = 0x8c3d37c819544da2ULL; - ctx512->state[1] = 0x73e1996689dcd4d6ULL; - ctx512->state[2] = 0x1dfab7ae32ff9c82ULL; - ctx512->state[3] = 0x679dd514582f9fcfULL; - ctx512->state[4] = 0x0f6d2b697bd44da8ULL; - ctx512->state[5] = 0x77e36f7304c48942ULL; - ctx512->state[6] = 0x3f9d85a86a1d36c8ULL; - ctx512->state[7] = 0x1112e6ad91d692a1ULL; - ctx512->count[0] = 0; - ctx512->count[1] = 0; - ctx512->ops = sha512_get_ops(); - break; - case SHA512_256_MECH_INFO_TYPE: + case SHA512_256: ctx512->state[0] = 0x22312194fc2bf72cULL; ctx512->state[1] = 0x9f555fa3c84c64c2ULL; ctx512->state[2] = 0x2393b86b6f53b151ULL; @@ -490,25 +476,21 @@ SHA2Update(SHA2_CTX *ctx, const void *data, size_t len) ASSERT3P(data, !=, NULL); switch (ctx->algotype) { - case SHA256_MECH_INFO_TYPE: + case SHA256: case SHA256_HMAC_MECH_INFO_TYPE: case SHA256_HMAC_GEN_MECH_INFO_TYPE: sha256_update(&ctx->sha256, data, len); break; - case SHA384_MECH_INFO_TYPE: case SHA384_HMAC_MECH_INFO_TYPE: case SHA384_HMAC_GEN_MECH_INFO_TYPE: sha512_update(&ctx->sha512, data, len); break; - case SHA512_MECH_INFO_TYPE: + case SHA512: case SHA512_HMAC_MECH_INFO_TYPE: case SHA512_HMAC_GEN_MECH_INFO_TYPE: sha512_update(&ctx->sha512, data, len); break; - case SHA512_224_MECH_INFO_TYPE: - sha512_update(&ctx->sha512, data, len); - break; - case SHA512_256_MECH_INFO_TYPE: + case SHA512_256: sha512_update(&ctx->sha512, data, len); break; } @@ -519,25 +501,21 @@ void SHA2Final(void *digest, SHA2_CTX *ctx) { switch (ctx->algotype) { - case SHA256_MECH_INFO_TYPE: + case SHA256: case SHA256_HMAC_MECH_INFO_TYPE: case SHA256_HMAC_GEN_MECH_INFO_TYPE: sha256_final(&ctx->sha256, digest, 256); break; - case SHA384_MECH_INFO_TYPE: case SHA384_HMAC_MECH_INFO_TYPE: case SHA384_HMAC_GEN_MECH_INFO_TYPE: sha512_final(&ctx->sha512, digest, 384); break; - case SHA512_MECH_INFO_TYPE: + case SHA512: case SHA512_HMAC_MECH_INFO_TYPE: case SHA512_HMAC_GEN_MECH_INFO_TYPE: sha512_final(&ctx->sha512, digest, 512); break; - case SHA512_224_MECH_INFO_TYPE: - sha512_final(&ctx->sha512, digest, 224); - break; - case SHA512_256_MECH_INFO_TYPE: + case SHA512_256: sha512_final(&ctx->sha512, digest, 256); break; } diff --git a/module/icp/io/sha2_mod.c b/module/icp/io/sha2_mod.c index c8e3b4fccdd..d80ea1e677b 100644 --- a/module/icp/io/sha2_mod.c +++ b/module/icp/io/sha2_mod.c @@ -60,24 +60,18 @@ * Mechanism info structure passed to KCF during registration. */ static const crypto_mech_info_t sha2_mech_info_tab[] = { - /* SHA256 */ - {SUN_CKM_SHA256, SHA256_MECH_INFO_TYPE, 0}, /* SHA256-HMAC */ {SUN_CKM_SHA256_HMAC, SHA256_HMAC_MECH_INFO_TYPE, CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, /* SHA256-HMAC GENERAL */ {SUN_CKM_SHA256_HMAC_GENERAL, SHA256_HMAC_GEN_MECH_INFO_TYPE, CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, - /* SHA384 */ - {SUN_CKM_SHA384, SHA384_MECH_INFO_TYPE, 0}, /* SHA384-HMAC */ {SUN_CKM_SHA384_HMAC, SHA384_HMAC_MECH_INFO_TYPE, CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, /* SHA384-HMAC GENERAL */ {SUN_CKM_SHA384_HMAC_GENERAL, SHA384_HMAC_GEN_MECH_INFO_TYPE, CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, - /* SHA512 */ - {SUN_CKM_SHA512, SHA512_MECH_INFO_TYPE, 0}, /* SHA512-HMAC */ {SUN_CKM_SHA512_HMAC, SHA512_HMAC_MECH_INFO_TYPE, CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, diff --git a/tests/zfs-tests/cmd/checksum/sha2_test.c b/tests/zfs-tests/cmd/checksum/sha2_test.c index efcf812d774..d36b670db8b 100644 --- a/tests/zfs-tests/cmd/checksum/sha2_test.c +++ b/tests/zfs-tests/cmd/checksum/sha2_test.c @@ -72,31 +72,6 @@ static const uint8_t sha256_test_digests[][32] = { /* no test vector for test_msg2 */ }; -static const uint8_t sha384_test_digests[][48] = { - { - /* for test_msg0 */ - 0xCB, 0x00, 0x75, 0x3F, 0x45, 0xA3, 0x5E, 0x8B, - 0xB5, 0xA0, 0x3D, 0x69, 0x9A, 0xC6, 0x50, 0x07, - 0x27, 0x2C, 0x32, 0xAB, 0x0E, 0xDE, 0xD1, 0x63, - 0x1A, 0x8B, 0x60, 0x5A, 0x43, 0xFF, 0x5B, 0xED, - 0x80, 0x86, 0x07, 0x2B, 0xA1, 0xE7, 0xCC, 0x23, - 0x58, 0xBA, 0xEC, 0xA1, 0x34, 0xC8, 0x25, 0xA7 - }, - { - /* no test vector for test_msg1 */ - 0 - }, - { - /* for test_msg2 */ - 0x09, 0x33, 0x0C, 0x33, 0xF7, 0x11, 0x47, 0xE8, - 0x3D, 0x19, 0x2F, 0xC7, 0x82, 0xCD, 0x1B, 0x47, - 0x53, 0x11, 0x1B, 0x17, 0x3B, 0x3B, 0x05, 0xD2, - 0x2F, 0xA0, 0x80, 0x86, 0xE3, 0xB0, 0xF7, 0x12, - 0xFC, 0xC7, 0xC7, 0x1A, 0x55, 0x7E, 0x2D, 0xB9, - 0x66, 0xC3, 0xE9, 0xFA, 0x91, 0x74, 0x60, 0x39 - } -}; - static const uint8_t sha512_test_digests[][64] = { { /* for test_msg0 */ @@ -126,27 +101,6 @@ static const uint8_t sha512_test_digests[][64] = { } }; -static const uint8_t sha512_224_test_digests[][28] = { - { - /* for test_msg0 */ - 0x46, 0x34, 0x27, 0x0F, 0x70, 0x7B, 0x6A, 0x54, - 0xDA, 0xAE, 0x75, 0x30, 0x46, 0x08, 0x42, 0xE2, - 0x0E, 0x37, 0xED, 0x26, 0x5C, 0xEE, 0xE9, 0xA4, - 0x3E, 0x89, 0x24, 0xAA - }, - { - /* no test vector for test_msg1 */ - 0 - }, - { - /* for test_msg2 */ - 0x23, 0xFE, 0xC5, 0xBB, 0x94, 0xD6, 0x0B, 0x23, - 0x30, 0x81, 0x92, 0x64, 0x0B, 0x0C, 0x45, 0x33, - 0x35, 0xD6, 0x64, 0x73, 0x4F, 0xE4, 0x0E, 0x72, - 0x68, 0x67, 0x4A, 0xF9 - } -}; - static const uint8_t sha512_256_test_digests[][32] = { { /* for test_msg0 */ @@ -191,7 +145,7 @@ main(int argc, char *argv[]) do { \ SHA2_CTX ctx; \ uint8_t digest[diglen / 8]; \ - SHA2Init(SHA ## mode ## _MECH_INFO_TYPE, &ctx); \ + SHA2Init(mode, &ctx); \ SHA2Update(&ctx, _m, strlen(_m)); \ SHA2Final(digest, &ctx); \ (void) printf("SHA%-9sMessage: " #_m \ @@ -215,7 +169,7 @@ main(int argc, char *argv[]) struct timeval start, end; \ memset(block, 0, sizeof (block)); \ (void) gettimeofday(&start, NULL); \ - SHA2Init(SHA ## mode ## _MECH_INFO_TYPE, &ctx); \ + SHA2Init(mode, &ctx); \ for (i = 0; i < 8192; i++) \ SHA2Update(&ctx, block, sizeof (block)); \ SHA2Final(digest, &ctx); \ @@ -231,16 +185,12 @@ main(int argc, char *argv[]) } while (0) (void) printf("Running algorithm correctness tests:\n"); - SHA2_ALGO_TEST(test_msg0, 256, 256, sha256_test_digests[0]); - SHA2_ALGO_TEST(test_msg1, 256, 256, sha256_test_digests[1]); - SHA2_ALGO_TEST(test_msg0, 384, 384, sha384_test_digests[0]); - SHA2_ALGO_TEST(test_msg2, 384, 384, sha384_test_digests[2]); - SHA2_ALGO_TEST(test_msg0, 512, 512, sha512_test_digests[0]); - SHA2_ALGO_TEST(test_msg2, 512, 512, sha512_test_digests[2]); - SHA2_ALGO_TEST(test_msg0, 512_224, 224, sha512_224_test_digests[0]); - SHA2_ALGO_TEST(test_msg2, 512_224, 224, sha512_224_test_digests[2]); - SHA2_ALGO_TEST(test_msg0, 512_256, 256, sha512_256_test_digests[0]); - SHA2_ALGO_TEST(test_msg2, 512_256, 256, sha512_256_test_digests[2]); + SHA2_ALGO_TEST(test_msg0, SHA256, 256, sha256_test_digests[0]); + SHA2_ALGO_TEST(test_msg1, SHA256, 256, sha256_test_digests[1]); + SHA2_ALGO_TEST(test_msg0, SHA512, 512, sha512_test_digests[0]); + SHA2_ALGO_TEST(test_msg2, SHA512, 512, sha512_test_digests[2]); + SHA2_ALGO_TEST(test_msg0, SHA512_256, 256, sha512_256_test_digests[0]); + SHA2_ALGO_TEST(test_msg2, SHA512_256, 256, sha512_256_test_digests[2]); if (failed) return (1); @@ -251,13 +201,13 @@ main(int argc, char *argv[]) for (id = 0; id < sha256->getcnt(); id++) { sha256->setid(id); const char *name = sha256->getname(); - SHA2_PERF_TEST(256, 256, name); + SHA2_PERF_TEST(SHA256, 256, name); } for (id = 0; id < sha512->getcnt(); id++) { sha512->setid(id); const char *name = sha512->getname(); - SHA2_PERF_TEST(512, 512, name); + SHA2_PERF_TEST(SHA512, 512, name); } return (0); From f39241aeb333027a4dfb3f716e80d475042a348d Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Sun, 19 May 2024 15:00:44 +1000 Subject: [PATCH 08/42] icp: remove unused SHA2 HMAC mechanisms Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Reviewed-by: Brian Behlendorf Signed-off-by: Rob Norris Closes #16209 --- include/sys/crypto/common.h | 5 - include/sys/sha2.h | 5 - module/icp/algs/sha2/sha2_generic.c | 33 +----- module/icp/io/sha2_mod.c | 158 +++------------------------- 4 files changed, 13 insertions(+), 188 deletions(-) diff --git a/include/sys/crypto/common.h b/include/sys/crypto/common.h index a73804f916b..c9ef3b367e0 100644 --- a/include/sys/crypto/common.h +++ b/include/sys/crypto/common.h @@ -79,12 +79,7 @@ typedef uint32_t crypto_keysize_unit_t; /* Mechanisms supported out-of-the-box */ -#define SUN_CKM_SHA256_HMAC "CKM_SHA256_HMAC" -#define SUN_CKM_SHA256_HMAC_GENERAL "CKM_SHA256_HMAC_GENERAL" -#define SUN_CKM_SHA384_HMAC "CKM_SHA384_HMAC" -#define SUN_CKM_SHA384_HMAC_GENERAL "CKM_SHA384_HMAC_GENERAL" #define SUN_CKM_SHA512_HMAC "CKM_SHA512_HMAC" -#define SUN_CKM_SHA512_HMAC_GENERAL "CKM_SHA512_HMAC_GENERAL" #define SUN_CKM_AES_CCM "CKM_AES_CCM" #define SUN_CKM_AES_GCM "CKM_AES_GCM" diff --git a/include/sys/sha2.h b/include/sys/sha2.h index 2d38885bd96..b344eb9d5ff 100644 --- a/include/sys/sha2.h +++ b/include/sys/sha2.h @@ -86,12 +86,7 @@ typedef struct { /* SHA2 algorithm types */ typedef enum sha2_mech_type { - SHA256_HMAC_MECH_INFO_TYPE, /* SUN_CKM_SHA256_HMAC */ - SHA256_HMAC_GEN_MECH_INFO_TYPE, /* SUN_CKM_SHA256_HMAC_GENERAL */ - SHA384_HMAC_MECH_INFO_TYPE, /* SUN_CKM_SHA384_HMAC */ - SHA384_HMAC_GEN_MECH_INFO_TYPE, /* SUN_CKM_SHA384_HMAC_GENERAL */ SHA512_HMAC_MECH_INFO_TYPE, /* SUN_CKM_SHA512_HMAC */ - SHA512_HMAC_GEN_MECH_INFO_TYPE, /* SUN_CKM_SHA512_HMAC_GENERAL */ /* Not true KCF mech types; used by direct callers to SHA2Init */ SHA256, diff --git a/module/icp/algs/sha2/sha2_generic.c b/module/icp/algs/sha2/sha2_generic.c index ab361b9d59f..d53f4b69990 100644 --- a/module/icp/algs/sha2/sha2_generic.c +++ b/module/icp/algs/sha2/sha2_generic.c @@ -400,15 +400,13 @@ SHA2Init(int algotype, SHA2_CTX *ctx) sha256_ctx *ctx256 = &ctx->sha256; sha512_ctx *ctx512 = &ctx->sha512; - ASSERT3S(algotype, >=, SHA256_HMAC_MECH_INFO_TYPE); + ASSERT3S(algotype, >=, SHA512_HMAC_MECH_INFO_TYPE); ASSERT3S(algotype, <=, SHA512_256); memset(ctx, 0, sizeof (*ctx)); ctx->algotype = algotype; switch (ctx->algotype) { case SHA256: - case SHA256_HMAC_MECH_INFO_TYPE: - case SHA256_HMAC_GEN_MECH_INFO_TYPE: ctx256->state[0] = 0x6a09e667; ctx256->state[1] = 0xbb67ae85; ctx256->state[2] = 0x3c6ef372; @@ -420,23 +418,8 @@ SHA2Init(int algotype, SHA2_CTX *ctx) ctx256->count[0] = 0; ctx256->ops = sha256_get_ops(); break; - case SHA384_HMAC_MECH_INFO_TYPE: - case SHA384_HMAC_GEN_MECH_INFO_TYPE: - ctx512->state[0] = 0xcbbb9d5dc1059ed8ULL; - ctx512->state[1] = 0x629a292a367cd507ULL; - ctx512->state[2] = 0x9159015a3070dd17ULL; - ctx512->state[3] = 0x152fecd8f70e5939ULL; - ctx512->state[4] = 0x67332667ffc00b31ULL; - ctx512->state[5] = 0x8eb44a8768581511ULL; - ctx512->state[6] = 0xdb0c2e0d64f98fa7ULL; - ctx512->state[7] = 0x47b5481dbefa4fa4ULL; - ctx512->count[0] = 0; - ctx512->count[1] = 0; - ctx512->ops = sha512_get_ops(); - break; case SHA512: case SHA512_HMAC_MECH_INFO_TYPE: - case SHA512_HMAC_GEN_MECH_INFO_TYPE: ctx512->state[0] = 0x6a09e667f3bcc908ULL; ctx512->state[1] = 0xbb67ae8584caa73bULL; ctx512->state[2] = 0x3c6ef372fe94f82bULL; @@ -477,17 +460,10 @@ SHA2Update(SHA2_CTX *ctx, const void *data, size_t len) switch (ctx->algotype) { case SHA256: - case SHA256_HMAC_MECH_INFO_TYPE: - case SHA256_HMAC_GEN_MECH_INFO_TYPE: sha256_update(&ctx->sha256, data, len); break; - case SHA384_HMAC_MECH_INFO_TYPE: - case SHA384_HMAC_GEN_MECH_INFO_TYPE: - sha512_update(&ctx->sha512, data, len); - break; case SHA512: case SHA512_HMAC_MECH_INFO_TYPE: - case SHA512_HMAC_GEN_MECH_INFO_TYPE: sha512_update(&ctx->sha512, data, len); break; case SHA512_256: @@ -502,17 +478,10 @@ SHA2Final(void *digest, SHA2_CTX *ctx) { switch (ctx->algotype) { case SHA256: - case SHA256_HMAC_MECH_INFO_TYPE: - case SHA256_HMAC_GEN_MECH_INFO_TYPE: sha256_final(&ctx->sha256, digest, 256); break; - case SHA384_HMAC_MECH_INFO_TYPE: - case SHA384_HMAC_GEN_MECH_INFO_TYPE: - sha512_final(&ctx->sha512, digest, 384); - break; case SHA512: case SHA512_HMAC_MECH_INFO_TYPE: - case SHA512_HMAC_GEN_MECH_INFO_TYPE: sha512_final(&ctx->sha512, digest, 512); break; case SHA512_256: diff --git a/module/icp/io/sha2_mod.c b/module/icp/io/sha2_mod.c index d80ea1e677b..e515dabc9dd 100644 --- a/module/icp/io/sha2_mod.c +++ b/module/icp/io/sha2_mod.c @@ -60,24 +60,9 @@ * Mechanism info structure passed to KCF during registration. */ static const crypto_mech_info_t sha2_mech_info_tab[] = { - /* SHA256-HMAC */ - {SUN_CKM_SHA256_HMAC, SHA256_HMAC_MECH_INFO_TYPE, - CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, - /* SHA256-HMAC GENERAL */ - {SUN_CKM_SHA256_HMAC_GENERAL, SHA256_HMAC_GEN_MECH_INFO_TYPE, - CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, - /* SHA384-HMAC */ - {SUN_CKM_SHA384_HMAC, SHA384_HMAC_MECH_INFO_TYPE, - CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, - /* SHA384-HMAC GENERAL */ - {SUN_CKM_SHA384_HMAC_GENERAL, SHA384_HMAC_GEN_MECH_INFO_TYPE, - CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, /* SHA512-HMAC */ {SUN_CKM_SHA512_HMAC, SHA512_HMAC_MECH_INFO_TYPE, CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, - /* SHA512-HMAC GENERAL */ - {SUN_CKM_SHA512_HMAC_GENERAL, SHA512_HMAC_GEN_MECH_INFO_TYPE, - CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, }; static int sha2_mac_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *, @@ -251,10 +236,8 @@ sha2_digest_final_uio(SHA2_CTX *sha2_ctx, crypto_data_t *digest, * The computed SHA2 digest will fit in the current * iovec. */ - if (((sha2_ctx->algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) && - (digest_len != SHA256_DIGEST_LENGTH)) || - ((sha2_ctx->algotype > SHA256_HMAC_GEN_MECH_INFO_TYPE) && - (digest_len != SHA512_DIGEST_LENGTH))) { + ASSERT3U(sha2_ctx->algotype, ==, SHA512_HMAC_MECH_INFO_TYPE); + if (digest_len != SHA512_DIGEST_LENGTH) { /* * The caller requested a short digest. Digest * into a scratch buffer and return to @@ -349,13 +332,9 @@ sha2_mac_init_ctx(sha2_hmac_ctx_t *ctx, void *keyval, uint_t length_in_bytes) int i, block_size, blocks_per_int64; /* Determine the block size */ - if (ctx->hc_mech_type <= SHA256_HMAC_GEN_MECH_INFO_TYPE) { - block_size = SHA256_HMAC_BLOCK_SIZE; - blocks_per_int64 = SHA256_HMAC_BLOCK_SIZE / sizeof (uint64_t); - } else { - block_size = SHA512_HMAC_BLOCK_SIZE; - blocks_per_int64 = SHA512_HMAC_BLOCK_SIZE / sizeof (uint64_t); - } + ASSERT3U(ctx->hc_mech_type, ==, SHA512_HMAC_MECH_INFO_TYPE); + block_size = SHA512_HMAC_BLOCK_SIZE; + blocks_per_int64 = SHA512_HMAC_BLOCK_SIZE / sizeof (uint64_t); (void) memset(ipad, 0, block_size); (void) memset(opad, 0, block_size); @@ -397,15 +376,7 @@ sha2_mac_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, * mechanism */ switch (mechanism->cm_type) { - case SHA256_HMAC_MECH_INFO_TYPE: - case SHA256_HMAC_GEN_MECH_INFO_TYPE: - sha_digest_len = SHA256_DIGEST_LENGTH; - sha_hmac_block_size = SHA256_HMAC_BLOCK_SIZE; - break; - case SHA384_HMAC_MECH_INFO_TYPE: - case SHA384_HMAC_GEN_MECH_INFO_TYPE: case SHA512_HMAC_MECH_INFO_TYPE: - case SHA512_HMAC_GEN_MECH_INFO_TYPE: sha_digest_len = SHA512_DIGEST_LENGTH; sha_hmac_block_size = SHA512_HMAC_BLOCK_SIZE; break; @@ -445,22 +416,6 @@ sha2_mac_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, } } - /* - * Get the mechanism parameters, if applicable. - */ - if (mechanism->cm_type % 3 == 2) { - if (mechanism->cm_param == NULL || - mechanism->cm_param_len != sizeof (ulong_t)) { - ret = CRYPTO_MECHANISM_PARAM_INVALID; - } else { - PROV_SHA2_GET_DIGEST_LEN(mechanism, - PROV_SHA2_HMAC_CTX(ctx)->hc_digest_len); - if (PROV_SHA2_HMAC_CTX(ctx)->hc_digest_len > - sha_digest_len) - ret = CRYPTO_MECHANISM_PARAM_INVALID; - } - } - if (ret != CRYPTO_SUCCESS) { memset(ctx->cc_provider_private, 0, sizeof (sha2_hmac_ctx_t)); kmem_free(ctx->cc_provider_private, sizeof (sha2_hmac_ctx_t)); @@ -509,24 +464,9 @@ sha2_mac_final(crypto_ctx_t *ctx, crypto_data_t *mac) /* Set the digest lengths to values appropriate to the mechanism */ switch (PROV_SHA2_HMAC_CTX(ctx)->hc_mech_type) { - case SHA256_HMAC_MECH_INFO_TYPE: - sha_digest_len = digest_len = SHA256_DIGEST_LENGTH; - break; - case SHA384_HMAC_MECH_INFO_TYPE: - sha_digest_len = digest_len = SHA384_DIGEST_LENGTH; - break; case SHA512_HMAC_MECH_INFO_TYPE: sha_digest_len = digest_len = SHA512_DIGEST_LENGTH; break; - case SHA256_HMAC_GEN_MECH_INFO_TYPE: - sha_digest_len = SHA256_DIGEST_LENGTH; - digest_len = PROV_SHA2_HMAC_CTX(ctx)->hc_digest_len; - break; - case SHA384_HMAC_GEN_MECH_INFO_TYPE: - case SHA512_HMAC_GEN_MECH_INFO_TYPE: - sha_digest_len = SHA512_DIGEST_LENGTH; - digest_len = PROV_SHA2_HMAC_CTX(ctx)->hc_digest_len; - break; default: return (CRYPTO_ARGUMENTS_BAD); } @@ -626,15 +566,7 @@ sha2_mac_atomic(crypto_mechanism_t *mechanism, * mechanism */ switch (mechanism->cm_type) { - case SHA256_HMAC_MECH_INFO_TYPE: - case SHA256_HMAC_GEN_MECH_INFO_TYPE: - sha_digest_len = digest_len = SHA256_DIGEST_LENGTH; - sha_hmac_block_size = SHA256_HMAC_BLOCK_SIZE; - break; - case SHA384_HMAC_MECH_INFO_TYPE: - case SHA384_HMAC_GEN_MECH_INFO_TYPE: case SHA512_HMAC_MECH_INFO_TYPE: - case SHA512_HMAC_GEN_MECH_INFO_TYPE: sha_digest_len = digest_len = SHA512_DIGEST_LENGTH; sha_hmac_block_size = SHA512_HMAC_BLOCK_SIZE; break; @@ -665,20 +597,6 @@ sha2_mac_atomic(crypto_mechanism_t *mechanism, } } - /* get the mechanism parameters, if applicable */ - if ((mechanism->cm_type % 3) == 2) { - if (mechanism->cm_param == NULL || - mechanism->cm_param_len != sizeof (ulong_t)) { - ret = CRYPTO_MECHANISM_PARAM_INVALID; - goto bail; - } - PROV_SHA2_GET_DIGEST_LEN(mechanism, digest_len); - if (digest_len > sha_digest_len) { - ret = CRYPTO_MECHANISM_PARAM_INVALID; - goto bail; - } - } - /* do a SHA2 update of the inner context using the specified data */ SHA2_MAC_UPDATE(data, sha2_hmac_ctx, ret); if (ret != CRYPTO_SUCCESS) @@ -693,16 +611,9 @@ sha2_mac_atomic(crypto_mechanism_t *mechanism, /* * Do an SHA2 update on the outer context, feeding the inner * digest as data. - * - * HMAC-SHA384 needs special handling as the outer hash needs only 48 - * bytes of the inner hash value. */ - if (mechanism->cm_type == SHA384_HMAC_MECH_INFO_TYPE || - mechanism->cm_type == SHA384_HMAC_GEN_MECH_INFO_TYPE) - SHA2Update(&sha2_hmac_ctx.hc_ocontext, digest, - SHA384_DIGEST_LENGTH); - else - SHA2Update(&sha2_hmac_ctx.hc_ocontext, digest, sha_digest_len); + ASSERT3U(mechanism->cm_type, ==, SHA512_HMAC_MECH_INFO_TYPE); + SHA2Update(&sha2_hmac_ctx.hc_ocontext, digest, sha_digest_len); /* * Do a SHA2 final on the outer context, storing the computed @@ -758,15 +669,7 @@ sha2_mac_verify_atomic(crypto_mechanism_t *mechanism, * mechanism */ switch (mechanism->cm_type) { - case SHA256_HMAC_MECH_INFO_TYPE: - case SHA256_HMAC_GEN_MECH_INFO_TYPE: - sha_digest_len = digest_len = SHA256_DIGEST_LENGTH; - sha_hmac_block_size = SHA256_HMAC_BLOCK_SIZE; - break; - case SHA384_HMAC_MECH_INFO_TYPE: - case SHA384_HMAC_GEN_MECH_INFO_TYPE: case SHA512_HMAC_MECH_INFO_TYPE: - case SHA512_HMAC_GEN_MECH_INFO_TYPE: sha_digest_len = digest_len = SHA512_DIGEST_LENGTH; sha_hmac_block_size = SHA512_HMAC_BLOCK_SIZE; break; @@ -797,20 +700,6 @@ sha2_mac_verify_atomic(crypto_mechanism_t *mechanism, } } - /* get the mechanism parameters, if applicable */ - if (mechanism->cm_type % 3 == 2) { - if (mechanism->cm_param == NULL || - mechanism->cm_param_len != sizeof (ulong_t)) { - ret = CRYPTO_MECHANISM_PARAM_INVALID; - goto bail; - } - PROV_SHA2_GET_DIGEST_LEN(mechanism, digest_len); - if (digest_len > sha_digest_len) { - ret = CRYPTO_MECHANISM_PARAM_INVALID; - goto bail; - } - } - if (mac->cd_length != digest_len) { ret = CRYPTO_INVALID_MAC; goto bail; @@ -828,16 +717,9 @@ sha2_mac_verify_atomic(crypto_mechanism_t *mechanism, /* * Do an SHA2 update on the outer context, feeding the inner * digest as data. - * - * HMAC-SHA384 needs special handling as the outer hash needs only 48 - * bytes of the inner hash value. */ - if (mechanism->cm_type == SHA384_HMAC_MECH_INFO_TYPE || - mechanism->cm_type == SHA384_HMAC_GEN_MECH_INFO_TYPE) - SHA2Update(&sha2_hmac_ctx.hc_ocontext, digest, - SHA384_DIGEST_LENGTH); - else - SHA2Update(&sha2_hmac_ctx.hc_ocontext, digest, sha_digest_len); + ASSERT3U(mechanism->cm_type, ==, SHA512_HMAC_MECH_INFO_TYPE); + SHA2Update(&sha2_hmac_ctx.hc_ocontext, digest, sha_digest_len); /* * Do a SHA2 final on the outer context, storing the computed @@ -929,15 +811,7 @@ sha2_create_ctx_template(crypto_mechanism_t *mechanism, crypto_key_t *key, * mechanism */ switch (mechanism->cm_type) { - case SHA256_HMAC_MECH_INFO_TYPE: - case SHA256_HMAC_GEN_MECH_INFO_TYPE: - sha_digest_len = SHA256_DIGEST_LENGTH; - sha_hmac_block_size = SHA256_HMAC_BLOCK_SIZE; - break; - case SHA384_HMAC_MECH_INFO_TYPE: - case SHA384_HMAC_GEN_MECH_INFO_TYPE: case SHA512_HMAC_MECH_INFO_TYPE: - case SHA512_HMAC_GEN_MECH_INFO_TYPE: sha_digest_len = SHA512_DIGEST_LENGTH; sha_hmac_block_size = SHA512_HMAC_BLOCK_SIZE; break; @@ -986,17 +860,9 @@ sha2_free_context(crypto_ctx_t *ctx) if (ctx->cc_provider_private == NULL) return (CRYPTO_SUCCESS); - /* - * We have to free either SHA2 or SHA2-HMAC contexts, which - * have different lengths. - * - * Note: Below is dependent on the mechanism ordering. - */ - - if (PROV_SHA2_CTX(ctx)->sc_mech_type % 3 == 0) - ctx_len = sizeof (sha2_ctx_t); - else - ctx_len = sizeof (sha2_hmac_ctx_t); + ASSERT3U(PROV_SHA2_CTX(ctx)->sc_mech_type, ==, + SHA512_HMAC_MECH_INFO_TYPE); + ctx_len = sizeof (sha2_hmac_ctx_t); memset(ctx->cc_provider_private, 0, ctx_len); kmem_free(ctx->cc_provider_private, ctx_len); From ae512620d0c372d83180204f3149d9a5df814931 Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Sun, 19 May 2024 15:00:00 +1000 Subject: [PATCH 09/42] icp: remove skein module Nothing calls it through the KCF interface, so this is all unused. Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Reviewed-by: Brian Behlendorf Signed-off-by: Rob Norris Closes #16209 --- include/sys/crypto/icp.h | 3 - lib/libicp/Makefile.am | 1 - module/Kbuild.in | 1 - module/icp/illumos-crypto.c | 2 - module/icp/io/skein_mod.c | 515 ------------------------------------ 5 files changed, 522 deletions(-) delete mode 100644 module/icp/io/skein_mod.c diff --git a/include/sys/crypto/icp.h b/include/sys/crypto/icp.h index 8c3f19886fd..efe283fa928 100644 --- a/include/sys/crypto/icp.h +++ b/include/sys/crypto/icp.h @@ -32,9 +32,6 @@ int aes_mod_fini(void); int sha2_mod_init(void); int sha2_mod_fini(void); -int skein_mod_init(void); -int skein_mod_fini(void); - int icp_init(void); void icp_fini(void); diff --git a/lib/libicp/Makefile.am b/lib/libicp/Makefile.am index f40512bec9c..ce24d13a760 100644 --- a/lib/libicp/Makefile.am +++ b/lib/libicp/Makefile.am @@ -31,7 +31,6 @@ nodist_libicp_la_SOURCES = \ module/icp/illumos-crypto.c \ module/icp/io/aes.c \ module/icp/io/sha2_mod.c \ - module/icp/io/skein_mod.c \ module/icp/core/kcf_sched.c \ module/icp/core/kcf_prov_lib.c \ module/icp/core/kcf_callprov.c \ diff --git a/module/Kbuild.in b/module/Kbuild.in index 6e2eab22588..9e44364b758 100644 --- a/module/Kbuild.in +++ b/module/Kbuild.in @@ -122,7 +122,6 @@ ICP_OBJS := \ illumos-crypto.o \ io/aes.o \ io/sha2_mod.o \ - io/skein_mod.o \ spi/kcf_spi.o ICP_OBJS_X86_64 := \ diff --git a/module/icp/illumos-crypto.c b/module/icp/illumos-crypto.c index 13f05c06ed5..f5ed3e13fac 100644 --- a/module/icp/illumos-crypto.c +++ b/module/icp/illumos-crypto.c @@ -107,7 +107,6 @@ void icp_fini(void) { - skein_mod_fini(); sha2_mod_fini(); aes_mod_fini(); kcf_sched_destroy(); @@ -134,7 +133,6 @@ icp_init(void) /* initialize algorithms */ aes_mod_init(); sha2_mod_init(); - skein_mod_init(); return (0); } diff --git a/module/icp/io/skein_mod.c b/module/icp/io/skein_mod.c deleted file mode 100644 index 3e969513be6..00000000000 --- a/module/icp/io/skein_mod.c +++ /dev/null @@ -1,515 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://opensource.org/licenses/CDDL-1.0. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ - -/* - * Copyright 2013 Saso Kiselkov. All rights reserved. - */ - -#include -#include -#include -#include -#define SKEIN_MODULE_IMPL -#include - -static const crypto_mech_info_t skein_mech_info_tab[] = { - {CKM_SKEIN_256_MAC, SKEIN_256_MAC_MECH_INFO_TYPE, - CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, - {CKM_SKEIN_512_MAC, SKEIN_512_MAC_MECH_INFO_TYPE, - CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, - {CKM_SKEIN1024_MAC, SKEIN1024_MAC_MECH_INFO_TYPE, - CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC}, -}; - -static int skein_update(crypto_ctx_t *, crypto_data_t *); -static int skein_final(crypto_ctx_t *, crypto_data_t *); - -static int skein_mac_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *, - crypto_spi_ctx_template_t); -static int skein_mac_atomic(crypto_mechanism_t *, crypto_key_t *, - crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t); - -static const crypto_mac_ops_t skein_mac_ops = { - .mac_init = skein_mac_init, - .mac = NULL, - .mac_update = skein_update, /* using regular digest update is OK here */ - .mac_final = skein_final, /* using regular digest final is OK here */ - .mac_atomic = skein_mac_atomic, - .mac_verify_atomic = NULL -}; - -static int skein_create_ctx_template(crypto_mechanism_t *, crypto_key_t *, - crypto_spi_ctx_template_t *, size_t *); -static int skein_free_context(crypto_ctx_t *); - -static const crypto_ctx_ops_t skein_ctx_ops = { - .create_ctx_template = skein_create_ctx_template, - .free_context = skein_free_context -}; - -static const crypto_ops_t skein_crypto_ops = { - NULL, - &skein_mac_ops, - &skein_ctx_ops, -}; - -static const crypto_provider_info_t skein_prov_info = { - "Skein Software Provider", - &skein_crypto_ops, - sizeof (skein_mech_info_tab) / sizeof (crypto_mech_info_t), - skein_mech_info_tab -}; - -static crypto_kcf_provider_handle_t skein_prov_handle = 0; - -typedef struct skein_ctx { - skein_mech_type_t sc_mech_type; - size_t sc_digest_bitlen; - /*LINTED(E_ANONYMOUS_UNION_DECL)*/ - union { - Skein_256_Ctxt_t sc_256; - Skein_512_Ctxt_t sc_512; - Skein1024_Ctxt_t sc_1024; - }; -} skein_ctx_t; -#define SKEIN_CTX(_ctx_) ((skein_ctx_t *)((_ctx_)->cc_provider_private)) -#define SKEIN_CTX_LVALUE(_ctx_) (_ctx_)->cc_provider_private -#define SKEIN_OP(_skein_ctx, _op, ...) \ - do { \ - skein_ctx_t *sc = (_skein_ctx); \ - switch (sc->sc_mech_type) { \ - case SKEIN_256_MAC_MECH_INFO_TYPE: \ - (void) Skein_256_ ## _op(&sc->sc_256, __VA_ARGS__);\ - break; \ - case SKEIN_512_MAC_MECH_INFO_TYPE: \ - (void) Skein_512_ ## _op(&sc->sc_512, __VA_ARGS__);\ - break; \ - case SKEIN1024_MAC_MECH_INFO_TYPE: \ - (void) Skein1024_ ## _op(&sc->sc_1024, __VA_ARGS__);\ - break; \ - } \ - } while (0) - -static int -skein_get_digest_bitlen(const crypto_mechanism_t *mechanism, size_t *result) -{ - if (mechanism->cm_param != NULL) { - /*LINTED(E_BAD_PTR_CAST_ALIGN)*/ - skein_param_t *param = (skein_param_t *)mechanism->cm_param; - - if (mechanism->cm_param_len != sizeof (*param) || - param->sp_digest_bitlen == 0) { - return (CRYPTO_MECHANISM_PARAM_INVALID); - } - *result = param->sp_digest_bitlen; - } else { - return (CRYPTO_MECHANISM_INVALID); - } - return (CRYPTO_SUCCESS); -} - -int -skein_mod_init(void) -{ - /* - * Try to register with KCF - failure shouldn't unload us, since we - * still may want to continue providing misc/skein functionality. - */ - (void) crypto_register_provider(&skein_prov_info, &skein_prov_handle); - - return (0); -} - -int -skein_mod_fini(void) -{ - int ret = 0; - - if (skein_prov_handle != 0) { - if ((ret = crypto_unregister_provider(skein_prov_handle)) != - CRYPTO_SUCCESS) { - cmn_err(CE_WARN, - "skein _fini: crypto_unregister_provider() " - "failed (0x%x)", ret); - return (EBUSY); - } - skein_prov_handle = 0; - } - - return (0); -} - -/* - * General Skein hashing helper functions. - */ - -/* - * Performs an Update on a context with uio input data. - */ -static int -skein_digest_update_uio(skein_ctx_t *ctx, const crypto_data_t *data) -{ - off_t offset = data->cd_offset; - size_t length = data->cd_length; - uint_t vec_idx = 0; - size_t cur_len; - zfs_uio_t *uio = data->cd_uio; - - /* we support only kernel buffer */ - if (zfs_uio_segflg(uio) != UIO_SYSSPACE) - return (CRYPTO_ARGUMENTS_BAD); - - /* - * Jump to the first iovec containing data to be - * digested. - */ - offset = zfs_uio_index_at_offset(uio, offset, &vec_idx); - if (vec_idx == zfs_uio_iovcnt(uio)) { - /* - * The caller specified an offset that is larger than the - * total size of the buffers it provided. - */ - return (CRYPTO_DATA_LEN_RANGE); - } - - /* - * Now do the digesting on the iovecs. - */ - while (vec_idx < zfs_uio_iovcnt(uio) && length > 0) { - cur_len = MIN(zfs_uio_iovlen(uio, vec_idx) - offset, length); - SKEIN_OP(ctx, Update, (uint8_t *)zfs_uio_iovbase(uio, vec_idx) - + offset, cur_len); - length -= cur_len; - vec_idx++; - offset = 0; - } - - if (vec_idx == zfs_uio_iovcnt(uio) && length > 0) { - /* - * The end of the specified iovec's was reached but - * the length requested could not be processed, i.e. - * The caller requested to digest more data than it provided. - */ - return (CRYPTO_DATA_LEN_RANGE); - } - - return (CRYPTO_SUCCESS); -} - -/* - * Performs a Final on a context and writes to a uio digest output. - */ -static int -skein_digest_final_uio(skein_ctx_t *ctx, crypto_data_t *digest) -{ - off_t offset = digest->cd_offset; - uint_t vec_idx = 0; - zfs_uio_t *uio = digest->cd_uio; - - /* we support only kernel buffer */ - if (zfs_uio_segflg(uio) != UIO_SYSSPACE) - return (CRYPTO_ARGUMENTS_BAD); - - /* - * Jump to the first iovec containing ptr to the digest to be returned. - */ - offset = zfs_uio_index_at_offset(uio, offset, &vec_idx); - if (vec_idx == zfs_uio_iovcnt(uio)) { - /* - * The caller specified an offset that is larger than the - * total size of the buffers it provided. - */ - return (CRYPTO_DATA_LEN_RANGE); - } - if (offset + CRYPTO_BITS2BYTES(ctx->sc_digest_bitlen) <= - zfs_uio_iovlen(uio, vec_idx)) { - /* The computed digest will fit in the current iovec. */ - SKEIN_OP(ctx, Final, - (uchar_t *)zfs_uio_iovbase(uio, vec_idx) + offset); - } else { - uint8_t *digest_tmp; - off_t scratch_offset = 0; - size_t length = CRYPTO_BITS2BYTES(ctx->sc_digest_bitlen); - size_t cur_len; - - digest_tmp = kmem_alloc(CRYPTO_BITS2BYTES( - ctx->sc_digest_bitlen), KM_SLEEP); - if (digest_tmp == NULL) - return (CRYPTO_HOST_MEMORY); - SKEIN_OP(ctx, Final, digest_tmp); - while (vec_idx < zfs_uio_iovcnt(uio) && length > 0) { - cur_len = MIN(zfs_uio_iovlen(uio, vec_idx) - offset, - length); - memcpy(zfs_uio_iovbase(uio, vec_idx) + offset, - digest_tmp + scratch_offset, cur_len); - - length -= cur_len; - vec_idx++; - scratch_offset += cur_len; - offset = 0; - } - kmem_free(digest_tmp, CRYPTO_BITS2BYTES(ctx->sc_digest_bitlen)); - - if (vec_idx == zfs_uio_iovcnt(uio) && length > 0) { - /* - * The end of the specified iovec's was reached but - * the length requested could not be processed, i.e. - * The caller requested to digest more data than it - * provided. - */ - return (CRYPTO_DATA_LEN_RANGE); - } - } - - return (CRYPTO_SUCCESS); -} - -/* - * KCF software provider digest entry points. - */ - -/* - * Performs a skein Update with the input message in `data' (successive calls - * can push more data). This is used both for digest and MAC operation. - * Supported input data formats are raw, uio and mblk. - */ -static int -skein_update(crypto_ctx_t *ctx, crypto_data_t *data) -{ - int error = CRYPTO_SUCCESS; - - ASSERT(SKEIN_CTX(ctx) != NULL); - - switch (data->cd_format) { - case CRYPTO_DATA_RAW: - SKEIN_OP(SKEIN_CTX(ctx), Update, - (uint8_t *)data->cd_raw.iov_base + data->cd_offset, - data->cd_length); - break; - case CRYPTO_DATA_UIO: - error = skein_digest_update_uio(SKEIN_CTX(ctx), data); - break; - default: - error = CRYPTO_ARGUMENTS_BAD; - } - - return (error); -} - -/* - * Performs a skein Final, writing the output to `digest'. This is used both - * for digest and MAC operation. - * Supported output digest formats are raw, uio and mblk. - */ -static int -skein_final_nofree(crypto_ctx_t *ctx, crypto_data_t *digest) -{ - int error = CRYPTO_SUCCESS; - - ASSERT(SKEIN_CTX(ctx) != NULL); - - if (digest->cd_length < - CRYPTO_BITS2BYTES(SKEIN_CTX(ctx)->sc_digest_bitlen)) { - digest->cd_length = - CRYPTO_BITS2BYTES(SKEIN_CTX(ctx)->sc_digest_bitlen); - return (CRYPTO_BUFFER_TOO_SMALL); - } - - switch (digest->cd_format) { - case CRYPTO_DATA_RAW: - SKEIN_OP(SKEIN_CTX(ctx), Final, - (uint8_t *)digest->cd_raw.iov_base + digest->cd_offset); - break; - case CRYPTO_DATA_UIO: - error = skein_digest_final_uio(SKEIN_CTX(ctx), digest); - break; - default: - error = CRYPTO_ARGUMENTS_BAD; - } - - if (error == CRYPTO_SUCCESS) - digest->cd_length = - CRYPTO_BITS2BYTES(SKEIN_CTX(ctx)->sc_digest_bitlen); - else - digest->cd_length = 0; - - return (error); -} - -static int -skein_final(crypto_ctx_t *ctx, crypto_data_t *digest) -{ - int error = skein_final_nofree(ctx, digest); - - if (error == CRYPTO_BUFFER_TOO_SMALL) - return (error); - - memset(SKEIN_CTX(ctx), 0, sizeof (*SKEIN_CTX(ctx))); - kmem_free(SKEIN_CTX(ctx), sizeof (*(SKEIN_CTX(ctx)))); - SKEIN_CTX_LVALUE(ctx) = NULL; - - return (error); -} - -/* - * Helper function that builds a Skein MAC context from the provided - * mechanism and key. - */ -static int -skein_mac_ctx_build(skein_ctx_t *ctx, crypto_mechanism_t *mechanism, - crypto_key_t *key) -{ - int error; - - if (!VALID_SKEIN_MAC_MECH(mechanism->cm_type)) - return (CRYPTO_MECHANISM_INVALID); - ctx->sc_mech_type = mechanism->cm_type; - error = skein_get_digest_bitlen(mechanism, &ctx->sc_digest_bitlen); - if (error != CRYPTO_SUCCESS) - return (error); - SKEIN_OP(ctx, InitExt, ctx->sc_digest_bitlen, 0, key->ck_data, - CRYPTO_BITS2BYTES(key->ck_length)); - - return (CRYPTO_SUCCESS); -} - -/* - * KCF software provide mac entry points. - */ -/* - * Initializes a skein MAC context. You may pass a ctx_template, in which - * case the template will be reused to make initialization more efficient. - * Otherwise a new context will be constructed. The mechanism cm_type must - * be one of SKEIN_*_MAC_MECH_INFO_TYPE. Same as in skein_digest_init, you - * may pass a skein_param_t in cm_param to configure the length of the - * digest. The key must be in raw format. - */ -static int -skein_mac_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, - crypto_key_t *key, crypto_spi_ctx_template_t ctx_template) -{ - int error; - - SKEIN_CTX_LVALUE(ctx) = kmem_alloc(sizeof (*SKEIN_CTX(ctx)), KM_SLEEP); - if (SKEIN_CTX(ctx) == NULL) - return (CRYPTO_HOST_MEMORY); - - if (ctx_template != NULL) { - memcpy(SKEIN_CTX(ctx), ctx_template, - sizeof (*SKEIN_CTX(ctx))); - } else { - error = skein_mac_ctx_build(SKEIN_CTX(ctx), mechanism, key); - if (error != CRYPTO_SUCCESS) - goto errout; - } - - return (CRYPTO_SUCCESS); -errout: - memset(SKEIN_CTX(ctx), 0, sizeof (*SKEIN_CTX(ctx))); - kmem_free(SKEIN_CTX(ctx), sizeof (*SKEIN_CTX(ctx))); - return (error); -} - -/* - * The MAC update and final calls are reused from the regular digest code. - */ - -/* - * Same as skein_digest_atomic, performs an atomic Skein MAC operation in - * one step. All the same properties apply to the arguments of this - * function as to those of the partial operations above. - */ -static int -skein_mac_atomic(crypto_mechanism_t *mechanism, - crypto_key_t *key, crypto_data_t *data, crypto_data_t *mac, - crypto_spi_ctx_template_t ctx_template) -{ - /* faux crypto context just for skein_digest_{update,final} */ - int error; - crypto_ctx_t ctx; - skein_ctx_t skein_ctx; - SKEIN_CTX_LVALUE(&ctx) = &skein_ctx; - - if (ctx_template != NULL) { - memcpy(&skein_ctx, ctx_template, sizeof (skein_ctx)); - } else { - error = skein_mac_ctx_build(&skein_ctx, mechanism, key); - if (error != CRYPTO_SUCCESS) - goto errout; - } - - if ((error = skein_update(&ctx, data)) != CRYPTO_SUCCESS) - goto errout; - if ((error = skein_final_nofree(&ctx, mac)) != CRYPTO_SUCCESS) - goto errout; - - return (CRYPTO_SUCCESS); -errout: - memset(&skein_ctx, 0, sizeof (skein_ctx)); - return (error); -} - -/* - * KCF software provider context management entry points. - */ - -/* - * Constructs a context template for the Skein MAC algorithm. The same - * properties apply to the arguments of this function as to those of - * skein_mac_init. - */ -static int -skein_create_ctx_template(crypto_mechanism_t *mechanism, crypto_key_t *key, - crypto_spi_ctx_template_t *ctx_template, size_t *ctx_template_size) -{ - int error; - skein_ctx_t *ctx_tmpl; - - ctx_tmpl = kmem_alloc(sizeof (*ctx_tmpl), KM_SLEEP); - if (ctx_tmpl == NULL) - return (CRYPTO_HOST_MEMORY); - error = skein_mac_ctx_build(ctx_tmpl, mechanism, key); - if (error != CRYPTO_SUCCESS) - goto errout; - *ctx_template = ctx_tmpl; - *ctx_template_size = sizeof (*ctx_tmpl); - - return (CRYPTO_SUCCESS); -errout: - memset(ctx_tmpl, 0, sizeof (*ctx_tmpl)); - kmem_free(ctx_tmpl, sizeof (*ctx_tmpl)); - return (error); -} - -/* - * Frees a skein context in a parent crypto context. - */ -static int -skein_free_context(crypto_ctx_t *ctx) -{ - if (SKEIN_CTX(ctx) != NULL) { - memset(SKEIN_CTX(ctx), 0, sizeof (*SKEIN_CTX(ctx))); - kmem_free(SKEIN_CTX(ctx), sizeof (*SKEIN_CTX(ctx))); - SKEIN_CTX_LVALUE(ctx) = NULL; - } - - return (CRYPTO_SUCCESS); -} From 4e714c0be10f53eea2d87e6af67bf46d67e94db2 Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Sun, 19 May 2024 21:40:59 +1000 Subject: [PATCH 10/42] icp: remove unused headers Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Reviewed-by: Brian Behlendorf Signed-off-by: Rob Norris Closes #16209 --- module/icp/include/sys/stack.h | 36 ---------------------------------- module/icp/include/sys/trap.h | 36 ---------------------------------- 2 files changed, 72 deletions(-) delete mode 100644 module/icp/include/sys/stack.h delete mode 100644 module/icp/include/sys/trap.h diff --git a/module/icp/include/sys/stack.h b/module/icp/include/sys/stack.h deleted file mode 100644 index 0bace018b5a..00000000000 --- a/module/icp/include/sys/stack.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or https://opensource.org/licenses/CDDL-1.0. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2005 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#ifndef _SYS_STACK_H -#define _SYS_STACK_H - -#if defined(__i386) || defined(__amd64) - -#include /* XX64 x86/sys/stack.h */ - -#endif - -#endif /* _SYS_STACK_H */ diff --git a/module/icp/include/sys/trap.h b/module/icp/include/sys/trap.h deleted file mode 100644 index 2f47d43939c..00000000000 --- a/module/icp/include/sys/trap.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or https://opensource.org/licenses/CDDL-1.0. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2005 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#ifndef _SYS_TRAP_H -#define _SYS_TRAP_H - -#if defined(__i386) || defined(__amd64) - -#include /* XX64 x86/sys/trap.h */ - -#endif - -#endif /* _SYS_TRAP_H */ From a72751a34265492efc115bb6a773e961bb9be82d Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Sun, 19 May 2024 21:49:19 +1000 Subject: [PATCH 11/42] icp: remove redundant FreeBSD check We don't build illumos-crypto for FreeBSD. Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Reviewed-by: Brian Behlendorf Signed-off-by: Rob Norris Closes #16209 --- module/icp/illumos-crypto.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/module/icp/illumos-crypto.c b/module/icp/illumos-crypto.c index f5ed3e13fac..89736a61bc8 100644 --- a/module/icp/illumos-crypto.c +++ b/module/icp/illumos-crypto.c @@ -136,8 +136,3 @@ icp_init(void) return (0); } - -#if defined(_KERNEL) && defined(__FreeBSD__) -module_exit(icp_fini); -module_init(icp_init); -#endif From 23a489a41167890cdd227366a5f950170df8cc9b Mon Sep 17 00:00:00 2001 From: Ameer Hamza Date: Tue, 4 Jun 2024 04:28:43 +0500 Subject: [PATCH 12/42] zdb: detect cachefile automatically otherwise force import If a pool is created with the cache file located in a non-default path /etc/default/zpool.cache, removed, or the cachefile property is set to none, zdb fails to show the pool unless we specify the cache file or use the -e option. This PR automates this process. Reviewed-by: Brian Behlendorf Reviewed-by: Alexander Motin Reviewed-by: Akash B Signed-off-by: Ameer Hamza Closes #16071 --- cmd/zdb/zdb.c | 66 ++++++++++++++++++++++++++++++--- include/libzfs_core.h | 1 + lib/libzfs_core/libzfs_core.abi | 6 +++ lib/libzfs_core/libzfs_core.c | 6 +++ 4 files changed, 74 insertions(+), 5 deletions(-) diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c index 704fcf4422d..2a3d58d77e0 100644 --- a/cmd/zdb/zdb.c +++ b/cmd/zdb/zdb.c @@ -89,6 +89,7 @@ #include #include +#include #include @@ -8924,6 +8925,9 @@ main(int argc, char **argv) boolean_t target_is_spa = B_TRUE, dataset_lookup = B_FALSE; nvlist_t *cfg = NULL; struct sigaction action; + boolean_t force_import = B_FALSE; + boolean_t config_path_console = B_FALSE; + char pbuf[MAXPATHLEN]; dprintf_setup(&argc, argv); @@ -9094,6 +9098,7 @@ main(int argc, char **argv) } break; case 'U': + config_path_console = B_TRUE; spa_config_path = optarg; if (spa_config_path[0] != '/') { (void) fprintf(stderr, @@ -9153,9 +9158,6 @@ main(int argc, char **argv) */ spa_mode_readable_spacemaps = B_TRUE; - kernel_init(SPA_MODE_READ); - kernel_init_done = B_TRUE; - if (dump_all) verbose = MAX(verbose, 1); @@ -9174,6 +9176,53 @@ main(int argc, char **argv) if (argc < 2 && dump_opt['R']) usage(); + target = argv[0]; + + /* + * Automate cachefile + */ + if (!spa_config_path_env && !config_path_console && target && + libzfs_core_init() == 0) { + char *pname = strdup(target); + const char *value; + nvlist_t *pnvl; + nvlist_t *vnvl; + + if (strpbrk(pname, "/@") != NULL) + *strpbrk(pname, "/@") = '\0'; + + if (pname && lzc_get_props(pname, &pnvl) == 0) { + if (nvlist_lookup_nvlist(pnvl, "cachefile", + &vnvl) == 0) { + value = fnvlist_lookup_string(vnvl, + ZPROP_VALUE); + } else { + value = "-"; + } + strlcpy(pbuf, value, sizeof (pbuf)); + if (pbuf[0] != '\0') { + if (pbuf[0] == '/') { + if (access(pbuf, F_OK) == 0) + spa_config_path = pbuf; + else + force_import = B_TRUE; + } else if ((strcmp(pbuf, "-") == 0 && + access(ZPOOL_CACHE, F_OK) != 0) || + strcmp(pbuf, "none") == 0) { + force_import = B_TRUE; + } + } + nvlist_free(vnvl); + } + + free(pname); + nvlist_free(pnvl); + libzfs_core_fini(); + } + + kernel_init(SPA_MODE_READ); + kernel_init_done = B_TRUE; + if (dump_opt['E']) { if (argc != 1) usage(); @@ -9210,7 +9259,6 @@ main(int argc, char **argv) fatal("internal error: %s", strerror(ENOMEM)); error = 0; - target = argv[0]; if (strpbrk(target, "/@") != NULL) { size_t targetlen; @@ -9256,9 +9304,17 @@ main(int argc, char **argv) target_pool = target; } - if (dump_opt['e']) { + if (dump_opt['e'] || force_import) { importargs_t args = { 0 }; + /* + * If path is not provided, search in /dev + */ + if (searchdirs == NULL) { + searchdirs = umem_alloc(sizeof (char *), UMEM_NOFAIL); + searchdirs[nsearch++] = (char *)ZFS_DEVDIR; + } + args.paths = nsearch; args.path = searchdirs; args.can_be_active = B_TRUE; diff --git a/include/libzfs_core.h b/include/libzfs_core.h index 867c18b9c22..b2fd97372cd 100644 --- a/include/libzfs_core.h +++ b/include/libzfs_core.h @@ -77,6 +77,7 @@ _LIBZFS_CORE_H int lzc_snaprange_space(const char *, const char *, uint64_t *); _LIBZFS_CORE_H int lzc_hold(nvlist_t *, int, nvlist_t **); _LIBZFS_CORE_H int lzc_release(nvlist_t *, nvlist_t **); _LIBZFS_CORE_H int lzc_get_holds(const char *, nvlist_t **); +_LIBZFS_CORE_H int lzc_get_props(const char *, nvlist_t **); enum lzc_send_flags { LZC_SEND_FLAG_EMBED_DATA = 1 << 0, diff --git a/lib/libzfs_core/libzfs_core.abi b/lib/libzfs_core/libzfs_core.abi index cf9d6bddc9f..c20698580ee 100644 --- a/lib/libzfs_core/libzfs_core.abi +++ b/lib/libzfs_core/libzfs_core.abi @@ -168,6 +168,7 @@ + @@ -2694,6 +2695,11 @@ + + + + + diff --git a/lib/libzfs_core/libzfs_core.c b/lib/libzfs_core/libzfs_core.c index 01d803e21db..070f8c1be67 100644 --- a/lib/libzfs_core/libzfs_core.c +++ b/lib/libzfs_core/libzfs_core.c @@ -596,6 +596,12 @@ lzc_get_holds(const char *snapname, nvlist_t **holdsp) return (lzc_ioctl(ZFS_IOC_GET_HOLDS, snapname, NULL, holdsp)); } +int +lzc_get_props(const char *poolname, nvlist_t **props) +{ + return (lzc_ioctl(ZFS_IOC_POOL_GET_PROPS, poolname, NULL, props)); +} + static unsigned int max_pipe_buffer(int infd) { From b558f0a9d65c3bdb8310504184a82e9802551168 Mon Sep 17 00:00:00 2001 From: Ameer Hamza Date: Fri, 7 Jun 2024 05:01:26 +0500 Subject: [PATCH 13/42] zdb: fix FreeBSD build failure This fixes FreeBSD build failure with clang-18 after 23a489a got merged. Reviewed-by: Alexander Motin Reviewed-by: Rob Norris Reviewed-by: Brian Behlendorf Signed-off-by: Ameer Hamza Closes #16252 --- cmd/zdb/zdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c index 2a3d58d77e0..3a7ef11612d 100644 --- a/cmd/zdb/zdb.c +++ b/cmd/zdb/zdb.c @@ -9185,7 +9185,7 @@ main(int argc, char **argv) libzfs_core_init() == 0) { char *pname = strdup(target); const char *value; - nvlist_t *pnvl; + nvlist_t *pnvl = NULL; nvlist_t *vnvl; if (strpbrk(pname, "/@") != NULL) From 4de260efe3375fe62b9d80452b9203c89ab0c045 Mon Sep 17 00:00:00 2001 From: Derek Schrock Date: Thu, 6 Jun 2024 20:37:26 -0400 Subject: [PATCH 14/42] contrib/bash_completion.d: squelch FreeBSD seq when first < last With seq x -1 z and x is less than z FreeBSD seq will print the error: $ seq 1 -1 2 seq: needs positive increment Hide this error. Alternatively $COMP_CWORD could be checked for < 2. Reviewed-by: Brian Behlendorf Signed-off-by: Derek Schrock Closes #16234 --- contrib/bash_completion.d/zfs.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/bash_completion.d/zfs.in b/contrib/bash_completion.d/zfs.in index c5cfd8e8efb..dbeb10d8994 100644 --- a/contrib/bash_completion.d/zfs.in +++ b/contrib/bash_completion.d/zfs.in @@ -155,7 +155,7 @@ __zfs_list_volumes() __zfs_argument_chosen() { local word property - for word in $(seq $((COMP_CWORD-1)) -1 2) + for word in $(seq $((COMP_CWORD-1)) -1 2 2>/dev/null) do local prev="${COMP_WORDS[$word]}" if [[ ${COMP_WORDS[$word-1]} != -[tos] ]] From 20c8bdd85ef4716d5e59d9f6f61347c0e4566750 Mon Sep 17 00:00:00 2001 From: bnovkov <72801811+bnovkov@users.noreply.github.com> Date: Fri, 7 Jun 2024 03:11:00 +0200 Subject: [PATCH 15/42] FreeBSD: Update use of UMA-related symbols in arc_available_memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recent UMA changes repurposed the use of UMA_MD_SMALL_ALLOC in a way that breaks arc_available_memory on -CURRENT. This change ensures that arc_available_memory uses the new symbol while maintaining compatibility with older FreeBSD releases. Reviewed-by: Brian Behlendorf Signed-off-by: Bojan Novković Closes #16230 --- module/os/freebsd/zfs/arc_os.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/module/os/freebsd/zfs/arc_os.c b/module/os/freebsd/zfs/arc_os.c index 92696c0bf1a..e271d3bf98a 100644 --- a/module/os/freebsd/zfs/arc_os.c +++ b/module/os/freebsd/zfs/arc_os.c @@ -89,17 +89,17 @@ arc_available_memory(void) if (n < lowest) { lowest = n; } -#if defined(__i386) || !defined(UMA_MD_SMALL_ALLOC) +#if !defined(UMA_MD_SMALL_ALLOC) && !defined(UMA_USE_DMAP) /* - * If we're on an i386 platform, it's possible that we'll exhaust the - * kernel heap space before we ever run out of available physical - * memory. Most checks of the size of the heap_area compare against - * tune.t_minarmem, which is the minimum available real memory that we - * can have in the system. However, this is generally fixed at 25 pages - * which is so low that it's useless. In this comparison, we seek to - * calculate the total heap-size, and reclaim if more than 3/4ths of the - * heap is allocated. (Or, in the calculation, if less than 1/4th is - * free) + * If we're on a platform without a direct map, it's possible that we'll + * exhaust the kernel heap space before we ever run out of available + * physical memory. Most checks of the size of the heap_area compare + * against tune.t_minarmem, which is the minimum available real memory + * that we can have in the system. However, this is generally fixed at + * 25 pages which is so low that it's useless. In this comparison, we + * seek to calculate the total heap-size, and reclaim if more than + * 3/4ths of the heap is allocated. (Or, in the calculation, if less + * than 1/4th is free) */ n = uma_avail() - (long)(uma_limit() / 4); if (n < lowest) { From 121a2d335414fe294c948795ee9406bab966588f Mon Sep 17 00:00:00 2001 From: Mateusz Guzik Date: Fri, 14 Jun 2024 02:49:50 +0200 Subject: [PATCH 16/42] FreeBSD: unregister mountroot eventhandler on unload Otherwise if zfs is unloaded and reroot is being used it trips over a stale pointer. Reviewed-by: Brian Behlendorf Reviewed-by: Alexander Motin Sponsored by: Rubicon Communications, LLC ("Netgate") Signed-off-by: Mateusz Guzik Closes #16242 --- module/os/freebsd/zfs/kmod_core.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/module/os/freebsd/zfs/kmod_core.c b/module/os/freebsd/zfs/kmod_core.c index 2bced9ab644..9bd0c9ed81a 100644 --- a/module/os/freebsd/zfs/kmod_core.c +++ b/module/os/freebsd/zfs/kmod_core.c @@ -111,6 +111,7 @@ static int zfs__fini(void); static void zfs_shutdown(void *, int); static eventhandler_tag zfs_shutdown_event_tag; +static eventhandler_tag zfs_mountroot_event_tag; #define ZFS_MIN_KSTACK_PAGES 4 @@ -305,16 +306,25 @@ zfs_modevent(module_t mod, int type, void *unused __unused) switch (type) { case MOD_LOAD: err = zfs__init(); - if (err == 0) + if (err == 0) { zfs_shutdown_event_tag = EVENTHANDLER_REGISTER( shutdown_post_sync, zfs_shutdown, NULL, SHUTDOWN_PRI_FIRST); + zfs_mountroot_event_tag = EVENTHANDLER_REGISTER( + mountroot, spa_boot_init, NULL, + SI_ORDER_ANY); + } return (err); case MOD_UNLOAD: err = zfs__fini(); - if (err == 0 && zfs_shutdown_event_tag != NULL) - EVENTHANDLER_DEREGISTER(shutdown_post_sync, - zfs_shutdown_event_tag); + if (err == 0) { + if (zfs_shutdown_event_tag != NULL) + EVENTHANDLER_DEREGISTER(shutdown_post_sync, + zfs_shutdown_event_tag); + if (zfs_mountroot_event_tag != NULL) + EVENTHANDLER_DEREGISTER(mountroot, + zfs_mountroot_event_tag); + } return (err); case MOD_SHUTDOWN: return (0); @@ -330,9 +340,6 @@ static moduledata_t zfs_mod = { 0 }; -#ifdef _KERNEL -EVENTHANDLER_DEFINE(mountroot, spa_boot_init, NULL, 0); -#endif FEATURE(zfs, "OpenZFS support"); From c98295eed2687cee704ef5f8f3218d3d44a6a1d8 Mon Sep 17 00:00:00 2001 From: Martin Wagner Date: Fri, 14 Jun 2024 03:08:49 +0200 Subject: [PATCH 17/42] disable automatic dependency tracking for dkms builds Previously the dkms build left some unwanted files in `/usr/lib/modules` which could cause package managers to not properly clean up old kernels. Reviewed-by: Tony Hutter Reviewed-by: Brian Behlendorf Signed-off-by: Martin Wagner Closes #16221 Closes #16241 --- scripts/dkms.mkconf | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/dkms.mkconf b/scripts/dkms.mkconf index 0bd38342043..046ce9edcef 100755 --- a/scripts/dkms.mkconf +++ b/scripts/dkms.mkconf @@ -26,6 +26,7 @@ PACKAGE_VERSION="${pkgver}" PACKAGE_CONFIG="${pkgcfg}" NO_WEAK_MODULES="yes" PRE_BUILD="configure + --disable-dependency-tracking --prefix=/usr --with-config=kernel --with-linux=\$( From 49f3ce338587410cabc078646b76152685ae102d Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Fri, 28 Jun 2024 09:52:03 -0700 Subject: [PATCH 18/42] Linux 6.9: Call add_disk() from workqueue to fix zfs_allow_010_pos (#16282) The 6.9 kernel behaves differently in how it releases block devices. In the common case it will async release the device only after the return to userspace. This is different from the 6.8 and older kernels which release the block devices synchronously. To get around this, call add_disk() from a workqueue so that the kernel uses a different codepath to release our zvols in the way we expect. This stops zfs_allow_010_pos from hanging. Fixes: #16089 Signed-off-by: Tony Hutter Reviewed-by: Tino Reichardt Reviewed-by: Rob Norris --- module/os/linux/zfs/zvol_os.c | 102 ++++++++++++++++++++++++++++++++-- 1 file changed, 97 insertions(+), 5 deletions(-) diff --git a/module/os/linux/zfs/zvol_os.c b/module/os/linux/zfs/zvol_os.c index 3e020e53226..acb42c3b4a3 100644 --- a/module/os/linux/zfs/zvol_os.c +++ b/module/os/linux/zfs/zvol_os.c @@ -41,6 +41,7 @@ #include #include +#include #ifdef HAVE_BLK_MQ #include @@ -1338,6 +1339,101 @@ zvol_wait_close(zvol_state_t *zv) { } +struct add_disk_work { + struct delayed_work work; + struct gendisk *disk; + int error; +}; + +static int +__zvol_os_add_disk(struct gendisk *disk) +{ + int error = 0; +#ifdef HAVE_ADD_DISK_RET + error = add_disk(disk); +#else + add_disk(disk); +#endif + return (error); +} + +#if defined(HAVE_BDEV_FILE_OPEN_BY_PATH) +static void +zvol_os_add_disk_work(struct work_struct *work) +{ + struct add_disk_work *add_disk_work; + add_disk_work = container_of(work, struct add_disk_work, work.work); + add_disk_work->error = __zvol_os_add_disk(add_disk_work->disk); +} +#endif + +/* + * SPECIAL CASE: + * + * This function basically calls add_disk() from a workqueue. You may be + * thinking: why not just call add_disk() directly? + * + * When you call add_disk(), the zvol appears to the world. When this happens, + * the kernel calls disk_scan_partitions() on the zvol, which behaves + * differently on the 6.9+ kernels: + * + * - 6.8 and older kernels - + * disk_scan_partitions() + * handle = bdev_open_by_dev( + * zvol_open() + * bdev_release(handle); + * zvol_release() + * + * + * - 6.9+ kernels - + * disk_scan_partitions() + * file = bdev_file_open_by_dev() + * zvol_open() + * fput(file) + * < wait for return to userspace > + * zvol_release() + * + * The difference is that the bdev_release() from the 6.8 kernel is synchronous + * while the fput() from the 6.9 kernel is async. Or more specifically it's + * async that has to wait until we return to userspace (since it adds the fput + * into the caller's work queue with the TWA_RESUME flag set). This is not the + * behavior we want, since we want do things like create+destroy a zvol within + * a single ZFS_IOC_CREATE ioctl, and the "create" part needs to release the + * reference to the zvol while we're in the IOCTL, which can't wait until we + * return to userspace. + * + * We can get around this since fput() has a special codepath for when it's + * running in a kernel thread or interrupt. In those cases, it just puts the + * fput into the system workqueue, which we can force to run with + * __flush_workqueue(). That is why we call add_disk() from a workqueue - so it + * run from a kernel thread and "tricks" the fput() codepaths. + * + * Note that __flush_workqueue() is slowly getting deprecated. This may be ok + * though, since our IOCTL will spin on EBUSY waiting for the zvol release (via + * fput) to happen, which it eventually, naturally, will from the system_wq + * without us explicitly calling __flush_workqueue(). + */ +static int +zvol_os_add_disk(struct gendisk *disk) +{ +#if defined(HAVE_BDEV_FILE_OPEN_BY_PATH) /* 6.9+ kernel */ + struct add_disk_work add_disk_work; + + INIT_DELAYED_WORK(&add_disk_work.work, zvol_os_add_disk_work); + add_disk_work.disk = disk; + add_disk_work.error = 0; + + /* Use *_delayed_work functions since they're not GPL'd */ + schedule_delayed_work(&add_disk_work.work, 0); + flush_delayed_work(&add_disk_work.work); + + __flush_workqueue(system_wq); + return (add_disk_work.error); +#else /* <= 6.8 kernel */ + return (__zvol_os_add_disk(disk)); +#endif +} + /* * Create a block device minor node and setup the linkage between it * and the specified volume. Once this function returns the block @@ -1549,11 +1645,7 @@ out_doi: rw_enter(&zvol_state_lock, RW_WRITER); zvol_insert(zv); rw_exit(&zvol_state_lock); -#ifdef HAVE_ADD_DISK_RET - error = add_disk(zv->zv_zso->zvo_disk); -#else - add_disk(zv->zv_zso->zvo_disk); -#endif + error = zvol_os_add_disk(zv->zv_zso->zvo_disk); } else { ida_simple_remove(&zvol_ida, idx); } From 5f220c62e1c24d9fdb2e9beb7d2796eaf4ef6a74 Mon Sep 17 00:00:00 2001 From: Allan Jude Date: Fri, 28 Jun 2024 13:38:22 -0400 Subject: [PATCH 19/42] Fix a mis-merge in the zdb man page (#16304) Sponsored-by: Klara, Inc. Sponsored-By: Wasabi Technology, Inc. Signed-off-by: Allan Jude Reviewed-by: Paul Dagnelie Reviewed-by: Tony Hutter --- man/man8/zdb.8 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/man/man8/zdb.8 b/man/man8/zdb.8 index d7f66d917ac..08f5a3f7004 100644 --- a/man/man8/zdb.8 +++ b/man/man8/zdb.8 @@ -314,7 +314,6 @@ fragmentation, and free space histogram, as well as overall pool fragmentation and histogram. .It Fl MM "Special" vdevs are added to -M's normal output. -.It Fl O , -object-lookups Ns = Ns Ar dataset path Also display information about the maximum contiguous free space and the percentage of free space in each space map. .It Fl MMM @@ -327,7 +326,7 @@ but force zdb to interpret the in .Op Ar poolname Ns Op / Ns Ar dataset Ns | Ns Ar objset-ID as a numeric objset ID. -.It Fl O Ar dataset path +.It Fl O , -object-lookups Ns = Ns Ar dataset path Look up the specified .Ar path inside of the From fd51786f86e4e39ce3aa0a4ab8c44935496ff7c2 Mon Sep 17 00:00:00 2001 From: Mateusz Piotrowski <0mp@FreeBSD.org> Date: Fri, 28 Jun 2024 20:21:08 +0200 Subject: [PATCH 20/42] zfs.4: Document the actual default for zfs_txg_history (#16305) Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Signed-off-by: Mateusz Piotrowski <0mp@FreeBSD.org> Reviewed-by: Tony Hutter --- man/man4/zfs.4 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/man/man4/zfs.4 b/man/man4/zfs.4 index f1d14b4d01a..3f7485fa78c 100644 --- a/man/man4/zfs.4 +++ b/man/man4/zfs.4 @@ -16,7 +16,7 @@ .\" own identifying information: .\" Portions Copyright [yyyy] [name of copyright owner] .\" -.Dd February 14, 2024 +.Dd June 27, 2024 .Dt ZFS 4 .Os . @@ -2113,7 +2113,7 @@ The default of .Sy 32 was determined to be a reasonable compromise. . -.It Sy zfs_txg_history Ns = Ns Sy 0 Pq uint +.It Sy zfs_txg_history Ns = Ns Sy 100 Pq uint Historical statistics for this many latest TXGs will be available in .Pa /proc/spl/kstat/zfs/ Ns Ao Ar pool Ac Ns Pa /TXGs . . From f72e081fbf3897cda90cf10463a1e4f25d52e257 Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Mon, 8 Jul 2024 19:59:08 -0500 Subject: [PATCH 21/42] FreeBSD: Use a statement expression to implement SET_ERROR() (#16284) This way we can avoid making assumptions about the SDT probe implementation. No functional change intended. Signed-off-by: Mark Johnston Reviewed-by: Alexander Motin Reviewed-by: Allan Jude Reviewed-by: Rob Norris Reviewed-by: Tino Reichardt Reviewed-by: Tony Hutter --- include/os/freebsd/spl/sys/sdt.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/include/os/freebsd/spl/sys/sdt.h b/include/os/freebsd/spl/sys/sdt.h index 2daa6de1af0..e2c4830cb96 100644 --- a/include/os/freebsd/spl/sys/sdt.h +++ b/include/os/freebsd/spl/sys/sdt.h @@ -31,13 +31,14 @@ #include_next #ifdef KDTRACE_HOOKS -/* CSTYLED */ +/* BEGIN CSTYLED */ SDT_PROBE_DECLARE(sdt, , , set__error); -#define SET_ERROR(err) \ - ((sdt_sdt___set__error->id ? \ - (*sdt_probe_func)(sdt_sdt___set__error->id, \ - (uintptr_t)err, 0, 0, 0, 0) : 0), err) +#define SET_ERROR(err) ({ \ + SDT_PROBE1(sdt, , , set__error, (uintptr_t)err); \ + err; \ +}) +/* END CSTYLED */ #else #define SET_ERROR(err) (err) #endif From 9ffe4413610ea81731462d45f81a30e709fef011 Mon Sep 17 00:00:00 2001 From: Tino Reichardt Date: Tue, 9 Jul 2024 18:36:17 +0200 Subject: [PATCH 22/42] Fix zdb "Memory fault" found on FreeBSD ZTS (#16332) Reason: nvlist_free() tries to free sth. which isn't allocted Solution: init this variable with NULL Closes #16311 Signed-off-by: Tino Reichardt Reviewed-by: Alexander Motin Reviewed-by: Ameer Hamza Reviewed-by: Tony Hutter --- cmd/zdb/zdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c index 3a7ef11612d..0d2f0b1da5f 100644 --- a/cmd/zdb/zdb.c +++ b/cmd/zdb/zdb.c @@ -9186,7 +9186,7 @@ main(int argc, char **argv) char *pname = strdup(target); const char *value; nvlist_t *pnvl = NULL; - nvlist_t *vnvl; + nvlist_t *vnvl = NULL; if (strpbrk(pname, "/@") != NULL) *strpbrk(pname, "/@") = '\0'; From 326040b285251598e84c7269ba26898832744b29 Mon Sep 17 00:00:00 2001 From: Peter Doherty Date: Tue, 9 Jul 2024 12:59:55 -0400 Subject: [PATCH 23/42] Fix the name of the zfs_prefetch_disable parameter (#16319) The ZFS module parameter name is zfs_prefetch_disable, not zfs_disable_prefetch. Signed-off-by: Peter Doherty Reviewed-by: Alexander Motin Reviewed-by: Tino Reichardt Reviewed-by: Tony Hutter --- man/man7/zfsprops.7 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/man7/zfsprops.7 b/man/man7/zfsprops.7 index 9ff0236f4d7..429369bd2e9 100644 --- a/man/man7/zfsprops.7 +++ b/man/man7/zfsprops.7 @@ -1639,7 +1639,7 @@ then only metadata are prefetched. The default value is .Sy all . .Pp -Please note that the module parameter zfs_disable_prefetch=1 can +Please note that the module parameter zfs_prefetch_disable=1 can be used to totally disable speculative prefetch, bypassing anything this property does. .It Sy setuid Ns = Ns Sy on Ns | Ns Sy off From 4367312760612cb755acfc218ffeda8eb35ce509 Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Wed, 10 Jul 2024 16:27:44 -0500 Subject: [PATCH 24/42] zvol: Fix suspend lock leaks (#16270) In several functions, we use a flag variable to track whether zv_suspend_lock is held. This flag was not getting reset in a particular case where we need to retry the underlying operation, resulting in a lock leak. Make sure to update the flag where necessary. Signed-off-by: Mark Johnston Reviewed-by: Allan Jude Reviewed-by: Alexander Motin Reviewed-by: Tino Reichardt Reviewed-by: Tony Hutter --- module/os/freebsd/zfs/zvol_os.c | 2 ++ module/os/linux/zfs/zvol_os.c | 1 + 2 files changed, 3 insertions(+) diff --git a/module/os/freebsd/zfs/zvol_os.c b/module/os/freebsd/zfs/zvol_os.c index 712ff1b837d..38e9debbe87 100644 --- a/module/os/freebsd/zfs/zvol_os.c +++ b/module/os/freebsd/zfs/zvol_os.c @@ -292,6 +292,7 @@ retry: if (!mutex_tryenter(&spa_namespace_lock)) { mutex_exit(&zv->zv_state_lock); rw_exit(&zv->zv_suspend_lock); + drop_suspend = B_FALSE; kern_yield(PRI_USER); goto retry; } else { @@ -983,6 +984,7 @@ retry: if (!mutex_tryenter(&spa_namespace_lock)) { mutex_exit(&zv->zv_state_lock); rw_exit(&zv->zv_suspend_lock); + drop_suspend = B_FALSE; kern_yield(PRI_USER); goto retry; } else { diff --git a/module/os/linux/zfs/zvol_os.c b/module/os/linux/zfs/zvol_os.c index acb42c3b4a3..1d5d54b80ea 100644 --- a/module/os/linux/zfs/zvol_os.c +++ b/module/os/linux/zfs/zvol_os.c @@ -791,6 +791,7 @@ retry: if (!mutex_tryenter(&spa_namespace_lock)) { mutex_exit(&zv->zv_state_lock); rw_exit(&zv->zv_suspend_lock); + drop_suspend = B_FALSE; #ifdef HAVE_BLKDEV_GET_ERESTARTSYS schedule(); From 156a64161b4f9da35f2e0484106173344cf78317 Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Thu, 11 Jul 2024 16:41:26 -0700 Subject: [PATCH 25/42] Linux 6.9: Fix UBSAN errors in zap_micro.c You can use the UBSAN_SANITIZE_* Kbuild options to exclude certain kernel objects from the UBSAN checks. We previously excluded zap_micro.o with: UBSAN_SANITIZE_zap_micro.o := n For some reason that didn't work for the 6.9 kernel, which wants us to use: UBSAN_SANITIZE_zfs/zap_micro.o := n Reviewed-by: Brian Behlendorf Signed-off-by: Tony Hutter Closes #16278 Closes #16330 --- module/Kbuild.in | 1 + 1 file changed, 1 insertion(+) diff --git a/module/Kbuild.in b/module/Kbuild.in index 9e44364b758..4707452711f 100644 --- a/module/Kbuild.in +++ b/module/Kbuild.in @@ -490,6 +490,7 @@ zfs-$(CONFIG_PPC64) += $(addprefix zfs/,$(ZFS_OBJS_PPC_PPC64)) UBSAN_SANITIZE_zap_leaf.o := n UBSAN_SANITIZE_zap_micro.o := n UBSAN_SANITIZE_sa.o := n +UBSAN_SANITIZE_zfs/zap_micro.o := n # Suppress incorrect warnings from versions of objtool which are not # aware of x86 EVEX prefix instructions used for AVX512. From a10faf5ce6fe0cb07012d9f2d118fa326a09d267 Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Thu, 11 Jul 2024 18:52:51 -0500 Subject: [PATCH 26/42] FreeBSD: Use the new freeuio() helper to free dynamically allocated UIOs (#16300) This freeuio() interface was introduced to FreeBSD recently. For now it simply calls free(), so this change has no effect. However, this may not always be true, and in CheriBSD this change is required. Signed-off-by: Mark Johnston Reviewed-by: Alexander Motin Reviewed-by: Brooks Davis Reviewed-by: Tony Hutter --- module/os/freebsd/spl/spl_uio.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/module/os/freebsd/spl/spl_uio.c b/module/os/freebsd/spl/spl_uio.c index ffccb6f2594..17886cbeb50 100644 --- a/module/os/freebsd/spl/spl_uio.c +++ b/module/os/freebsd/spl/spl_uio.c @@ -45,6 +45,16 @@ #include #include +static void +zfs_freeuio(struct uio *uio) +{ +#if __FreeBSD_version > 1500013 + freeuio(uio); +#else + free(uio, M_IOV); +#endif +} + int zfs_uiomove(void *cp, size_t n, zfs_uio_rw_t dir, zfs_uio_t *uio) { @@ -77,7 +87,7 @@ zfs_uiocopy(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio, size_t *cbytes) error = vn_io_fault_uiomove(p, n, uio_clone); *cbytes = zfs_uio_resid(uio) - uio_clone->uio_resid; if (uio_clone != &small_uio_clone) - free(uio_clone, M_IOV); + zfs_freeuio(uio_clone); return (error); } From cbd95a950aa52d94aca05a621c8e7778b49a957e Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Sat, 13 Jul 2024 03:58:03 +1000 Subject: [PATCH 27/42] ZTS: handle FreeBSD version numbers correctly (#16340) FreeBSD patchlevel versions are optional and, if present, in a different location in the version string. Sponsored-by: https://despairlabs.com/sponsor/ Signed-off-by: Rob Norris Reviewed-by: Alexander Motin Reviewed-by: Tino Reichardt Reviewed-by: Tony Hutter --- tests/zfs-tests/include/libtest.shlib | 30 ++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib index dfab48d2cda..a2f42999a31 100644 --- a/tests/zfs-tests/include/libtest.shlib +++ b/tests/zfs-tests/include/libtest.shlib @@ -62,11 +62,39 @@ function compare_version_gte } # Helper function used by linux_version() and freebsd_version() +# $1, if provided, should be a MAJOR, MAJOR.MINOR or MAJOR.MINOR.PATCH +# version number function kernel_version { typeset ver="$1" - [ -z "$ver" ] && ver=$(uname -r | grep -Eo "^[0-9]+\.[0-9]+\.[0-9]+") + [ -z "$ver" ] && case "$UNAME" in + Linux) + # Linux version numbers are X.Y.Z followed by optional + # vendor/distro specific stuff + # RHEL7: 3.10.0-1160.108.1.el7.x86_64 + # Fedora 37: 6.5.12-100.fc37.x86_64 + # Debian 12.6: 6.1.0-22-amd64 + ver=$(uname -r | grep -Eo "^[0-9]+\.[0-9]+\.[0-9]+") + ;; + FreeBSD) + # FreeBSD version numbers are X.Y-BRANCH-pZ. Depending on + # branch, -pZ may not be present, but this is typically only + # on pre-release or true .0 releases, so can be assumed 0 + # if not present. + # eg: + # 13.2-RELEASE-p4 + # 14.1-RELEASE + # 15.0-CURRENT + ver=$(uname -r | \ + grep -Eo "[0-9]+\.[0-9]+(-[A-Z0-9]+-p[0-9]+)?" | \ + sed -E "s/-[^-]+-p/./") + ;; + *) + # Unknown system + log_fail "Don't know how to get kernel version for '$UNAME'" + ;; + esac typeset version major minor _ IFS='.' read -r version major minor _ <<<"$ver" From 398e675f5839bdddf44682ea5390e49fa963ea1b Mon Sep 17 00:00:00 2001 From: a1ea321 Date: Sat, 13 Jul 2024 01:27:12 +0200 Subject: [PATCH 28/42] one-word manpage correction: snapshot->rollback (#16294) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit fixes what is probably a copy-paste mistake. The `dracut.zfs` manpage claims that the `bootfs.rollback` option executes `zfs snapshot -Rf`. `zfs snapshot` does not have a `-R` option. `zfs rollback` does. Signed-off-by: Alphan Yılmaz Reviewed-by: Rob Norris Reviewed-by: George Melikov Reviewed-by: Tony Hutter --- man/man7/dracut.zfs.7 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/man7/dracut.zfs.7 b/man/man7/dracut.zfs.7 index c1475c695e8..b67e1cecb24 100644 --- a/man/man7/dracut.zfs.7 +++ b/man/man7/dracut.zfs.7 @@ -154,7 +154,7 @@ defaults to the current kernel release. . .It Sy bootfs.rollback Ns Op Sy = Ns Ar snapshot-name Execute -.Nm zfs Cm snapshot Fl Rf Ar boot-dataset Ns Sy @ Ns Ar snapshot-name +.Nm zfs Cm rollback Fl Rf Ar boot-dataset Ns Sy @ Ns Ar snapshot-name before pivoting to the real root. .Ar snapshot-name defaults to the current kernel release. From f7d8b1333699957de59d72fc57c9a771951f6582 Mon Sep 17 00:00:00 2001 From: Daniel Berlin Date: Fri, 12 Jul 2024 20:44:10 -0400 Subject: [PATCH 29/42] Fix missing semicolon in trace_dbuf.h (#16281) On fedora 40, on the 6.9.4 kernel (in updates-testing), assign_str expands to a "do { } while(0)" loop. Without this semicolon, the while(0) is unterminated, causing a cascade of useless errors. With this semicolon, it compiles fine. It also compiles fine on 6.8.11 (the previous kernel). I have not tested earlier kernels than that, but at worst it should add a pointless semicolon. All other instances in the source tree are already terminated with semicolons. Signed-off-by: Daniel Berlin Reviewed-by: Alexander Motin Reviewed-by: Tony Hutter --- include/os/linux/zfs/sys/trace_dbuf.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/os/linux/zfs/sys/trace_dbuf.h b/include/os/linux/zfs/sys/trace_dbuf.h index 0f6a98b47d6..0e9cbdd725b 100644 --- a/include/os/linux/zfs/sys/trace_dbuf.h +++ b/include/os/linux/zfs/sys/trace_dbuf.h @@ -80,7 +80,7 @@ snprintf(__get_str(msg), TRACE_DBUF_MSG_MAX, \ DBUF_TP_PRINTK_FMT, DBUF_TP_PRINTK_ARGS); \ } else { \ - __assign_str(os_spa, "NULL") \ + __assign_str(os_spa, "NULL"); \ __entry->ds_object = 0; \ __entry->db_object = 0; \ __entry->db_level = 0; \ From c87cb22ba9fa5f4ed4c9afa0eb66092c2f08841e Mon Sep 17 00:00:00 2001 From: George Amanakis Date: Mon, 15 Jul 2024 18:05:42 +0200 Subject: [PATCH 30/42] head_errlog: fix use-after-free In the commit of the head_errlog feature we introduced a bug in dsl_dataset_promote_sync(): we may dereference origin_head and hds, both dereferencing ddpa after calling promote_sync() on ddpa. Reviewed-by: Brian Behlendorf Reviewed-by: Chunwei Chen Reviewed-by: Rob Norris Reviewed-by: Tony Hutter Signed-off-by: George Amanakis Closes #16272 Closes #16273 --- module/zfs/dsl_dataset.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/module/zfs/dsl_dataset.c b/module/zfs/dsl_dataset.c index b4de0e7ff07..45d8a290d67 100644 --- a/module/zfs/dsl_dataset.c +++ b/module/zfs/dsl_dataset.c @@ -3712,16 +3712,19 @@ dsl_dataset_promote_sync(void *arg, dmu_tx_t *tx) spa_history_log_internal_ds(hds, "promote", tx, " "); dsl_dir_rele(odd, FTAG); - promote_rele(ddpa, FTAG); /* - * Transfer common error blocks from old head to new head. + * Transfer common error blocks from old head to new head, before + * calling promote_rele() on ddpa since we need to dereference + * origin_head and hds. */ if (spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_HEAD_ERRLOG)) { uint64_t old_head = origin_head->ds_object; uint64_t new_head = hds->ds_object; spa_swap_errlog(dp->dp_spa, new_head, old_head, tx); } + + promote_rele(ddpa, FTAG); } /* From a7fc4c85e3ad282a62d95c84c5eaf1b59c1fb23a Mon Sep 17 00:00:00 2001 From: Mateusz Guzik Date: Mon, 15 Jul 2024 23:51:37 +0200 Subject: [PATCH 31/42] zstd: don't call zstd_mempool_reap if there are no buffers (#16302) zfs_zstd_cache_reap_now is issued every second. zstd_mempool_reap checks for both pool existence and buffer count, but that's still 2 func calls which are trivially avoidable. With clang it even avoids pushing the stack pointer (but still suffers the mispredict due to a forward jump, not modified in case someone is using zstd): <+0>: cmpq $0x0,0x0(%rip) # <+8>: je 0x217de4 <+10>: push %rbp <+11>: mov %rsp,%rbp <+14>: mov 0x0(%rip),%rdi # <+21>: call 0x217df0 <+26>: mov 0x0(%rip),%rdi # <+33>: pop %rbp <+34>: jmp 0x217df0 <+36>: ret Preferably the call would not be made to begin with if zstd is not used, but this retains all the logic confined to zstd code. Sponsored by: Rubicon Communications, LLC ("Netgate") Signed-off-by: Mateusz Guzik Reviewed-by: Allan Jude Reviewed-by: Tony Hutter --- module/zstd/zfs_zstd.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/module/zstd/zfs_zstd.c b/module/zstd/zfs_zstd.c index ed0271a8d68..05120d27b8d 100644 --- a/module/zstd/zfs_zstd.c +++ b/module/zstd/zfs_zstd.c @@ -844,6 +844,13 @@ zstd_mempool_deinit(void) void zfs_zstd_cache_reap_now(void) { + + /* + * Short-circuit if there are no buffers to begin with. + */ + if (ZSTDSTAT(zstd_stat_buffers) == 0) + return; + /* * calling alloc with zero size seeks * and releases old unused objects From 4ee66cdf4ed487a1fc087ba7bf0fca4bf74667b5 Mon Sep 17 00:00:00 2001 From: Zhao Yongming Date: Tue, 16 Jul 2024 11:58:00 +1200 Subject: [PATCH 32/42] Add building support for Artix Linux (#16265) Artix Linux is systemd free distribution based on Arch Linux, with openrc dinit runit s6 as init alternatives. This patch will make init scripts installation work the way Gentoo Linux with openrc. The scripts tweaking for other init will be left to packager. Signed-off-by: Yongming Zhao Reviewed-by: Rob Norris Reviewed-by: Tony Hutter --- config/zfs-build.m4 | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/config/zfs-build.m4 b/config/zfs-build.m4 index 368684e1c51..255813f46d1 100644 --- a/config/zfs-build.m4 +++ b/config/zfs-build.m4 @@ -516,6 +516,8 @@ AC_DEFUN([ZFS_AC_DEFAULT_PACKAGE], [ VENDOR=alpine ; elif test -f /etc/arch-release ; then VENDOR=arch ; + elif test -f /etc/artix-release ; then + VENDOR=artix ; elif test -f /etc/fedora-release ; then VENDOR=fedora ; elif test -f /bin/freebsd-version ; then @@ -551,7 +553,7 @@ AC_DEFUN([ZFS_AC_DEFAULT_PACKAGE], [ AC_MSG_CHECKING([default package type]) case "$VENDOR" in - alpine|arch|gentoo|lunar|slackware) + alpine|arch|artix|gentoo|lunar|slackware) DEFAULT_PACKAGE=tgz ;; debian|ubuntu) DEFAULT_PACKAGE=deb ;; @@ -576,6 +578,8 @@ AC_DEFUN([ZFS_AC_DEFAULT_PACKAGE], [ case "$VENDOR" in alpine|gentoo) DEFAULT_INIT_SHELL=/sbin/openrc-run IS_SYSV_RC=false ;; + artix) DEFAULT_INIT_SHELL=/usr/bin/openrc-run + IS_SYSV_RC=false ;; *) DEFAULT_INIT_SHELL=/bin/sh IS_SYSV_RC=true ;; esac @@ -594,7 +598,7 @@ AC_DEFUN([ZFS_AC_DEFAULT_PACKAGE], [ AC_MSG_CHECKING([default init config directory]) case "$VENDOR" in - alpine|gentoo) + alpine|artix|gentoo) initconfdir=/etc/conf.d ;; fedora|openeuler|redhat|sles|toss) @@ -623,7 +627,7 @@ AC_DEFUN([ZFS_AC_DEFAULT_PACKAGE], [ AC_MSG_CHECKING([default bash completion directory]) case "$VENDOR" in - alpine|debian|gentoo|ubuntu) + alpine|artix|debian|gentoo|ubuntu) bashcompletiondir=/usr/share/bash-completion/completions ;; freebsd) From b409892ae5028965a6fe98dde1346594807e6e45 Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Mon, 27 May 2024 21:32:07 -0400 Subject: [PATCH 33/42] Linux 6.10: rework queue limits setup Linux has started moving to a model where instead of applying block queue limits through individual modification functions, a complete limits structure is built up and applied atomically, either when the block device or open, or some time afterwards. As of 6.10 this transition appears only partly completed. This commit matches that model within OpenZFS in a way that should work for past and future kernels. We set up a queue limits structure with any limits that have had their modification functions removed. For newer kernels that can have limits applied at block device open (HAVE_BLK_ALLOC_DISK_2ARG), we have a conversion function to turn the OpenZFS queue limits structure into Linux's queue_limits structure, which can then be passed in. For older kernels, we provide an application function that just calls the old functions for each limit in the structure. Signed-off-by: Rob Norris Sponsored-by: https://despairlabs.com/sponsor/ Reviewed-by: Tony Hutter Reviewed-by: Brian Behlendorf --- config/kernel-blk-queue.m4 | 4 +- module/os/linux/zfs/zvol_os.c | 186 +++++++++++++++++++++------------- 2 files changed, 118 insertions(+), 72 deletions(-) diff --git a/config/kernel-blk-queue.m4 b/config/kernel-blk-queue.m4 index 15dbe1c7dff..2f0b386e663 100644 --- a/config/kernel-blk-queue.m4 +++ b/config/kernel-blk-queue.m4 @@ -332,7 +332,7 @@ AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_MAX_HW_SECTORS], [ ZFS_LINUX_TEST_RESULT([blk_queue_max_hw_sectors], [ AC_MSG_RESULT(yes) ],[ - ZFS_LINUX_TEST_ERROR([blk_queue_max_hw_sectors]) + AC_MSG_RESULT(no) ]) ]) @@ -355,7 +355,7 @@ AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_MAX_SEGMENTS], [ ZFS_LINUX_TEST_RESULT([blk_queue_max_segments], [ AC_MSG_RESULT(yes) ], [ - ZFS_LINUX_TEST_ERROR([blk_queue_max_segments]) + AC_MSG_RESULT(no) ]) ]) diff --git a/module/os/linux/zfs/zvol_os.c b/module/os/linux/zfs/zvol_os.c index 1d5d54b80ea..c01caa6da8b 100644 --- a/module/os/linux/zfs/zvol_os.c +++ b/module/os/linux/zfs/zvol_os.c @@ -1076,8 +1076,106 @@ static const struct block_device_operations zvol_ops = { #endif }; +typedef struct zvol_queue_limits { + unsigned int zql_max_hw_sectors; + unsigned short zql_max_segments; + unsigned int zql_max_segment_size; + unsigned int zql_io_opt; +} zvol_queue_limits_t; + +static void +zvol_queue_limits_init(zvol_queue_limits_t *limits, zvol_state_t *zv, + boolean_t use_blk_mq) +{ + limits->zql_max_hw_sectors = (DMU_MAX_ACCESS / 4) >> 9; + + if (use_blk_mq) { + /* + * IO requests can be really big (1MB). When an IO request + * comes in, it is passed off to zvol_read() or zvol_write() + * in a new thread, where it is chunked up into 'volblocksize' + * sized pieces and processed. So for example, if the request + * is a 1MB write and your volblocksize is 128k, one zvol_write + * thread will take that request and sequentially do ten 128k + * IOs. This is due to the fact that the thread needs to lock + * each volblocksize sized block. So you might be wondering: + * "instead of passing the whole 1MB request to one thread, + * why not pass ten individual 128k chunks to ten threads and + * process the whole write in parallel?" The short answer is + * that there's a sweet spot number of chunks that balances + * the greater parallelism with the added overhead of more + * threads. The sweet spot can be different depending on if you + * have a read or write heavy workload. Writes typically want + * high chunk counts while reads typically want lower ones. On + * a test pool with 6 NVMe drives in a 3x 2-disk mirror + * configuration, with volblocksize=8k, the sweet spot for good + * sequential reads and writes was at 8 chunks. + */ + + /* + * Below we tell the kernel how big we want our requests + * to be. You would think that blk_queue_io_opt() would be + * used to do this since it is used to "set optimal request + * size for the queue", but that doesn't seem to do + * anything - the kernel still gives you huge requests + * with tons of little PAGE_SIZE segments contained within it. + * + * Knowing that the kernel will just give you PAGE_SIZE segments + * no matter what, you can say "ok, I want PAGE_SIZE byte + * segments, and I want 'N' of them per request", where N is + * the correct number of segments for the volblocksize and + * number of chunks you want. + */ +#ifdef HAVE_BLK_MQ + if (zvol_blk_mq_blocks_per_thread != 0) { + unsigned int chunks; + chunks = MIN(zvol_blk_mq_blocks_per_thread, UINT16_MAX); + + limits->zql_max_segment_size = PAGE_SIZE; + limits->zql_max_segments = + (zv->zv_volblocksize * chunks) / PAGE_SIZE; + } else { + /* + * Special case: zvol_blk_mq_blocks_per_thread = 0 + * Max everything out. + */ + limits->zql_max_segments = UINT16_MAX; + limits->zql_max_segment_size = UINT_MAX; + } + } else { +#endif + limits->zql_max_segments = UINT16_MAX; + limits->zql_max_segment_size = UINT_MAX; + } + + limits->zql_io_opt = zv->zv_volblocksize; +} + +#ifdef HAVE_BLK_ALLOC_DISK_2ARG +static void +zvol_queue_limits_convert(zvol_queue_limits_t *limits, + struct queue_limits *qlimits) +{ + memset(qlimits, 0, sizeof (struct queue_limits)); + qlimits->max_hw_sectors = limits->zql_max_hw_sectors; + qlimits->max_segments = limits->zql_max_segments; + qlimits->max_segment_size = limits->zql_max_segment_size; + qlimits->io_opt = limits->zql_io_opt; +} +#else +static void +zvol_queue_limits_apply(zvol_queue_limits_t *limits, + struct request_queue *queue) +{ + blk_queue_max_hw_sectors(queue, limits->zql_max_hw_sectors); + blk_queue_max_segments(queue, limits->zql_max_segments); + blk_queue_max_segment_size(queue, limits->zql_max_segment_size); + blk_queue_io_opt(queue, limits->zql_io_opt); +} +#endif + static int -zvol_alloc_non_blk_mq(struct zvol_state_os *zso) +zvol_alloc_non_blk_mq(struct zvol_state_os *zso, zvol_queue_limits_t *limits) { #if defined(HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS) #if defined(HAVE_BLK_ALLOC_DISK) @@ -1087,8 +1185,11 @@ zvol_alloc_non_blk_mq(struct zvol_state_os *zso) zso->zvo_disk->minors = ZVOL_MINORS; zso->zvo_queue = zso->zvo_disk->queue; + zvol_queue_limits_apply(limits, zso->zvo_queue); #elif defined(HAVE_BLK_ALLOC_DISK_2ARG) - struct gendisk *disk = blk_alloc_disk(NULL, NUMA_NO_NODE); + struct queue_limits qlimits; + zvol_queue_limits_convert(limits, &qlimits); + struct gendisk *disk = blk_alloc_disk(&qlimits, NUMA_NO_NODE); if (IS_ERR(disk)) { zso->zvo_disk = NULL; return (1); @@ -1109,6 +1210,7 @@ zvol_alloc_non_blk_mq(struct zvol_state_os *zso) } zso->zvo_disk->queue = zso->zvo_queue; + zvol_queue_limits_apply(limits, zso->zvo_queue); #endif /* HAVE_BLK_ALLOC_DISK */ #else zso->zvo_queue = blk_generic_alloc_queue(zvol_request, NUMA_NO_NODE); @@ -1122,13 +1224,14 @@ zvol_alloc_non_blk_mq(struct zvol_state_os *zso) } zso->zvo_disk->queue = zso->zvo_queue; + zvol_queue_limits_apply(limits, zso->zvo_queue); #endif /* HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS */ return (0); } static int -zvol_alloc_blk_mq(zvol_state_t *zv) +zvol_alloc_blk_mq(zvol_state_t *zv, zvol_queue_limits_t *limits) { #ifdef HAVE_BLK_MQ struct zvol_state_os *zso = zv->zv_zso; @@ -1144,9 +1247,12 @@ zvol_alloc_blk_mq(zvol_state_t *zv) return (1); } zso->zvo_queue = zso->zvo_disk->queue; + zvol_queue_limits_apply(limits, zso->zvo_queue); zso->zvo_disk->minors = ZVOL_MINORS; #elif defined(HAVE_BLK_ALLOC_DISK_2ARG) - struct gendisk *disk = blk_mq_alloc_disk(&zso->tag_set, NULL, zv); + struct queue_limits qlimits; + zvol_queue_limits_convert(limits, &qlimits); + struct gendisk *disk = blk_mq_alloc_disk(&zso->tag_set, &qlimits, zv); if (IS_ERR(disk)) { zso->zvo_disk = NULL; blk_mq_free_tag_set(&zso->tag_set); @@ -1172,6 +1278,7 @@ zvol_alloc_blk_mq(zvol_state_t *zv) /* Our queue is now created, assign it to our disk */ zso->zvo_disk->queue = zso->zvo_queue; + zvol_queue_limits_apply(limits, zso->zvo_queue); #endif #endif @@ -1211,6 +1318,9 @@ zvol_alloc(dev_t dev, const char *name) zv->zv_zso->use_blk_mq = zvol_use_blk_mq; #endif + zvol_queue_limits_t limits; + zvol_queue_limits_init(&limits, zv, zv->zv_zso->use_blk_mq); + /* * The block layer has 3 interfaces for getting BIOs: * @@ -1227,10 +1337,10 @@ zvol_alloc(dev_t dev, const char *name) * disk and the queue separately. (5.13 kernel or older) */ if (zv->zv_zso->use_blk_mq) { - ret = zvol_alloc_blk_mq(zv); + ret = zvol_alloc_blk_mq(zv, &limits); zso->zvo_disk->fops = &zvol_ops_blk_mq; } else { - ret = zvol_alloc_non_blk_mq(zso); + ret = zvol_alloc_non_blk_mq(zso, &limits); zso->zvo_disk->fops = &zvol_ops; } if (ret != 0) @@ -1514,74 +1624,10 @@ zvol_os_create_minor(const char *name) set_capacity(zv->zv_zso->zvo_disk, zv->zv_volsize >> 9); - blk_queue_max_hw_sectors(zv->zv_zso->zvo_queue, - (DMU_MAX_ACCESS / 4) >> 9); - if (zv->zv_zso->use_blk_mq) { - /* - * IO requests can be really big (1MB). When an IO request - * comes in, it is passed off to zvol_read() or zvol_write() - * in a new thread, where it is chunked up into 'volblocksize' - * sized pieces and processed. So for example, if the request - * is a 1MB write and your volblocksize is 128k, one zvol_write - * thread will take that request and sequentially do ten 128k - * IOs. This is due to the fact that the thread needs to lock - * each volblocksize sized block. So you might be wondering: - * "instead of passing the whole 1MB request to one thread, - * why not pass ten individual 128k chunks to ten threads and - * process the whole write in parallel?" The short answer is - * that there's a sweet spot number of chunks that balances - * the greater parallelism with the added overhead of more - * threads. The sweet spot can be different depending on if you - * have a read or write heavy workload. Writes typically want - * high chunk counts while reads typically want lower ones. On - * a test pool with 6 NVMe drives in a 3x 2-disk mirror - * configuration, with volblocksize=8k, the sweet spot for good - * sequential reads and writes was at 8 chunks. - */ - - /* - * Below we tell the kernel how big we want our requests - * to be. You would think that blk_queue_io_opt() would be - * used to do this since it is used to "set optimal request - * size for the queue", but that doesn't seem to do - * anything - the kernel still gives you huge requests - * with tons of little PAGE_SIZE segments contained within it. - * - * Knowing that the kernel will just give you PAGE_SIZE segments - * no matter what, you can say "ok, I want PAGE_SIZE byte - * segments, and I want 'N' of them per request", where N is - * the correct number of segments for the volblocksize and - * number of chunks you want. - */ -#ifdef HAVE_BLK_MQ - if (zvol_blk_mq_blocks_per_thread != 0) { - unsigned int chunks; - chunks = MIN(zvol_blk_mq_blocks_per_thread, UINT16_MAX); - - blk_queue_max_segment_size(zv->zv_zso->zvo_queue, - PAGE_SIZE); - blk_queue_max_segments(zv->zv_zso->zvo_queue, - (zv->zv_volblocksize * chunks) / PAGE_SIZE); - } else { - /* - * Special case: zvol_blk_mq_blocks_per_thread = 0 - * Max everything out. - */ - blk_queue_max_segments(zv->zv_zso->zvo_queue, - UINT16_MAX); - blk_queue_max_segment_size(zv->zv_zso->zvo_queue, - UINT_MAX); - } -#endif - } else { - blk_queue_max_segments(zv->zv_zso->zvo_queue, UINT16_MAX); - blk_queue_max_segment_size(zv->zv_zso->zvo_queue, UINT_MAX); - } blk_queue_physical_block_size(zv->zv_zso->zvo_queue, zv->zv_volblocksize); - blk_queue_io_opt(zv->zv_zso->zvo_queue, zv->zv_volblocksize); blk_queue_max_discard_sectors(zv->zv_zso->zvo_queue, (zvol_max_discard_blocks * zv->zv_volblocksize) >> 9); blk_queue_discard_granularity(zv->zv_zso->zvo_queue, From e951dba48a6330aca9c161c50189f6974e6877f0 Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Tue, 28 May 2024 11:56:41 -0400 Subject: [PATCH 34/42] Linux 6.10: work harder to avoid kmem_cache_alloc reuse Linux 6.10 change kmem_cache_alloc to be a macro, rather than a function, such that the old #undef for it in spl-kmem-cache.c would remove its definition completely, breaking the build. This inverts the model used before. Rather than always defining the kmem_cache_* macro, then undefining then inside spl-kmem-cache.c, instead we make a special tag to indicate we're currently inside spl-kmem-cache.c, and not defining those in macros in the first place, so we can use the kernel-supplied kmem_cache_* functions to implement spl_kmem_cache_*, as we expect. For all other callers, we create the macros as normal and remove access to the kernel's own conflicting names. Signed-off-by: Rob Norris Sponsored-by: https://despairlabs.com/sponsor/ Reviewed-by: Tony Hutter Reviewed-by: Brian Behlendorf --- include/os/linux/spl/sys/kmem_cache.h | 19 +++++++++++-------- module/os/linux/spl/spl-kmem-cache.c | 12 ++---------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/include/os/linux/spl/sys/kmem_cache.h b/include/os/linux/spl/sys/kmem_cache.h index b159bb52d11..905ff57a143 100644 --- a/include/os/linux/spl/sys/kmem_cache.h +++ b/include/os/linux/spl/sys/kmem_cache.h @@ -192,22 +192,25 @@ extern void spl_kmem_reap(void); extern uint64_t spl_kmem_cache_inuse(kmem_cache_t *cache); extern uint64_t spl_kmem_cache_entry_size(kmem_cache_t *cache); +#ifndef SPL_KMEM_CACHE_IMPLEMENTING +/* + * Macros for the kmem_cache_* API expected by ZFS and SPL clients. We don't + * define them inside spl-kmem-cache.c, as that uses the kernel's incompatible + * kmem_cache_* facilities to implement ours. + */ + +/* Avoid conflicts with kernel names that might be implemented as macros. */ +#undef kmem_cache_alloc + #define kmem_cache_create(name, size, align, ctor, dtor, rclm, priv, vmp, fl) \ spl_kmem_cache_create(name, size, align, ctor, dtor, rclm, priv, vmp, fl) #define kmem_cache_set_move(skc, move) spl_kmem_cache_set_move(skc, move) #define kmem_cache_destroy(skc) spl_kmem_cache_destroy(skc) -/* - * This is necessary to be compatible with other kernel modules - * or in-tree filesystem that may define kmem_cache_alloc, - * like bcachefs does it now. - */ -#ifdef kmem_cache_alloc -#undef kmem_cache_alloc -#endif #define kmem_cache_alloc(skc, flags) spl_kmem_cache_alloc(skc, flags) #define kmem_cache_free(skc, obj) spl_kmem_cache_free(skc, obj) #define kmem_cache_reap_now(skc) spl_kmem_cache_reap_now(skc) #define kmem_reap() spl_kmem_reap() +#endif /* * The following functions are only available for internal use. diff --git a/module/os/linux/spl/spl-kmem-cache.c b/module/os/linux/spl/spl-kmem-cache.c index 42821ad6025..737c2e063f7 100644 --- a/module/os/linux/spl/spl-kmem-cache.c +++ b/module/os/linux/spl/spl-kmem-cache.c @@ -21,6 +21,8 @@ * with the SPL. If not, see . */ +#define SPL_KMEM_CACHE_IMPLEMENTING + #include #include #include @@ -33,16 +35,6 @@ #include #include -/* - * Within the scope of spl-kmem.c file the kmem_cache_* definitions - * are removed to allow access to the real Linux slab allocator. - */ -#undef kmem_cache_destroy -#undef kmem_cache_create -#undef kmem_cache_alloc -#undef kmem_cache_free - - /* * Linux 3.16 replaced smp_mb__{before,after}_{atomic,clear}_{dec,inc,bit}() * with smp_mb__{before,after}_atomic() because they were redundant. This is From 7ca7bb7fd723a91366ce767aea53c4f5c2d65afb Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Tue, 28 May 2024 16:16:28 -0400 Subject: [PATCH 35/42] Linux 5.16: use bdev_nr_bytes() to get device capacity This helper was introduced long ago, in 5.16. Since 6.10, bd_inode no longer exists, but the helper has been updated, so detect it and use it in all versions where it is available. Signed-off-by: Rob Norris Sponsored-by: https://despairlabs.com/sponsor/ Reviewed-by: Tony Hutter Reviewed-by: Brian Behlendorf --- config/kernel-blkdev.m4 | 26 ++++++++++++++++++++++++++ module/os/linux/zfs/vdev_disk.c | 14 +++++++++----- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/config/kernel-blkdev.m4 b/config/kernel-blkdev.m4 index b6ce1e1cf08..4f60f96acb5 100644 --- a/config/kernel-blkdev.m4 +++ b/config/kernel-blkdev.m4 @@ -534,6 +534,30 @@ AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_BDEV_WHOLE], [ ]) ]) +dnl # +dnl # 5.16 API change +dnl # Added bdev_nr_bytes() helper. +dnl # +AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_NR_BYTES], [ + ZFS_LINUX_TEST_SRC([bdev_nr_bytes], [ + #include + ],[ + struct block_device *bdev = NULL; + loff_t nr_bytes __attribute__ ((unused)) = 0; + nr_bytes = bdev_nr_bytes(bdev); + ]) +]) + +AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_BDEV_NR_BYTES], [ + AC_MSG_CHECKING([whether bdev_nr_bytes() is available]) + ZFS_LINUX_TEST_RESULT([bdev_nr_bytes], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_BDEV_NR_BYTES, 1, [bdev_nr_bytes() is available]) + ],[ + AC_MSG_RESULT(no) + ]) +]) + dnl # dnl # 5.20 API change, dnl # Removed bdevname(), snprintf(.., %pg) should be used. @@ -747,6 +771,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV], [ ZFS_AC_KERNEL_SRC_BLKDEV_CHECK_DISK_CHANGE ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_CHECK_MEDIA_CHANGE ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_WHOLE + ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_NR_BYTES ZFS_AC_KERNEL_SRC_BLKDEV_BDEVNAME ZFS_AC_KERNEL_SRC_BLKDEV_ISSUE_DISCARD ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_KOBJ @@ -767,6 +792,7 @@ AC_DEFUN([ZFS_AC_KERNEL_BLKDEV], [ ZFS_AC_KERNEL_BLKDEV_CHECK_DISK_CHANGE ZFS_AC_KERNEL_BLKDEV_BDEV_CHECK_MEDIA_CHANGE ZFS_AC_KERNEL_BLKDEV_BDEV_WHOLE + ZFS_AC_KERNEL_BLKDEV_BDEV_NR_BYTES ZFS_AC_KERNEL_BLKDEV_BDEVNAME ZFS_AC_KERNEL_BLKDEV_GET_ERESTARTSYS ZFS_AC_KERNEL_BLKDEV_ISSUE_DISCARD diff --git a/module/os/linux/zfs/vdev_disk.c b/module/os/linux/zfs/vdev_disk.c index 7284b922b3b..e69c5f3841e 100644 --- a/module/os/linux/zfs/vdev_disk.c +++ b/module/os/linux/zfs/vdev_disk.c @@ -150,7 +150,11 @@ vdev_bdev_mode(spa_mode_t smode) static uint64_t bdev_capacity(struct block_device *bdev) { +#ifdef HAVE_BDEV_NR_BYTES + return (bdev_nr_bytes(bdev)); +#else return (i_size_read(bdev->bd_inode)); +#endif } #if !defined(HAVE_BDEV_WHOLE) @@ -209,7 +213,7 @@ bdev_max_capacity(struct block_device *bdev, uint64_t wholedisk) * "reserved" EFI partition: in such cases return the device * usable capacity. */ - available = i_size_read(bdev_whole(bdev)->bd_inode) - + available = bdev_capacity(bdev_whole(bdev)) - ((EFI_MIN_RESV_SIZE + NEW_START_BLOCK + PARTITION_END_ALIGNMENT) << SECTOR_BITS); psize = MAX(available, bdev_capacity(bdev)); @@ -925,12 +929,12 @@ vdev_disk_io_rw(zio_t *zio) /* * Accessing outside the block device is never allowed. */ - if (zio->io_offset + zio->io_size > bdev->bd_inode->i_size) { + if (zio->io_offset + zio->io_size > bdev_capacity(bdev)) { vdev_dbgmsg(zio->io_vd, "Illegal access %llu size %llu, device size %llu", (u_longlong_t)zio->io_offset, (u_longlong_t)zio->io_size, - (u_longlong_t)i_size_read(bdev->bd_inode)); + (u_longlong_t)bdev_capacity(bdev)); return (SET_ERROR(EIO)); } @@ -1123,12 +1127,12 @@ vdev_classic_physio(zio_t *zio) /* * Accessing outside the block device is never allowed. */ - if (io_offset + io_size > bdev->bd_inode->i_size) { + if (io_offset + io_size > bdev_capacity(bdev)) { vdev_dbgmsg(zio->io_vd, "Illegal access %llu size %llu, device size %llu", (u_longlong_t)io_offset, (u_longlong_t)io_size, - (u_longlong_t)i_size_read(bdev->bd_inode)); + (u_longlong_t)bdev_capacity(bdev)); return (SET_ERROR(EIO)); } From f2ebbe46f6c04a4ad37bb6ec44d22a2a063a4430 Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Tue, 16 Jul 2024 16:27:54 -0700 Subject: [PATCH 36/42] Linux 6.9 compat: META (#16358) Update the META file to reflect compatibility with the 6.9 kernel. Signed-off-by: Tony Hutter Reviewed-by: Brian Behlendorf --- META | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/META b/META index 19a796050f5..7aac80c541b 100644 --- a/META +++ b/META @@ -6,5 +6,5 @@ Release: 1 Release-Tags: relext License: CDDL Author: OpenZFS -Linux-Maximum: 6.8 +Linux-Maximum: 6.9 Linux-Minimum: 3.10 From 41902c8e6da1c4dc5c4c12ce041acb4aba188e6f Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Tue, 16 Jul 2024 18:27:29 -0600 Subject: [PATCH 37/42] Use kmap_local_page instead of kmap_atomic (#16329) Changed zfs_k(un)map_atomic to zfs_k(un)map_local Signed-off-by: Jason Lee Reviewed-by: Alexander Motin Reviewed-by: Brian Atkinson --- config/kernel-kmap-local-page.m4 | 23 +++++++++++++++++++++ config/kernel.m4 | 2 ++ include/os/linux/kernel/linux/kmap_compat.h | 10 +++++++-- module/os/linux/zfs/abd_os.c | 8 +++---- module/os/linux/zfs/zfs_uio.c | 8 +++---- 5 files changed, 41 insertions(+), 10 deletions(-) create mode 100644 config/kernel-kmap-local-page.m4 diff --git a/config/kernel-kmap-local-page.m4 b/config/kernel-kmap-local-page.m4 new file mode 100644 index 00000000000..1990914d493 --- /dev/null +++ b/config/kernel-kmap-local-page.m4 @@ -0,0 +1,23 @@ +dnl # +dnl # 5.11 API change +dnl # kmap_atomic() was deprecated in favor of kmap_local_page() +dnl # +AC_DEFUN([ZFS_AC_KERNEL_SRC_KMAP_LOCAL_PAGE], [ + ZFS_LINUX_TEST_SRC([kmap_local_page], [ + #include + ],[ + struct page page; + kmap_local_page(&page); + ]) +]) + +AC_DEFUN([ZFS_AC_KERNEL_KMAP_LOCAL_PAGE], [ + AC_MSG_CHECKING([whether kmap_local_page exists]) + ZFS_LINUX_TEST_RESULT([kmap_local_page], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_KMAP_LOCAL_PAGE, 1, + [kernel has kmap_local_page]) + ],[ + AC_MSG_RESULT(no) + ]) +]) diff --git a/config/kernel.m4 b/config/kernel.m4 index b51477b6a95..f282ccd8b9d 100644 --- a/config/kernel.m4 +++ b/config/kernel.m4 @@ -125,6 +125,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [ ZFS_AC_KERNEL_SRC_VFS_DEDUPE_FILE_RANGE ZFS_AC_KERNEL_SRC_VFS_FILE_OPERATIONS_EXTEND ZFS_AC_KERNEL_SRC_KMAP_ATOMIC_ARGS + ZFS_AC_KERNEL_SRC_KMAP_LOCAL_PAGE ZFS_AC_KERNEL_SRC_FOLLOW_DOWN_ONE ZFS_AC_KERNEL_SRC_MAKE_REQUEST_FN ZFS_AC_KERNEL_SRC_GENERIC_IO_ACCT @@ -276,6 +277,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [ ZFS_AC_KERNEL_VFS_DEDUPE_FILE_RANGE ZFS_AC_KERNEL_VFS_FILE_OPERATIONS_EXTEND ZFS_AC_KERNEL_KMAP_ATOMIC_ARGS + ZFS_AC_KERNEL_KMAP_LOCAL_PAGE ZFS_AC_KERNEL_FOLLOW_DOWN_ONE ZFS_AC_KERNEL_MAKE_REQUEST_FN ZFS_AC_KERNEL_GENERIC_IO_ACCT diff --git a/include/os/linux/kernel/linux/kmap_compat.h b/include/os/linux/kernel/linux/kmap_compat.h index 7f9c00af802..fb59c5f0267 100644 --- a/include/os/linux/kernel/linux/kmap_compat.h +++ b/include/os/linux/kernel/linux/kmap_compat.h @@ -29,9 +29,15 @@ #include #include +#ifdef HAVE_KMAP_LOCAL_PAGE +/* 5.11 API change */ +#define zfs_kmap_local(page) kmap_local_page(page) +#define zfs_kunmap_local(addr) kunmap_local(addr) +#else /* 2.6.37 API change */ -#define zfs_kmap_atomic(page) kmap_atomic(page) -#define zfs_kunmap_atomic(addr) kunmap_atomic(addr) +#define zfs_kmap_local(page) kmap_atomic(page) +#define zfs_kunmap_local(addr) kunmap_atomic(addr) +#endif /* 5.0 API change - no more 'type' argument for access_ok() */ #ifdef HAVE_ACCESS_OK_TYPE diff --git a/module/os/linux/zfs/abd_os.c b/module/os/linux/zfs/abd_os.c index cee7410c883..4bf9eaf771b 100644 --- a/module/os/linux/zfs/abd_os.c +++ b/module/os/linux/zfs/abd_os.c @@ -526,8 +526,8 @@ abd_alloc_zero_scatter(void) #define PAGE_SHIFT (highbit64(PAGESIZE)-1) #endif -#define zfs_kmap_atomic(chunk) ((void *)chunk) -#define zfs_kunmap_atomic(addr) do { (void)(addr); } while (0) +#define zfs_kmap_local(chunk) ((void *)chunk) +#define zfs_kunmap_local(addr) do { (void)(addr); } while (0) #define local_irq_save(flags) do { (void)(flags); } while (0) #define local_irq_restore(flags) do { (void)(flags); } while (0) #define nth_page(pg, i) \ @@ -980,7 +980,7 @@ abd_iter_map(struct abd_iter *aiter) aiter->iter_mapsize = MIN(aiter->iter_sg->length - offset, aiter->iter_abd->abd_size - aiter->iter_pos); - paddr = zfs_kmap_atomic(sg_page(aiter->iter_sg)); + paddr = zfs_kmap_local(sg_page(aiter->iter_sg)); } aiter->iter_mapaddr = (char *)paddr + offset; @@ -999,7 +999,7 @@ abd_iter_unmap(struct abd_iter *aiter) if (!abd_is_linear(aiter->iter_abd)) { /* LINTED E_FUNC_SET_NOT_USED */ - zfs_kunmap_atomic(aiter->iter_mapaddr - aiter->iter_offset); + zfs_kunmap_local(aiter->iter_mapaddr - aiter->iter_offset); } ASSERT3P(aiter->iter_mapaddr, !=, NULL); diff --git a/module/os/linux/zfs/zfs_uio.c b/module/os/linux/zfs/zfs_uio.c index c2ed67c438c..a99a1ba8825 100644 --- a/module/os/linux/zfs/zfs_uio.c +++ b/module/os/linux/zfs/zfs_uio.c @@ -136,7 +136,7 @@ zfs_uiomove_bvec_impl(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio) void *paddr; cnt = MIN(bv->bv_len - skip, n); - paddr = zfs_kmap_atomic(bv->bv_page); + paddr = zfs_kmap_local(bv->bv_page); if (rw == UIO_READ) { /* Copy from buffer 'p' to the bvec data */ memcpy(paddr + bv->bv_offset + skip, p, cnt); @@ -144,7 +144,7 @@ zfs_uiomove_bvec_impl(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio) /* Copy from bvec data to buffer 'p' */ memcpy(p, paddr + bv->bv_offset + skip, cnt); } - zfs_kunmap_atomic(paddr); + zfs_kunmap_local(paddr); skip += cnt; if (skip == bv->bv_len) { @@ -168,7 +168,7 @@ zfs_copy_bvec(void *p, size_t skip, size_t cnt, zfs_uio_rw_t rw, { void *paddr; - paddr = zfs_kmap_atomic(bv->bv_page); + paddr = zfs_kmap_local(bv->bv_page); if (rw == UIO_READ) { /* Copy from buffer 'p' to the bvec data */ memcpy(paddr + bv->bv_offset + skip, p, cnt); @@ -176,7 +176,7 @@ zfs_copy_bvec(void *p, size_t skip, size_t cnt, zfs_uio_rw_t rw, /* Copy from bvec data to buffer 'p' */ memcpy(p, paddr + bv->bv_offset + skip, cnt); } - zfs_kunmap_atomic(paddr); + zfs_kunmap_local(paddr); } /* From 393b7ad6952217a7c0823f705f5b4a41d6b4f3f5 Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Thu, 9 May 2024 20:22:21 +1000 Subject: [PATCH 38/42] zts: test single-disk pool resumes properly after disk pull A single disk pool should suspend when its disk fails and hold the IO. When the disk is returned, the pool should return and the IO be reissued, leaving everything in good shape. Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Signed-off-by: Rob Norris Reviewed-by: Jorgen Lundman Reviewed-by: Tony Hutter Reviewed-by: Don Brady --- tests/runfiles/linux.run | 2 +- tests/test-runner/bin/zts-report.py.in | 1 + tests/zfs-tests/tests/Makefile.am | 1 + .../fault/suspend_resume_single.ksh | 102 ++++++++++++++++++ 4 files changed, 105 insertions(+), 1 deletion(-) create mode 100755 tests/zfs-tests/tests/functional/fault/suspend_resume_single.ksh diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index 92ce09ec6fc..bd6cc56f358 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -121,7 +121,7 @@ tests = ['auto_offline_001_pos', 'auto_online_001_pos', 'auto_online_002_pos', 'auto_replace_001_pos', 'auto_replace_002_pos', 'auto_spare_001_pos', 'auto_spare_002_pos', 'auto_spare_multiple', 'auto_spare_ashift', 'auto_spare_shared', 'decrypt_fault', 'decompress_fault', - 'scrub_after_resilver', 'zpool_status_-s'] + 'scrub_after_resilver', 'suspend_resume_single', 'zpool_status_-s'] tags = ['functional', 'fault'] [tests/functional/features/large_dnode:Linux] diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in index 5ca13093136..de06c7c6e2c 100755 --- a/tests/test-runner/bin/zts-report.py.in +++ b/tests/test-runner/bin/zts-report.py.in @@ -379,6 +379,7 @@ if os.environ.get('CI') == 'true': 'fault/auto_replace_002_pos': ['SKIP', ci_reason], 'fault/auto_spare_ashift': ['SKIP', ci_reason], 'fault/auto_spare_shared': ['SKIP', ci_reason], + 'fault/suspend_resume_single': ['SKIP', ci_reason], 'procfs/pool_state': ['SKIP', ci_reason], }) diff --git a/tests/zfs-tests/tests/Makefile.am b/tests/zfs-tests/tests/Makefile.am index 44eedcf6fae..00f306122da 100644 --- a/tests/zfs-tests/tests/Makefile.am +++ b/tests/zfs-tests/tests/Makefile.am @@ -1483,6 +1483,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/fault/decompress_fault.ksh \ functional/fault/decrypt_fault.ksh \ functional/fault/scrub_after_resilver.ksh \ + functional/fault/suspend_resume_single.ksh \ functional/fault/setup.ksh \ functional/fault/zpool_status_-s.ksh \ functional/features/async_destroy/async_destroy_001_pos.ksh \ diff --git a/tests/zfs-tests/tests/functional/fault/suspend_resume_single.ksh b/tests/zfs-tests/tests/functional/fault/suspend_resume_single.ksh new file mode 100755 index 00000000000..041dadb1ead --- /dev/null +++ b/tests/zfs-tests/tests/functional/fault/suspend_resume_single.ksh @@ -0,0 +1,102 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2024, Klara Inc. +# + +. $STF_SUITE/include/libtest.shlib + +set -x + +DATAFILE="$TMPDIR/datafile" + +function cleanup +{ + destroy_pool $TESTPOOL + unload_scsi_debug + rm -f $DATA_FILE +} + +log_onexit cleanup + +log_assert "ensure single-disk pool resumes properly after suspend and clear" + +# create a file, and take a checksum, so we can compare later +log_must dd if=/dev/random of=$DATAFILE bs=128K count=1 +typeset sum1=$(cat $DATAFILE | md5sum) + +# make a debug device that we can "unplug" +load_scsi_debug 100 1 1 1 '512b' +sd=$(get_debug_device) + +# create a single-device pool +log_must zpool create $TESTPOOL $sd +log_must zpool sync + +# "pull" the disk +log_must eval "echo offline > /sys/block/$sd/device/state" + +# copy data onto the pool. it'll appear to succeed, but only be in memory +log_must cp $DATAFILE /$TESTPOOL/file + +# wait until sync starts, and the pool suspends +log_note "waiting for pool to suspend" +typeset -i tries=10 +until [[ $(cat /proc/spl/kstat/zfs/$TESTPOOL/state) == "SUSPENDED" ]] ; do + if ((tries-- == 0)); then + log_fail "pool didn't suspend" + fi + sleep 1 +done + +# return the disk +log_must eval "echo running > /sys/block/$sd/device/state" + +# clear the error states, which should reopen the vdev, get the pool back +# online, and replay the failed IO +log_must zpool clear $TESTPOOL + +# wait a while for everything to sync out. if something is going to go wrong, +# this is where it will happen +log_note "giving pool time to settle and complete txg" +sleep 7 + +# if the pool suspended, then everything is bad +if [[ $(cat /proc/spl/kstat/zfs/$TESTPOOL/state) == "SUSPENDED" ]] ; then + log_fail "pool suspended" +fi + +# export the pool, to make sure it exports clean, and also to clear the file +# out of the cache +log_must zpool export $TESTPOOL + +# import the pool +log_must zpool import $TESTPOOL + +# sum the file we wrote earlier +typeset sum2=$(cat /$TESTPOOL/file | md5sum) + +# make sure the checksums match +log_must test "$sum1" = "$sum2" + +log_pass "single-disk pool resumes properly after disk suspend and clear" From 5de3ac223623d5348e491cc89c70a803ddcd7184 Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Tue, 11 Jun 2024 20:49:10 +1000 Subject: [PATCH 39/42] vdev_open: clear async fault flag after reopen After c3f2f1aa2, vdev_fault_wanted is set on a vdev after a probe fails. An end-of-txg async task is charged with actually faulting the vdev. In a single-disk pool, the probe failure will degrade the last disk, and then suspend the pool. However, vdev_fault_wanted is not cleared. After the pool returns, the transaction finishes and the async task runs and faults the vdev, which suspends the pool again. The fix is simple: when reopening a vdev, clear the async fault flag. If the vdev is still failed, the startup probe will quickly notice and degrade/suspend it again. If not, all is well! Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Co-authored-by: Don Brady Signed-off-by: Rob Norris Reviewed-by: Jorgen Lundman Reviewed-by: Tony Hutter Reviewed-by: Don Brady --- module/zfs/vdev.c | 1 + 1 file changed, 1 insertion(+) diff --git a/module/zfs/vdev.c b/module/zfs/vdev.c index c74f72159dc..11cc39ba352 100644 --- a/module/zfs/vdev.c +++ b/module/zfs/vdev.c @@ -2021,6 +2021,7 @@ vdev_open(vdev_t *vd) vd->vdev_stat.vs_aux = VDEV_AUX_NONE; vd->vdev_cant_read = B_FALSE; vd->vdev_cant_write = B_FALSE; + vd->vdev_fault_wanted = B_FALSE; vd->vdev_min_asize = vdev_get_min_asize(vd); /* From dc91e74524826b461d23f3399a0974ca33f84005 Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Thu, 18 Jul 2024 05:02:28 +1000 Subject: [PATCH 40/42] zdb: dump ZAP_FLAG_UINT64_KEY ZAPs properly (#16334) These are used for DDT and BRT stores. There's limited information available to produce meaningful output, but at least we can put something on screen rather than crashing. Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Signed-off-by: Rob Norris Reviewed-by: Alexander Motin Reviewed-by: Tony Hutter --- cmd/zdb/zdb.c | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c index 0d2f0b1da5f..f55c7f7b817 100644 --- a/cmd/zdb/zdb.c +++ b/cmd/zdb/zdb.c @@ -48,6 +48,7 @@ #include #include #include +#include #include #include #include @@ -1126,16 +1127,33 @@ dump_zap(objset_t *os, uint64_t object, void *data, size_t size) for (zap_cursor_init(&zc, os, object); zap_cursor_retrieve(&zc, &attr) == 0; zap_cursor_advance(&zc)) { - (void) printf("\t\t%s = ", attr.za_name); + boolean_t key64 = + !!(zap_getflags(zc.zc_zap) & ZAP_FLAG_UINT64_KEY); + + if (key64) + (void) printf("\t\t0x%010lx = ", + *(uint64_t *)attr.za_name); + else + (void) printf("\t\t%s = ", attr.za_name); + if (attr.za_num_integers == 0) { (void) printf("\n"); continue; } prop = umem_zalloc(attr.za_num_integers * attr.za_integer_length, UMEM_NOFAIL); - (void) zap_lookup(os, object, attr.za_name, - attr.za_integer_length, attr.za_num_integers, prop); - if (attr.za_integer_length == 1) { + + if (key64) + (void) zap_lookup_uint64(os, object, + (const uint64_t *)attr.za_name, 1, + attr.za_integer_length, attr.za_num_integers, + prop); + else + (void) zap_lookup(os, object, attr.za_name, + attr.za_integer_length, attr.za_num_integers, + prop); + + if (attr.za_integer_length == 1 && !key64) { if (strcmp(attr.za_name, DSL_CRYPTO_KEY_MASTER_KEY) == 0 || strcmp(attr.za_name, @@ -1154,6 +1172,10 @@ dump_zap(objset_t *os, uint64_t object, void *data, size_t size) } else { for (i = 0; i < attr.za_num_integers; i++) { switch (attr.za_integer_length) { + case 1: + (void) printf("%u ", + ((uint8_t *)prop)[i]); + break; case 2: (void) printf("%u ", ((uint16_t *)prop)[i]); From ab6d9bd89a87ed65c83db241fb476e81d88e166f Mon Sep 17 00:00:00 2001 From: youzhongyang Date: Wed, 17 Jul 2024 16:54:11 -0400 Subject: [PATCH 41/42] Make sure avl_tree.avl_pad is not in kernel module (#16280) The commit b192a2c (Remove avl_size field from struct avl_tree) uses a def _KERNEL to decide to include avl_pad or not, but this _KERNEL is defined in sys/sysmacros.h. If avl.h and sysmacros.h are not included in the right order, it can cause a headache when working on a zfs related kernel module. Add sysmacros.h in avl_impl.h to fix. sysmacros.h is also removed from spa.h as it's reduntant. Signed-off-by: Youzhong Yang Co-authored-by: Youzhong Yang Reviewed-by: Rob Norris Reviewed-by: Alexander Motin Reviewed-by: Tino Reichardt Reviewed-by: Tony Hutter --- include/sys/avl_impl.h | 1 + include/sys/spa.h | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/sys/avl_impl.h b/include/sys/avl_impl.h index 85277b42b47..b2b4eb4bce2 100644 --- a/include/sys/avl_impl.h +++ b/include/sys/avl_impl.h @@ -34,6 +34,7 @@ */ #include +#include #ifdef __cplusplus extern "C" { diff --git a/include/sys/spa.h b/include/sys/spa.h index 3073c4d1b93..f50cb5e04ee 100644 --- a/include/sys/spa.h +++ b/include/sys/spa.h @@ -35,11 +35,10 @@ #ifndef _SYS_SPA_H #define _SYS_SPA_H -#include #include +#include #include #include -#include #include #include #include From 1147a279785329a2eaeb01c0610c35fe835a62a6 Mon Sep 17 00:00:00 2001 From: glibg10b <56197853+glibg10b@users.noreply.github.com> Date: Thu, 18 Jul 2024 02:18:12 +0200 Subject: [PATCH 42/42] Fix printf typo for `zfs receive -cv` (#16295) Current output: > receiving correctivefull stream of a into b New output: > receiving corrective full stream of a into b Signed-off-by: glibg10b <56197853+glibg10b@users.noreply.github.com> Reviewed-by: Rob Norris Reviewed-by: Tino Reichardt Reviewed-by: Tony Hutter --- lib/libzfs/libzfs_sendrecv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/libzfs/libzfs_sendrecv.c b/lib/libzfs/libzfs_sendrecv.c index 0370112c022..ee01ee9b218 100644 --- a/lib/libzfs/libzfs_sendrecv.c +++ b/lib/libzfs/libzfs_sendrecv.c @@ -4952,7 +4952,7 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, if (flags->verbose) { (void) printf("%s %s%s stream of %s into %s\n", flags->dryrun ? "would receive" : "receiving", - flags->heal ? " corrective" : "", + flags->heal ? "corrective " : "", drrb->drr_fromguid ? "incremental" : "full", drrb->drr_toname, destsnap); (void) fflush(stdout);