From 22bd0c9731d73167352019c0c49d454196d029dc Mon Sep 17 00:00:00 2001 From: Mitchell Horne Date: Fri, 4 Dec 2020 21:12:17 +0000 Subject: [PATCH] ossl: port to arm64 Enable in-kernel acceleration of SHA1 and SHA2 operations on arm64 by adding support for the ossl(4) crypto driver. This uses OpenSSL's assembly routines under the hood, which will detect and use SHA intrinsics if they are supported by the CPU. Reviewed by: jhb Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D27390 --- share/man/man4/ossl.4 | 6 +- sys/conf/files.arm64 | 7 +++ sys/crypto/openssl/aarch64/arm_arch.h | 84 +++++++++++++++++++++++++++ sys/crypto/openssl/ossl_aarch64.c | 62 ++++++++++++++++++++ sys/modules/Makefile | 2 +- sys/modules/ossl/Makefile | 12 ++++ tests/sys/opencrypto/runtests.sh | 2 +- 7 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 sys/crypto/openssl/aarch64/arm_arch.h create mode 100644 sys/crypto/openssl/ossl_aarch64.c diff --git a/share/man/man4/ossl.4 b/share/man/man4/ossl.4 index 44da8e09831..ce97a9fc0f7 100644 --- a/share/man/man4/ossl.4 +++ b/share/man/man4/ossl.4 @@ -26,12 +26,12 @@ .\" .\" $FreeBSD$ .\" -.Dd October 19, 2020 +.Dd December 4, 2020 .Dt OSSL 4 .Os .Sh NAME .Nm ossl -.Nd "driver using OpenSSL assembly routines on x86 CPUs" +.Nd "driver using OpenSSL assembly routines" .Sh SYNOPSIS To compile this driver into the kernel, place the following lines in your @@ -61,6 +61,8 @@ architectures: .Pp .Bl -bullet -compact .It +arm64 +.It amd64 .It i386 diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 index 1eac2b68ef3..a1625a33589 100644 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -246,6 +246,13 @@ armv8_crypto_wrap.o optional armv8crypto \ no-implicit-rule \ clean "armv8_crypto_wrap.o" crypto/des/des_enc.c optional netsmb +crypto/openssl/ossl_aarch64.c optional ossl +crypto/openssl/aarch64/sha1-armv8.S optional ossl \ + compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC}" +crypto/openssl/aarch64/sha256-armv8.S optional ossl \ + compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC}" +crypto/openssl/aarch64/sha512-armv8.S optional ossl \ + compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC}" dev/acpica/acpi_bus_if.m optional acpi dev/acpica/acpi_if.m optional acpi dev/acpica/acpi_pci_link.c optional acpi pci diff --git a/sys/crypto/openssl/aarch64/arm_arch.h b/sys/crypto/openssl/aarch64/arm_arch.h new file mode 100644 index 00000000000..8b7105571d7 --- /dev/null +++ b/sys/crypto/openssl/aarch64/arm_arch.h @@ -0,0 +1,84 @@ +/* + * Copyright 2011-2018 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#ifndef OSSL_CRYPTO_ARM_ARCH_H +# define OSSL_CRYPTO_ARM_ARCH_H + +# if !defined(__ARM_ARCH__) +# if defined(__CC_ARM) +# define __ARM_ARCH__ __TARGET_ARCH_ARM +# if defined(__BIG_ENDIAN) +# define __ARMEB__ +# else +# define __ARMEL__ +# endif +# elif defined(__GNUC__) +# if defined(__aarch64__) +# define __ARM_ARCH__ 8 +# if __BYTE_ORDER__==__ORDER_BIG_ENDIAN__ +# define __ARMEB__ +# else +# define __ARMEL__ +# endif + /* + * Why doesn't gcc define __ARM_ARCH__? Instead it defines + * bunch of below macros. See all_architectures[] table in + * gcc/config/arm/arm.c. On a side note it defines + * __ARMEL__/__ARMEB__ for little-/big-endian. + */ +# elif defined(__ARM_ARCH) +# define __ARM_ARCH__ __ARM_ARCH +# elif defined(__ARM_ARCH_8A__) +# define __ARM_ARCH__ 8 +# elif defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || \ + defined(__ARM_ARCH_7R__)|| defined(__ARM_ARCH_7M__) || \ + defined(__ARM_ARCH_7EM__) +# define __ARM_ARCH__ 7 +# elif defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || \ + defined(__ARM_ARCH_6K__)|| defined(__ARM_ARCH_6M__) || \ + defined(__ARM_ARCH_6Z__)|| defined(__ARM_ARCH_6ZK__) || \ + defined(__ARM_ARCH_6T2__) +# define __ARM_ARCH__ 6 +# elif defined(__ARM_ARCH_5__) || defined(__ARM_ARCH_5T__) || \ + defined(__ARM_ARCH_5E__)|| defined(__ARM_ARCH_5TE__) || \ + defined(__ARM_ARCH_5TEJ__) +# define __ARM_ARCH__ 5 +# elif defined(__ARM_ARCH_4__) || defined(__ARM_ARCH_4T__) +# define __ARM_ARCH__ 4 +# else +# error "unsupported ARM architecture" +# endif +# endif +# endif + +# if !defined(__ARM_MAX_ARCH__) +# define __ARM_MAX_ARCH__ __ARM_ARCH__ +# endif + +# if __ARM_MAX_ARCH__<__ARM_ARCH__ +# error "__ARM_MAX_ARCH__ can't be less than __ARM_ARCH__" +# elif __ARM_MAX_ARCH__!=__ARM_ARCH__ +# if __ARM_ARCH__<7 && __ARM_MAX_ARCH__>=7 && defined(__ARMEB__) +# error "can't build universal big-endian binary" +# endif +# endif + +# ifndef __ASSEMBLER__ +extern unsigned int OPENSSL_armcap_P; +# endif + +# define ARMV7_NEON (1<<0) +# define ARMV7_TICK (1<<1) +# define ARMV8_AES (1<<2) +# define ARMV8_SHA1 (1<<3) +# define ARMV8_SHA256 (1<<4) +# define ARMV8_PMULL (1<<5) +# define ARMV8_SHA512 (1<<6) + +#endif diff --git a/sys/crypto/openssl/ossl_aarch64.c b/sys/crypto/openssl/ossl_aarch64.c new file mode 100644 index 00000000000..2a45a848808 --- /dev/null +++ b/sys/crypto/openssl/ossl_aarch64.c @@ -0,0 +1,62 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2020 The FreeBSD Foundation + * + * This software was developed by Mitchell Horne + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include + +#include +#include + +#include +#include + +/* + * Feature bits defined in arm_arch.h + */ +unsigned int OPENSSL_armcap_P; + +void +ossl_cpuid(void) +{ + /* SHA features */ + if ((elf_hwcap & HWCAP_SHA1) != 0) + OPENSSL_armcap_P |= ARMV8_SHA1; + if ((elf_hwcap & HWCAP_SHA2) != 0) + OPENSSL_armcap_P |= ARMV8_SHA256; + if ((elf_hwcap & HWCAP_SHA512) != 0) + OPENSSL_armcap_P |= ARMV8_SHA512; + + /* AES features */ + if ((elf_hwcap & HWCAP_AES) != 0) + OPENSSL_armcap_P |= ARMV8_AES; + if ((elf_hwcap & HWCAP_PMULL) != 0) + OPENSSL_armcap_P |= ARMV8_PMULL; +} diff --git a/sys/modules/Makefile b/sys/modules/Makefile index 654ec66268b..2457fc0a533 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -518,6 +518,7 @@ _mthca= mthca _mlx4ib= mlx4ib _mlx5ib= mlx5ib .endif +_ossl= ossl _vmware= vmware .endif @@ -634,7 +635,6 @@ _nctgpio= nctgpio _ndis= ndis _ntb= ntb _ocs_fc= ocs_fc -_ossl= ossl _pccard= pccard _qat= qat _qatfw= qatfw diff --git a/sys/modules/ossl/Makefile b/sys/modules/ossl/Makefile index 064677bc9b2..6fe8c5a6c81 100644 --- a/sys/modules/ossl/Makefile +++ b/sys/modules/ossl/Makefile @@ -13,6 +13,12 @@ SRCS= bus_if.h \ ossl_sha512.c \ ${SRCS.${MACHINE_CPUARCH}} +SRCS.aarch64= \ + sha1-armv8.S \ + sha256-armv8.S \ + sha512-armv8.S \ + ossl_aarch64.c + SRCS.amd64= \ sha1-x86_64.S \ sha256-x86_64.S \ @@ -25,4 +31,10 @@ SRCS.i386= \ sha512-586.S \ ossl_x86.c +# For arm64, we are forced to rewrite the compiler invocation for the assembly +# files, to remove -mgeneral-regs-only. +${SRCS.aarch64:M*.S:S/S/o/}: ${.TARGET:R}.S + ${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC} + ${CTFCONVERT_CMD} + .include diff --git a/tests/sys/opencrypto/runtests.sh b/tests/sys/opencrypto/runtests.sh index e411142e51b..a4e33ea38f9 100644 --- a/tests/sys/opencrypto/runtests.sh +++ b/tests/sys/opencrypto/runtests.sh @@ -65,7 +65,7 @@ cpu_module= case ${cpu_type} in aarch64) - cpu_module=nexus/armv8crypto + cpu_module="nexus/armv8crypto nexus/ossl" ;; amd64|i386) cpu_module="nexus/aesni nexus/ossl"