From: Eric Biggers Date: Thu, 11 Dec 2025 01:18:37 +0000 (-0800) Subject: lib/crypto: x86/nh: Migrate optimized code into library X-Git-Url: http://git.monstr.eu/?a=commitdiff_plain;h=a229d83235c7627c490deb7dd4744a72567cea12;p=linux-2.6-microblaze.git lib/crypto: x86/nh: Migrate optimized code into library Migrate the x86_64 implementations of NH into lib/crypto/. This makes the nh() function be optimized on x86_64 kernels. Note: this temporarily makes the adiantum template not utilize the x86_64 optimized NH code. This is resolved in a later commit that converts the adiantum template to use nh() instead of "nhpoly1305". Link: https://lore.kernel.org/r/20251211011846.8179-6-ebiggers@kernel.org Signed-off-by: Eric Biggers --- diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 3fd2423d3cf8..ebb0838eaf30 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -333,26 +333,6 @@ config CRYPTO_AEGIS128_AESNI_SSE2 - AES-NI (AES New Instructions) - SSE4.1 (Streaming SIMD Extensions 4.1) -config CRYPTO_NHPOLY1305_SSE2 - tristate "Hash functions: NHPoly1305 (SSE2)" - depends on 64BIT - select CRYPTO_NHPOLY1305 - help - NHPoly1305 hash function for Adiantum - - Architecture: x86_64 using: - - SSE2 (Streaming SIMD Extensions 2) - -config CRYPTO_NHPOLY1305_AVX2 - tristate "Hash functions: NHPoly1305 (AVX2)" - depends on 64BIT - select CRYPTO_NHPOLY1305 - help - NHPoly1305 hash function for Adiantum - - Architecture: x86_64 using: - - AVX2 (Advanced Vector Extensions 2) - config CRYPTO_SM3_AVX_X86_64 tristate "Hash functions: SM3 (AVX)" depends on 64BIT diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 5f2fb4f148fe..b21ad0978c52 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -53,11 +53,6 @@ aesni-intel-$(CONFIG_64BIT) += aes-ctr-avx-x86_64.o \ obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o -obj-$(CONFIG_CRYPTO_NHPOLY1305_SSE2) += nhpoly1305-sse2.o -nhpoly1305-sse2-y := nh-sse2-x86_64.o nhpoly1305-sse2-glue.o -obj-$(CONFIG_CRYPTO_NHPOLY1305_AVX2) += nhpoly1305-avx2.o -nhpoly1305-avx2-y := nh-avx2-x86_64.o nhpoly1305-avx2-glue.o - obj-$(CONFIG_CRYPTO_SM3_AVX_X86_64) += sm3-avx-x86_64.o sm3-avx-x86_64-y := sm3-avx-asm_64.o sm3_avx_glue.o diff --git a/arch/x86/crypto/nh-avx2-x86_64.S b/arch/x86/crypto/nh-avx2-x86_64.S deleted file mode 100644 index 791386d9a83a..000000000000 --- a/arch/x86/crypto/nh-avx2-x86_64.S +++ /dev/null @@ -1,159 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * NH - ε-almost-universal hash function, x86_64 AVX2 accelerated - * - * Copyright 2018 Google LLC - * - * Author: Eric Biggers - */ - -#include -#include - -#define PASS0_SUMS %ymm0 -#define PASS1_SUMS %ymm1 -#define PASS2_SUMS %ymm2 -#define PASS3_SUMS %ymm3 -#define K0 %ymm4 -#define K0_XMM %xmm4 -#define K1 %ymm5 -#define K1_XMM %xmm5 -#define K2 %ymm6 -#define K2_XMM %xmm6 -#define K3 %ymm7 -#define K3_XMM %xmm7 -#define T0 %ymm8 -#define T1 %ymm9 -#define T2 %ymm10 -#define T2_XMM %xmm10 -#define T3 %ymm11 -#define T3_XMM %xmm11 -#define T4 %ymm12 -#define T5 %ymm13 -#define T6 %ymm14 -#define T7 %ymm15 -#define KEY %rdi -#define MESSAGE %rsi -#define MESSAGE_LEN %rdx -#define HASH %rcx - -.macro _nh_2xstride k0, k1, k2, k3 - - // Add message words to key words - vpaddd \k0, T3, T0 - vpaddd \k1, T3, T1 - vpaddd \k2, T3, T2 - vpaddd \k3, T3, T3 - - // Multiply 32x32 => 64 and accumulate - vpshufd $0x10, T0, T4 - vpshufd $0x32, T0, T0 - vpshufd $0x10, T1, T5 - vpshufd $0x32, T1, T1 - vpshufd $0x10, T2, T6 - vpshufd $0x32, T2, T2 - vpshufd $0x10, T3, T7 - vpshufd $0x32, T3, T3 - vpmuludq T4, T0, T0 - vpmuludq T5, T1, T1 - vpmuludq T6, T2, T2 - vpmuludq T7, T3, T3 - vpaddq T0, PASS0_SUMS, PASS0_SUMS - vpaddq T1, PASS1_SUMS, PASS1_SUMS - vpaddq T2, PASS2_SUMS, PASS2_SUMS - vpaddq T3, PASS3_SUMS, PASS3_SUMS -.endm - -/* - * void nh_avx2(const u32 *key, const u8 *message, size_t message_len, - * __le64 hash[NH_NUM_PASSES]) - * - * It's guaranteed that message_len % 16 == 0. - */ -SYM_TYPED_FUNC_START(nh_avx2) - - vmovdqu 0x00(KEY), K0 - vmovdqu 0x10(KEY), K1 - add $0x20, KEY - vpxor PASS0_SUMS, PASS0_SUMS, PASS0_SUMS - vpxor PASS1_SUMS, PASS1_SUMS, PASS1_SUMS - vpxor PASS2_SUMS, PASS2_SUMS, PASS2_SUMS - vpxor PASS3_SUMS, PASS3_SUMS, PASS3_SUMS - - sub $0x40, MESSAGE_LEN - jl .Lloop4_done -.Lloop4: - vmovdqu (MESSAGE), T3 - vmovdqu 0x00(KEY), K2 - vmovdqu 0x10(KEY), K3 - _nh_2xstride K0, K1, K2, K3 - - vmovdqu 0x20(MESSAGE), T3 - vmovdqu 0x20(KEY), K0 - vmovdqu 0x30(KEY), K1 - _nh_2xstride K2, K3, K0, K1 - - add $0x40, MESSAGE - add $0x40, KEY - sub $0x40, MESSAGE_LEN - jge .Lloop4 - -.Lloop4_done: - and $0x3f, MESSAGE_LEN - jz .Ldone - - cmp $0x20, MESSAGE_LEN - jl .Llast - - // 2 or 3 strides remain; do 2 more. - vmovdqu (MESSAGE), T3 - vmovdqu 0x00(KEY), K2 - vmovdqu 0x10(KEY), K3 - _nh_2xstride K0, K1, K2, K3 - add $0x20, MESSAGE - add $0x20, KEY - sub $0x20, MESSAGE_LEN - jz .Ldone - vmovdqa K2, K0 - vmovdqa K3, K1 -.Llast: - // Last stride. Zero the high 128 bits of the message and keys so they - // don't affect the result when processing them like 2 strides. - vmovdqu (MESSAGE), T3_XMM - vmovdqa K0_XMM, K0_XMM - vmovdqa K1_XMM, K1_XMM - vmovdqu 0x00(KEY), K2_XMM - vmovdqu 0x10(KEY), K3_XMM - _nh_2xstride K0, K1, K2, K3 - -.Ldone: - // Sum the accumulators for each pass, then store the sums to 'hash' - - // PASS0_SUMS is (0A 0B 0C 0D) - // PASS1_SUMS is (1A 1B 1C 1D) - // PASS2_SUMS is (2A 2B 2C 2D) - // PASS3_SUMS is (3A 3B 3C 3D) - // We need the horizontal sums: - // (0A + 0B + 0C + 0D, - // 1A + 1B + 1C + 1D, - // 2A + 2B + 2C + 2D, - // 3A + 3B + 3C + 3D) - // - - vpunpcklqdq PASS1_SUMS, PASS0_SUMS, T0 // T0 = (0A 1A 0C 1C) - vpunpckhqdq PASS1_SUMS, PASS0_SUMS, T1 // T1 = (0B 1B 0D 1D) - vpunpcklqdq PASS3_SUMS, PASS2_SUMS, T2 // T2 = (2A 3A 2C 3C) - vpunpckhqdq PASS3_SUMS, PASS2_SUMS, T3 // T3 = (2B 3B 2D 3D) - - vinserti128 $0x1, T2_XMM, T0, T4 // T4 = (0A 1A 2A 3A) - vinserti128 $0x1, T3_XMM, T1, T5 // T5 = (0B 1B 2B 3B) - vperm2i128 $0x31, T2, T0, T0 // T0 = (0C 1C 2C 3C) - vperm2i128 $0x31, T3, T1, T1 // T1 = (0D 1D 2D 3D) - - vpaddq T5, T4, T4 - vpaddq T1, T0, T0 - vpaddq T4, T0, T0 - vmovdqu T0, (HASH) - vzeroupper - RET -SYM_FUNC_END(nh_avx2) diff --git a/arch/x86/crypto/nh-sse2-x86_64.S b/arch/x86/crypto/nh-sse2-x86_64.S deleted file mode 100644 index 75fb994b6d17..000000000000 --- a/arch/x86/crypto/nh-sse2-x86_64.S +++ /dev/null @@ -1,124 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * NH - ε-almost-universal hash function, x86_64 SSE2 accelerated - * - * Copyright 2018 Google LLC - * - * Author: Eric Biggers - */ - -#include -#include - -#define PASS0_SUMS %xmm0 -#define PASS1_SUMS %xmm1 -#define PASS2_SUMS %xmm2 -#define PASS3_SUMS %xmm3 -#define K0 %xmm4 -#define K1 %xmm5 -#define K2 %xmm6 -#define K3 %xmm7 -#define T0 %xmm8 -#define T1 %xmm9 -#define T2 %xmm10 -#define T3 %xmm11 -#define T4 %xmm12 -#define T5 %xmm13 -#define T6 %xmm14 -#define T7 %xmm15 -#define KEY %rdi -#define MESSAGE %rsi -#define MESSAGE_LEN %rdx -#define HASH %rcx - -.macro _nh_stride k0, k1, k2, k3, offset - - // Load next message stride - movdqu \offset(MESSAGE), T1 - - // Load next key stride - movdqu \offset(KEY), \k3 - - // Add message words to key words - movdqa T1, T2 - movdqa T1, T3 - paddd T1, \k0 // reuse k0 to avoid a move - paddd \k1, T1 - paddd \k2, T2 - paddd \k3, T3 - - // Multiply 32x32 => 64 and accumulate - pshufd $0x10, \k0, T4 - pshufd $0x32, \k0, \k0 - pshufd $0x10, T1, T5 - pshufd $0x32, T1, T1 - pshufd $0x10, T2, T6 - pshufd $0x32, T2, T2 - pshufd $0x10, T3, T7 - pshufd $0x32, T3, T3 - pmuludq T4, \k0 - pmuludq T5, T1 - pmuludq T6, T2 - pmuludq T7, T3 - paddq \k0, PASS0_SUMS - paddq T1, PASS1_SUMS - paddq T2, PASS2_SUMS - paddq T3, PASS3_SUMS -.endm - -/* - * void nh_sse2(const u32 *key, const u8 *message, size_t message_len, - * __le64 hash[NH_NUM_PASSES]) - * - * It's guaranteed that message_len % 16 == 0. - */ -SYM_TYPED_FUNC_START(nh_sse2) - - movdqu 0x00(KEY), K0 - movdqu 0x10(KEY), K1 - movdqu 0x20(KEY), K2 - add $0x30, KEY - pxor PASS0_SUMS, PASS0_SUMS - pxor PASS1_SUMS, PASS1_SUMS - pxor PASS2_SUMS, PASS2_SUMS - pxor PASS3_SUMS, PASS3_SUMS - - sub $0x40, MESSAGE_LEN - jl .Lloop4_done -.Lloop4: - _nh_stride K0, K1, K2, K3, 0x00 - _nh_stride K1, K2, K3, K0, 0x10 - _nh_stride K2, K3, K0, K1, 0x20 - _nh_stride K3, K0, K1, K2, 0x30 - add $0x40, KEY - add $0x40, MESSAGE - sub $0x40, MESSAGE_LEN - jge .Lloop4 - -.Lloop4_done: - and $0x3f, MESSAGE_LEN - jz .Ldone - _nh_stride K0, K1, K2, K3, 0x00 - - sub $0x10, MESSAGE_LEN - jz .Ldone - _nh_stride K1, K2, K3, K0, 0x10 - - sub $0x10, MESSAGE_LEN - jz .Ldone - _nh_stride K2, K3, K0, K1, 0x20 - -.Ldone: - // Sum the accumulators for each pass, then store the sums to 'hash' - movdqa PASS0_SUMS, T0 - movdqa PASS2_SUMS, T1 - punpcklqdq PASS1_SUMS, T0 // => (PASS0_SUM_A PASS1_SUM_A) - punpcklqdq PASS3_SUMS, T1 // => (PASS2_SUM_A PASS3_SUM_A) - punpckhqdq PASS1_SUMS, PASS0_SUMS // => (PASS0_SUM_B PASS1_SUM_B) - punpckhqdq PASS3_SUMS, PASS2_SUMS // => (PASS2_SUM_B PASS3_SUM_B) - paddq PASS0_SUMS, T0 - paddq PASS2_SUMS, T1 - movdqu T0, 0x00(HASH) - movdqu T1, 0x10(HASH) - RET -SYM_FUNC_END(nh_sse2) diff --git a/arch/x86/crypto/nhpoly1305-avx2-glue.c b/arch/x86/crypto/nhpoly1305-avx2-glue.c deleted file mode 100644 index c3a872f4d6a7..000000000000 --- a/arch/x86/crypto/nhpoly1305-avx2-glue.c +++ /dev/null @@ -1,81 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum - * (AVX2 accelerated version) - * - * Copyright 2018 Google LLC - */ - -#include -#include -#include -#include -#include -#include - -asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len, - __le64 hash[NH_NUM_PASSES]); - -static int nhpoly1305_avx2_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - if (srclen < 64 || !crypto_simd_usable()) - return crypto_nhpoly1305_update(desc, src, srclen); - - do { - unsigned int n = min_t(unsigned int, srclen, SZ_4K); - - kernel_fpu_begin(); - crypto_nhpoly1305_update_helper(desc, src, n, nh_avx2); - kernel_fpu_end(); - src += n; - srclen -= n; - } while (srclen); - return 0; -} - -static int nhpoly1305_avx2_digest(struct shash_desc *desc, - const u8 *src, unsigned int srclen, u8 *out) -{ - return crypto_nhpoly1305_init(desc) ?: - nhpoly1305_avx2_update(desc, src, srclen) ?: - crypto_nhpoly1305_final(desc, out); -} - -static struct shash_alg nhpoly1305_alg = { - .base.cra_name = "nhpoly1305", - .base.cra_driver_name = "nhpoly1305-avx2", - .base.cra_priority = 300, - .base.cra_ctxsize = sizeof(struct nhpoly1305_key), - .base.cra_module = THIS_MODULE, - .digestsize = POLY1305_DIGEST_SIZE, - .init = crypto_nhpoly1305_init, - .update = nhpoly1305_avx2_update, - .final = crypto_nhpoly1305_final, - .digest = nhpoly1305_avx2_digest, - .setkey = crypto_nhpoly1305_setkey, - .descsize = sizeof(struct nhpoly1305_state), -}; - -static int __init nhpoly1305_mod_init(void) -{ - if (!boot_cpu_has(X86_FEATURE_AVX2) || - !boot_cpu_has(X86_FEATURE_OSXSAVE)) - return -ENODEV; - - return crypto_register_shash(&nhpoly1305_alg); -} - -static void __exit nhpoly1305_mod_exit(void) -{ - crypto_unregister_shash(&nhpoly1305_alg); -} - -module_init(nhpoly1305_mod_init); -module_exit(nhpoly1305_mod_exit); - -MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (AVX2-accelerated)"); -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Eric Biggers "); -MODULE_ALIAS_CRYPTO("nhpoly1305"); -MODULE_ALIAS_CRYPTO("nhpoly1305-avx2"); diff --git a/arch/x86/crypto/nhpoly1305-sse2-glue.c b/arch/x86/crypto/nhpoly1305-sse2-glue.c deleted file mode 100644 index a268a8439a5c..000000000000 --- a/arch/x86/crypto/nhpoly1305-sse2-glue.c +++ /dev/null @@ -1,80 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum - * (SSE2 accelerated version) - * - * Copyright 2018 Google LLC - */ - -#include -#include -#include -#include -#include -#include - -asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len, - __le64 hash[NH_NUM_PASSES]); - -static int nhpoly1305_sse2_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - if (srclen < 64 || !crypto_simd_usable()) - return crypto_nhpoly1305_update(desc, src, srclen); - - do { - unsigned int n = min_t(unsigned int, srclen, SZ_4K); - - kernel_fpu_begin(); - crypto_nhpoly1305_update_helper(desc, src, n, nh_sse2); - kernel_fpu_end(); - src += n; - srclen -= n; - } while (srclen); - return 0; -} - -static int nhpoly1305_sse2_digest(struct shash_desc *desc, - const u8 *src, unsigned int srclen, u8 *out) -{ - return crypto_nhpoly1305_init(desc) ?: - nhpoly1305_sse2_update(desc, src, srclen) ?: - crypto_nhpoly1305_final(desc, out); -} - -static struct shash_alg nhpoly1305_alg = { - .base.cra_name = "nhpoly1305", - .base.cra_driver_name = "nhpoly1305-sse2", - .base.cra_priority = 200, - .base.cra_ctxsize = sizeof(struct nhpoly1305_key), - .base.cra_module = THIS_MODULE, - .digestsize = POLY1305_DIGEST_SIZE, - .init = crypto_nhpoly1305_init, - .update = nhpoly1305_sse2_update, - .final = crypto_nhpoly1305_final, - .digest = nhpoly1305_sse2_digest, - .setkey = crypto_nhpoly1305_setkey, - .descsize = sizeof(struct nhpoly1305_state), -}; - -static int __init nhpoly1305_mod_init(void) -{ - if (!boot_cpu_has(X86_FEATURE_XMM2)) - return -ENODEV; - - return crypto_register_shash(&nhpoly1305_alg); -} - -static void __exit nhpoly1305_mod_exit(void) -{ - crypto_unregister_shash(&nhpoly1305_alg); -} - -module_init(nhpoly1305_mod_init); -module_exit(nhpoly1305_mod_exit); - -MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (SSE2-accelerated)"); -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Eric Biggers "); -MODULE_ALIAS_CRYPTO("nhpoly1305"); -MODULE_ALIAS_CRYPTO("nhpoly1305-sse2"); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index aa3f850ece24..33cf46bbadc8 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -119,6 +119,7 @@ config CRYPTO_LIB_NH_ARCH depends on CRYPTO_LIB_NH && !UML default y if ARM && KERNEL_MODE_NEON default y if ARM64 && KERNEL_MODE_NEON + default y if X86_64 config CRYPTO_LIB_POLY1305 tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index e3a13952bc2a..45128eccedef 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -137,6 +137,7 @@ ifeq ($(CONFIG_CRYPTO_LIB_NH_ARCH),y) CFLAGS_nh.o += -I$(src)/$(SRCARCH) libnh-$(CONFIG_ARM) += arm/nh-neon-core.o libnh-$(CONFIG_ARM64) += arm64/nh-neon-core.o +libnh-$(CONFIG_X86) += x86/nh-sse2.o x86/nh-avx2.o endif ################################################################################ diff --git a/lib/crypto/x86/nh-avx2.S b/lib/crypto/x86/nh-avx2.S new file mode 100644 index 000000000000..9c085a31b137 --- /dev/null +++ b/lib/crypto/x86/nh-avx2.S @@ -0,0 +1,158 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * NH - ε-almost-universal hash function, x86_64 AVX2 accelerated + * + * Copyright 2018 Google LLC + * + * Author: Eric Biggers + */ + +#include + +#define PASS0_SUMS %ymm0 +#define PASS1_SUMS %ymm1 +#define PASS2_SUMS %ymm2 +#define PASS3_SUMS %ymm3 +#define K0 %ymm4 +#define K0_XMM %xmm4 +#define K1 %ymm5 +#define K1_XMM %xmm5 +#define K2 %ymm6 +#define K2_XMM %xmm6 +#define K3 %ymm7 +#define K3_XMM %xmm7 +#define T0 %ymm8 +#define T1 %ymm9 +#define T2 %ymm10 +#define T2_XMM %xmm10 +#define T3 %ymm11 +#define T3_XMM %xmm11 +#define T4 %ymm12 +#define T5 %ymm13 +#define T6 %ymm14 +#define T7 %ymm15 +#define KEY %rdi +#define MESSAGE %rsi +#define MESSAGE_LEN %rdx +#define HASH %rcx + +.macro _nh_2xstride k0, k1, k2, k3 + + // Add message words to key words + vpaddd \k0, T3, T0 + vpaddd \k1, T3, T1 + vpaddd \k2, T3, T2 + vpaddd \k3, T3, T3 + + // Multiply 32x32 => 64 and accumulate + vpshufd $0x10, T0, T4 + vpshufd $0x32, T0, T0 + vpshufd $0x10, T1, T5 + vpshufd $0x32, T1, T1 + vpshufd $0x10, T2, T6 + vpshufd $0x32, T2, T2 + vpshufd $0x10, T3, T7 + vpshufd $0x32, T3, T3 + vpmuludq T4, T0, T0 + vpmuludq T5, T1, T1 + vpmuludq T6, T2, T2 + vpmuludq T7, T3, T3 + vpaddq T0, PASS0_SUMS, PASS0_SUMS + vpaddq T1, PASS1_SUMS, PASS1_SUMS + vpaddq T2, PASS2_SUMS, PASS2_SUMS + vpaddq T3, PASS3_SUMS, PASS3_SUMS +.endm + +/* + * void nh_avx2(const u32 *key, const u8 *message, size_t message_len, + * __le64 hash[NH_NUM_PASSES]) + * + * It's guaranteed that message_len % 16 == 0. + */ +SYM_FUNC_START(nh_avx2) + + vmovdqu 0x00(KEY), K0 + vmovdqu 0x10(KEY), K1 + add $0x20, KEY + vpxor PASS0_SUMS, PASS0_SUMS, PASS0_SUMS + vpxor PASS1_SUMS, PASS1_SUMS, PASS1_SUMS + vpxor PASS2_SUMS, PASS2_SUMS, PASS2_SUMS + vpxor PASS3_SUMS, PASS3_SUMS, PASS3_SUMS + + sub $0x40, MESSAGE_LEN + jl .Lloop4_done +.Lloop4: + vmovdqu (MESSAGE), T3 + vmovdqu 0x00(KEY), K2 + vmovdqu 0x10(KEY), K3 + _nh_2xstride K0, K1, K2, K3 + + vmovdqu 0x20(MESSAGE), T3 + vmovdqu 0x20(KEY), K0 + vmovdqu 0x30(KEY), K1 + _nh_2xstride K2, K3, K0, K1 + + add $0x40, MESSAGE + add $0x40, KEY + sub $0x40, MESSAGE_LEN + jge .Lloop4 + +.Lloop4_done: + and $0x3f, MESSAGE_LEN + jz .Ldone + + cmp $0x20, MESSAGE_LEN + jl .Llast + + // 2 or 3 strides remain; do 2 more. + vmovdqu (MESSAGE), T3 + vmovdqu 0x00(KEY), K2 + vmovdqu 0x10(KEY), K3 + _nh_2xstride K0, K1, K2, K3 + add $0x20, MESSAGE + add $0x20, KEY + sub $0x20, MESSAGE_LEN + jz .Ldone + vmovdqa K2, K0 + vmovdqa K3, K1 +.Llast: + // Last stride. Zero the high 128 bits of the message and keys so they + // don't affect the result when processing them like 2 strides. + vmovdqu (MESSAGE), T3_XMM + vmovdqa K0_XMM, K0_XMM + vmovdqa K1_XMM, K1_XMM + vmovdqu 0x00(KEY), K2_XMM + vmovdqu 0x10(KEY), K3_XMM + _nh_2xstride K0, K1, K2, K3 + +.Ldone: + // Sum the accumulators for each pass, then store the sums to 'hash' + + // PASS0_SUMS is (0A 0B 0C 0D) + // PASS1_SUMS is (1A 1B 1C 1D) + // PASS2_SUMS is (2A 2B 2C 2D) + // PASS3_SUMS is (3A 3B 3C 3D) + // We need the horizontal sums: + // (0A + 0B + 0C + 0D, + // 1A + 1B + 1C + 1D, + // 2A + 2B + 2C + 2D, + // 3A + 3B + 3C + 3D) + // + + vpunpcklqdq PASS1_SUMS, PASS0_SUMS, T0 // T0 = (0A 1A 0C 1C) + vpunpckhqdq PASS1_SUMS, PASS0_SUMS, T1 // T1 = (0B 1B 0D 1D) + vpunpcklqdq PASS3_SUMS, PASS2_SUMS, T2 // T2 = (2A 3A 2C 3C) + vpunpckhqdq PASS3_SUMS, PASS2_SUMS, T3 // T3 = (2B 3B 2D 3D) + + vinserti128 $0x1, T2_XMM, T0, T4 // T4 = (0A 1A 2A 3A) + vinserti128 $0x1, T3_XMM, T1, T5 // T5 = (0B 1B 2B 3B) + vperm2i128 $0x31, T2, T0, T0 // T0 = (0C 1C 2C 3C) + vperm2i128 $0x31, T3, T1, T1 // T1 = (0D 1D 2D 3D) + + vpaddq T5, T4, T4 + vpaddq T1, T0, T0 + vpaddq T4, T0, T0 + vmovdqu T0, (HASH) + vzeroupper + RET +SYM_FUNC_END(nh_avx2) diff --git a/lib/crypto/x86/nh-sse2.S b/lib/crypto/x86/nh-sse2.S new file mode 100644 index 000000000000..d36c0e6d5556 --- /dev/null +++ b/lib/crypto/x86/nh-sse2.S @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * NH - ε-almost-universal hash function, x86_64 SSE2 accelerated + * + * Copyright 2018 Google LLC + * + * Author: Eric Biggers + */ + +#include + +#define PASS0_SUMS %xmm0 +#define PASS1_SUMS %xmm1 +#define PASS2_SUMS %xmm2 +#define PASS3_SUMS %xmm3 +#define K0 %xmm4 +#define K1 %xmm5 +#define K2 %xmm6 +#define K3 %xmm7 +#define T0 %xmm8 +#define T1 %xmm9 +#define T2 %xmm10 +#define T3 %xmm11 +#define T4 %xmm12 +#define T5 %xmm13 +#define T6 %xmm14 +#define T7 %xmm15 +#define KEY %rdi +#define MESSAGE %rsi +#define MESSAGE_LEN %rdx +#define HASH %rcx + +.macro _nh_stride k0, k1, k2, k3, offset + + // Load next message stride + movdqu \offset(MESSAGE), T1 + + // Load next key stride + movdqu \offset(KEY), \k3 + + // Add message words to key words + movdqa T1, T2 + movdqa T1, T3 + paddd T1, \k0 // reuse k0 to avoid a move + paddd \k1, T1 + paddd \k2, T2 + paddd \k3, T3 + + // Multiply 32x32 => 64 and accumulate + pshufd $0x10, \k0, T4 + pshufd $0x32, \k0, \k0 + pshufd $0x10, T1, T5 + pshufd $0x32, T1, T1 + pshufd $0x10, T2, T6 + pshufd $0x32, T2, T2 + pshufd $0x10, T3, T7 + pshufd $0x32, T3, T3 + pmuludq T4, \k0 + pmuludq T5, T1 + pmuludq T6, T2 + pmuludq T7, T3 + paddq \k0, PASS0_SUMS + paddq T1, PASS1_SUMS + paddq T2, PASS2_SUMS + paddq T3, PASS3_SUMS +.endm + +/* + * void nh_sse2(const u32 *key, const u8 *message, size_t message_len, + * __le64 hash[NH_NUM_PASSES]) + * + * It's guaranteed that message_len % 16 == 0. + */ +SYM_FUNC_START(nh_sse2) + + movdqu 0x00(KEY), K0 + movdqu 0x10(KEY), K1 + movdqu 0x20(KEY), K2 + add $0x30, KEY + pxor PASS0_SUMS, PASS0_SUMS + pxor PASS1_SUMS, PASS1_SUMS + pxor PASS2_SUMS, PASS2_SUMS + pxor PASS3_SUMS, PASS3_SUMS + + sub $0x40, MESSAGE_LEN + jl .Lloop4_done +.Lloop4: + _nh_stride K0, K1, K2, K3, 0x00 + _nh_stride K1, K2, K3, K0, 0x10 + _nh_stride K2, K3, K0, K1, 0x20 + _nh_stride K3, K0, K1, K2, 0x30 + add $0x40, KEY + add $0x40, MESSAGE + sub $0x40, MESSAGE_LEN + jge .Lloop4 + +.Lloop4_done: + and $0x3f, MESSAGE_LEN + jz .Ldone + _nh_stride K0, K1, K2, K3, 0x00 + + sub $0x10, MESSAGE_LEN + jz .Ldone + _nh_stride K1, K2, K3, K0, 0x10 + + sub $0x10, MESSAGE_LEN + jz .Ldone + _nh_stride K2, K3, K0, K1, 0x20 + +.Ldone: + // Sum the accumulators for each pass, then store the sums to 'hash' + movdqa PASS0_SUMS, T0 + movdqa PASS2_SUMS, T1 + punpcklqdq PASS1_SUMS, T0 // => (PASS0_SUM_A PASS1_SUM_A) + punpcklqdq PASS3_SUMS, T1 // => (PASS2_SUM_A PASS3_SUM_A) + punpckhqdq PASS1_SUMS, PASS0_SUMS // => (PASS0_SUM_B PASS1_SUM_B) + punpckhqdq PASS3_SUMS, PASS2_SUMS // => (PASS2_SUM_B PASS3_SUM_B) + paddq PASS0_SUMS, T0 + paddq PASS2_SUMS, T1 + movdqu T0, 0x00(HASH) + movdqu T1, 0x10(HASH) + RET +SYM_FUNC_END(nh_sse2) diff --git a/lib/crypto/x86/nh.h b/lib/crypto/x86/nh.h new file mode 100644 index 000000000000..83361c2e9783 --- /dev/null +++ b/lib/crypto/x86/nh.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * x86_64 accelerated implementation of NH + * + * Copyright 2018 Google LLC + */ + +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sse2); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_avx2); + +asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len, + __le64 hash[NH_NUM_PASSES]); +asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len, + __le64 hash[NH_NUM_PASSES]); + +static bool nh_arch(const u32 *key, const u8 *message, size_t message_len, + __le64 hash[NH_NUM_PASSES]) +{ + if (message_len >= 64 && static_branch_likely(&have_sse2) && + irq_fpu_usable()) { + kernel_fpu_begin(); + if (static_branch_likely(&have_avx2)) + nh_avx2(key, message, message_len, hash); + else + nh_sse2(key, message, message_len, hash); + kernel_fpu_end(); + return true; + } + return false; +} + +#define nh_mod_init_arch nh_mod_init_arch +static void nh_mod_init_arch(void) +{ + if (boot_cpu_has(X86_FEATURE_XMM2)) { + static_branch_enable(&have_sse2); + if (boot_cpu_has(X86_FEATURE_AVX2) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, + NULL)) + static_branch_enable(&have_avx2); + } +}