- AES-NI (AES New Instructions)
- SSE4.1 (Streaming SIMD Extensions 4.1)
-config CRYPTO_NHPOLY1305_SSE2
- tristate "Hash functions: NHPoly1305 (SSE2)"
- depends on 64BIT
- select CRYPTO_NHPOLY1305
- help
- NHPoly1305 hash function for Adiantum
-
- Architecture: x86_64 using:
- - SSE2 (Streaming SIMD Extensions 2)
-
-config CRYPTO_NHPOLY1305_AVX2
- tristate "Hash functions: NHPoly1305 (AVX2)"
- depends on 64BIT
- select CRYPTO_NHPOLY1305
- help
- NHPoly1305 hash function for Adiantum
-
- Architecture: x86_64 using:
- - AVX2 (Advanced Vector Extensions 2)
-
config CRYPTO_SM3_AVX_X86_64
tristate "Hash functions: SM3 (AVX)"
depends on 64BIT
obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o
ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o
-obj-$(CONFIG_CRYPTO_NHPOLY1305_SSE2) += nhpoly1305-sse2.o
-nhpoly1305-sse2-y := nh-sse2-x86_64.o nhpoly1305-sse2-glue.o
-obj-$(CONFIG_CRYPTO_NHPOLY1305_AVX2) += nhpoly1305-avx2.o
-nhpoly1305-avx2-y := nh-avx2-x86_64.o nhpoly1305-avx2-glue.o
-
obj-$(CONFIG_CRYPTO_SM3_AVX_X86_64) += sm3-avx-x86_64.o
sm3-avx-x86_64-y := sm3-avx-asm_64.o sm3_avx_glue.o
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * NH - ε-almost-universal hash function, x86_64 AVX2 accelerated
- *
- * Copyright 2018 Google LLC
- *
- * Author: Eric Biggers <ebiggers@google.com>
- */
-
-#include <linux/linkage.h>
-#include <linux/cfi_types.h>
-
-#define PASS0_SUMS %ymm0
-#define PASS1_SUMS %ymm1
-#define PASS2_SUMS %ymm2
-#define PASS3_SUMS %ymm3
-#define K0 %ymm4
-#define K0_XMM %xmm4
-#define K1 %ymm5
-#define K1_XMM %xmm5
-#define K2 %ymm6
-#define K2_XMM %xmm6
-#define K3 %ymm7
-#define K3_XMM %xmm7
-#define T0 %ymm8
-#define T1 %ymm9
-#define T2 %ymm10
-#define T2_XMM %xmm10
-#define T3 %ymm11
-#define T3_XMM %xmm11
-#define T4 %ymm12
-#define T5 %ymm13
-#define T6 %ymm14
-#define T7 %ymm15
-#define KEY %rdi
-#define MESSAGE %rsi
-#define MESSAGE_LEN %rdx
-#define HASH %rcx
-
-.macro _nh_2xstride k0, k1, k2, k3
-
- // Add message words to key words
- vpaddd \k0, T3, T0
- vpaddd \k1, T3, T1
- vpaddd \k2, T3, T2
- vpaddd \k3, T3, T3
-
- // Multiply 32x32 => 64 and accumulate
- vpshufd $0x10, T0, T4
- vpshufd $0x32, T0, T0
- vpshufd $0x10, T1, T5
- vpshufd $0x32, T1, T1
- vpshufd $0x10, T2, T6
- vpshufd $0x32, T2, T2
- vpshufd $0x10, T3, T7
- vpshufd $0x32, T3, T3
- vpmuludq T4, T0, T0
- vpmuludq T5, T1, T1
- vpmuludq T6, T2, T2
- vpmuludq T7, T3, T3
- vpaddq T0, PASS0_SUMS, PASS0_SUMS
- vpaddq T1, PASS1_SUMS, PASS1_SUMS
- vpaddq T2, PASS2_SUMS, PASS2_SUMS
- vpaddq T3, PASS3_SUMS, PASS3_SUMS
-.endm
-
-/*
- * void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
- * __le64 hash[NH_NUM_PASSES])
- *
- * It's guaranteed that message_len % 16 == 0.
- */
-SYM_TYPED_FUNC_START(nh_avx2)
-
- vmovdqu 0x00(KEY), K0
- vmovdqu 0x10(KEY), K1
- add $0x20, KEY
- vpxor PASS0_SUMS, PASS0_SUMS, PASS0_SUMS
- vpxor PASS1_SUMS, PASS1_SUMS, PASS1_SUMS
- vpxor PASS2_SUMS, PASS2_SUMS, PASS2_SUMS
- vpxor PASS3_SUMS, PASS3_SUMS, PASS3_SUMS
-
- sub $0x40, MESSAGE_LEN
- jl .Lloop4_done
-.Lloop4:
- vmovdqu (MESSAGE), T3
- vmovdqu 0x00(KEY), K2
- vmovdqu 0x10(KEY), K3
- _nh_2xstride K0, K1, K2, K3
-
- vmovdqu 0x20(MESSAGE), T3
- vmovdqu 0x20(KEY), K0
- vmovdqu 0x30(KEY), K1
- _nh_2xstride K2, K3, K0, K1
-
- add $0x40, MESSAGE
- add $0x40, KEY
- sub $0x40, MESSAGE_LEN
- jge .Lloop4
-
-.Lloop4_done:
- and $0x3f, MESSAGE_LEN
- jz .Ldone
-
- cmp $0x20, MESSAGE_LEN
- jl .Llast
-
- // 2 or 3 strides remain; do 2 more.
- vmovdqu (MESSAGE), T3
- vmovdqu 0x00(KEY), K2
- vmovdqu 0x10(KEY), K3
- _nh_2xstride K0, K1, K2, K3
- add $0x20, MESSAGE
- add $0x20, KEY
- sub $0x20, MESSAGE_LEN
- jz .Ldone
- vmovdqa K2, K0
- vmovdqa K3, K1
-.Llast:
- // Last stride. Zero the high 128 bits of the message and keys so they
- // don't affect the result when processing them like 2 strides.
- vmovdqu (MESSAGE), T3_XMM
- vmovdqa K0_XMM, K0_XMM
- vmovdqa K1_XMM, K1_XMM
- vmovdqu 0x00(KEY), K2_XMM
- vmovdqu 0x10(KEY), K3_XMM
- _nh_2xstride K0, K1, K2, K3
-
-.Ldone:
- // Sum the accumulators for each pass, then store the sums to 'hash'
-
- // PASS0_SUMS is (0A 0B 0C 0D)
- // PASS1_SUMS is (1A 1B 1C 1D)
- // PASS2_SUMS is (2A 2B 2C 2D)
- // PASS3_SUMS is (3A 3B 3C 3D)
- // We need the horizontal sums:
- // (0A + 0B + 0C + 0D,
- // 1A + 1B + 1C + 1D,
- // 2A + 2B + 2C + 2D,
- // 3A + 3B + 3C + 3D)
- //
-
- vpunpcklqdq PASS1_SUMS, PASS0_SUMS, T0 // T0 = (0A 1A 0C 1C)
- vpunpckhqdq PASS1_SUMS, PASS0_SUMS, T1 // T1 = (0B 1B 0D 1D)
- vpunpcklqdq PASS3_SUMS, PASS2_SUMS, T2 // T2 = (2A 3A 2C 3C)
- vpunpckhqdq PASS3_SUMS, PASS2_SUMS, T3 // T3 = (2B 3B 2D 3D)
-
- vinserti128 $0x1, T2_XMM, T0, T4 // T4 = (0A 1A 2A 3A)
- vinserti128 $0x1, T3_XMM, T1, T5 // T5 = (0B 1B 2B 3B)
- vperm2i128 $0x31, T2, T0, T0 // T0 = (0C 1C 2C 3C)
- vperm2i128 $0x31, T3, T1, T1 // T1 = (0D 1D 2D 3D)
-
- vpaddq T5, T4, T4
- vpaddq T1, T0, T0
- vpaddq T4, T0, T0
- vmovdqu T0, (HASH)
- vzeroupper
- RET
-SYM_FUNC_END(nh_avx2)
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * NH - ε-almost-universal hash function, x86_64 SSE2 accelerated
- *
- * Copyright 2018 Google LLC
- *
- * Author: Eric Biggers <ebiggers@google.com>
- */
-
-#include <linux/linkage.h>
-#include <linux/cfi_types.h>
-
-#define PASS0_SUMS %xmm0
-#define PASS1_SUMS %xmm1
-#define PASS2_SUMS %xmm2
-#define PASS3_SUMS %xmm3
-#define K0 %xmm4
-#define K1 %xmm5
-#define K2 %xmm6
-#define K3 %xmm7
-#define T0 %xmm8
-#define T1 %xmm9
-#define T2 %xmm10
-#define T3 %xmm11
-#define T4 %xmm12
-#define T5 %xmm13
-#define T6 %xmm14
-#define T7 %xmm15
-#define KEY %rdi
-#define MESSAGE %rsi
-#define MESSAGE_LEN %rdx
-#define HASH %rcx
-
-.macro _nh_stride k0, k1, k2, k3, offset
-
- // Load next message stride
- movdqu \offset(MESSAGE), T1
-
- // Load next key stride
- movdqu \offset(KEY), \k3
-
- // Add message words to key words
- movdqa T1, T2
- movdqa T1, T3
- paddd T1, \k0 // reuse k0 to avoid a move
- paddd \k1, T1
- paddd \k2, T2
- paddd \k3, T3
-
- // Multiply 32x32 => 64 and accumulate
- pshufd $0x10, \k0, T4
- pshufd $0x32, \k0, \k0
- pshufd $0x10, T1, T5
- pshufd $0x32, T1, T1
- pshufd $0x10, T2, T6
- pshufd $0x32, T2, T2
- pshufd $0x10, T3, T7
- pshufd $0x32, T3, T3
- pmuludq T4, \k0
- pmuludq T5, T1
- pmuludq T6, T2
- pmuludq T7, T3
- paddq \k0, PASS0_SUMS
- paddq T1, PASS1_SUMS
- paddq T2, PASS2_SUMS
- paddq T3, PASS3_SUMS
-.endm
-
-/*
- * void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
- * __le64 hash[NH_NUM_PASSES])
- *
- * It's guaranteed that message_len % 16 == 0.
- */
-SYM_TYPED_FUNC_START(nh_sse2)
-
- movdqu 0x00(KEY), K0
- movdqu 0x10(KEY), K1
- movdqu 0x20(KEY), K2
- add $0x30, KEY
- pxor PASS0_SUMS, PASS0_SUMS
- pxor PASS1_SUMS, PASS1_SUMS
- pxor PASS2_SUMS, PASS2_SUMS
- pxor PASS3_SUMS, PASS3_SUMS
-
- sub $0x40, MESSAGE_LEN
- jl .Lloop4_done
-.Lloop4:
- _nh_stride K0, K1, K2, K3, 0x00
- _nh_stride K1, K2, K3, K0, 0x10
- _nh_stride K2, K3, K0, K1, 0x20
- _nh_stride K3, K0, K1, K2, 0x30
- add $0x40, KEY
- add $0x40, MESSAGE
- sub $0x40, MESSAGE_LEN
- jge .Lloop4
-
-.Lloop4_done:
- and $0x3f, MESSAGE_LEN
- jz .Ldone
- _nh_stride K0, K1, K2, K3, 0x00
-
- sub $0x10, MESSAGE_LEN
- jz .Ldone
- _nh_stride K1, K2, K3, K0, 0x10
-
- sub $0x10, MESSAGE_LEN
- jz .Ldone
- _nh_stride K2, K3, K0, K1, 0x20
-
-.Ldone:
- // Sum the accumulators for each pass, then store the sums to 'hash'
- movdqa PASS0_SUMS, T0
- movdqa PASS2_SUMS, T1
- punpcklqdq PASS1_SUMS, T0 // => (PASS0_SUM_A PASS1_SUM_A)
- punpcklqdq PASS3_SUMS, T1 // => (PASS2_SUM_A PASS3_SUM_A)
- punpckhqdq PASS1_SUMS, PASS0_SUMS // => (PASS0_SUM_B PASS1_SUM_B)
- punpckhqdq PASS3_SUMS, PASS2_SUMS // => (PASS2_SUM_B PASS3_SUM_B)
- paddq PASS0_SUMS, T0
- paddq PASS2_SUMS, T1
- movdqu T0, 0x00(HASH)
- movdqu T1, 0x10(HASH)
- RET
-SYM_FUNC_END(nh_sse2)
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0
-/*
- * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
- * (AVX2 accelerated version)
- *
- * Copyright 2018 Google LLC
- */
-
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <crypto/nhpoly1305.h>
-#include <linux/module.h>
-#include <linux/sizes.h>
-#include <asm/simd.h>
-
-asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
- __le64 hash[NH_NUM_PASSES]);
-
-static int nhpoly1305_avx2_update(struct shash_desc *desc,
- const u8 *src, unsigned int srclen)
-{
- if (srclen < 64 || !crypto_simd_usable())
- return crypto_nhpoly1305_update(desc, src, srclen);
-
- do {
- unsigned int n = min_t(unsigned int, srclen, SZ_4K);
-
- kernel_fpu_begin();
- crypto_nhpoly1305_update_helper(desc, src, n, nh_avx2);
- kernel_fpu_end();
- src += n;
- srclen -= n;
- } while (srclen);
- return 0;
-}
-
-static int nhpoly1305_avx2_digest(struct shash_desc *desc,
- const u8 *src, unsigned int srclen, u8 *out)
-{
- return crypto_nhpoly1305_init(desc) ?:
- nhpoly1305_avx2_update(desc, src, srclen) ?:
- crypto_nhpoly1305_final(desc, out);
-}
-
-static struct shash_alg nhpoly1305_alg = {
- .base.cra_name = "nhpoly1305",
- .base.cra_driver_name = "nhpoly1305-avx2",
- .base.cra_priority = 300,
- .base.cra_ctxsize = sizeof(struct nhpoly1305_key),
- .base.cra_module = THIS_MODULE,
- .digestsize = POLY1305_DIGEST_SIZE,
- .init = crypto_nhpoly1305_init,
- .update = nhpoly1305_avx2_update,
- .final = crypto_nhpoly1305_final,
- .digest = nhpoly1305_avx2_digest,
- .setkey = crypto_nhpoly1305_setkey,
- .descsize = sizeof(struct nhpoly1305_state),
-};
-
-static int __init nhpoly1305_mod_init(void)
-{
- if (!boot_cpu_has(X86_FEATURE_AVX2) ||
- !boot_cpu_has(X86_FEATURE_OSXSAVE))
- return -ENODEV;
-
- return crypto_register_shash(&nhpoly1305_alg);
-}
-
-static void __exit nhpoly1305_mod_exit(void)
-{
- crypto_unregister_shash(&nhpoly1305_alg);
-}
-
-module_init(nhpoly1305_mod_init);
-module_exit(nhpoly1305_mod_exit);
-
-MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (AVX2-accelerated)");
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
-MODULE_ALIAS_CRYPTO("nhpoly1305");
-MODULE_ALIAS_CRYPTO("nhpoly1305-avx2");
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0
-/*
- * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
- * (SSE2 accelerated version)
- *
- * Copyright 2018 Google LLC
- */
-
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <crypto/nhpoly1305.h>
-#include <linux/module.h>
-#include <linux/sizes.h>
-#include <asm/simd.h>
-
-asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
- __le64 hash[NH_NUM_PASSES]);
-
-static int nhpoly1305_sse2_update(struct shash_desc *desc,
- const u8 *src, unsigned int srclen)
-{
- if (srclen < 64 || !crypto_simd_usable())
- return crypto_nhpoly1305_update(desc, src, srclen);
-
- do {
- unsigned int n = min_t(unsigned int, srclen, SZ_4K);
-
- kernel_fpu_begin();
- crypto_nhpoly1305_update_helper(desc, src, n, nh_sse2);
- kernel_fpu_end();
- src += n;
- srclen -= n;
- } while (srclen);
- return 0;
-}
-
-static int nhpoly1305_sse2_digest(struct shash_desc *desc,
- const u8 *src, unsigned int srclen, u8 *out)
-{
- return crypto_nhpoly1305_init(desc) ?:
- nhpoly1305_sse2_update(desc, src, srclen) ?:
- crypto_nhpoly1305_final(desc, out);
-}
-
-static struct shash_alg nhpoly1305_alg = {
- .base.cra_name = "nhpoly1305",
- .base.cra_driver_name = "nhpoly1305-sse2",
- .base.cra_priority = 200,
- .base.cra_ctxsize = sizeof(struct nhpoly1305_key),
- .base.cra_module = THIS_MODULE,
- .digestsize = POLY1305_DIGEST_SIZE,
- .init = crypto_nhpoly1305_init,
- .update = nhpoly1305_sse2_update,
- .final = crypto_nhpoly1305_final,
- .digest = nhpoly1305_sse2_digest,
- .setkey = crypto_nhpoly1305_setkey,
- .descsize = sizeof(struct nhpoly1305_state),
-};
-
-static int __init nhpoly1305_mod_init(void)
-{
- if (!boot_cpu_has(X86_FEATURE_XMM2))
- return -ENODEV;
-
- return crypto_register_shash(&nhpoly1305_alg);
-}
-
-static void __exit nhpoly1305_mod_exit(void)
-{
- crypto_unregister_shash(&nhpoly1305_alg);
-}
-
-module_init(nhpoly1305_mod_init);
-module_exit(nhpoly1305_mod_exit);
-
-MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (SSE2-accelerated)");
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
-MODULE_ALIAS_CRYPTO("nhpoly1305");
-MODULE_ALIAS_CRYPTO("nhpoly1305-sse2");
depends on CRYPTO_LIB_NH && !UML
default y if ARM && KERNEL_MODE_NEON
default y if ARM64 && KERNEL_MODE_NEON
+ default y if X86_64
config CRYPTO_LIB_POLY1305
tristate
CFLAGS_nh.o += -I$(src)/$(SRCARCH)
libnh-$(CONFIG_ARM) += arm/nh-neon-core.o
libnh-$(CONFIG_ARM64) += arm64/nh-neon-core.o
+libnh-$(CONFIG_X86) += x86/nh-sse2.o x86/nh-avx2.o
endif
################################################################################
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * NH - ε-almost-universal hash function, x86_64 AVX2 accelerated
+ *
+ * Copyright 2018 Google LLC
+ *
+ * Author: Eric Biggers <ebiggers@google.com>
+ */
+
+#include <linux/linkage.h>
+
+#define PASS0_SUMS %ymm0
+#define PASS1_SUMS %ymm1
+#define PASS2_SUMS %ymm2
+#define PASS3_SUMS %ymm3
+#define K0 %ymm4
+#define K0_XMM %xmm4
+#define K1 %ymm5
+#define K1_XMM %xmm5
+#define K2 %ymm6
+#define K2_XMM %xmm6
+#define K3 %ymm7
+#define K3_XMM %xmm7
+#define T0 %ymm8
+#define T1 %ymm9
+#define T2 %ymm10
+#define T2_XMM %xmm10
+#define T3 %ymm11
+#define T3_XMM %xmm11
+#define T4 %ymm12
+#define T5 %ymm13
+#define T6 %ymm14
+#define T7 %ymm15
+#define KEY %rdi
+#define MESSAGE %rsi
+#define MESSAGE_LEN %rdx
+#define HASH %rcx
+
+.macro _nh_2xstride k0, k1, k2, k3
+
+ // Add message words to key words
+ vpaddd \k0, T3, T0
+ vpaddd \k1, T3, T1
+ vpaddd \k2, T3, T2
+ vpaddd \k3, T3, T3
+
+ // Multiply 32x32 => 64 and accumulate
+ vpshufd $0x10, T0, T4
+ vpshufd $0x32, T0, T0
+ vpshufd $0x10, T1, T5
+ vpshufd $0x32, T1, T1
+ vpshufd $0x10, T2, T6
+ vpshufd $0x32, T2, T2
+ vpshufd $0x10, T3, T7
+ vpshufd $0x32, T3, T3
+ vpmuludq T4, T0, T0
+ vpmuludq T5, T1, T1
+ vpmuludq T6, T2, T2
+ vpmuludq T7, T3, T3
+ vpaddq T0, PASS0_SUMS, PASS0_SUMS
+ vpaddq T1, PASS1_SUMS, PASS1_SUMS
+ vpaddq T2, PASS2_SUMS, PASS2_SUMS
+ vpaddq T3, PASS3_SUMS, PASS3_SUMS
+.endm
+
+/*
+ * void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
+ * __le64 hash[NH_NUM_PASSES])
+ *
+ * It's guaranteed that message_len % 16 == 0.
+ */
+SYM_FUNC_START(nh_avx2)
+
+ vmovdqu 0x00(KEY), K0
+ vmovdqu 0x10(KEY), K1
+ add $0x20, KEY
+ vpxor PASS0_SUMS, PASS0_SUMS, PASS0_SUMS
+ vpxor PASS1_SUMS, PASS1_SUMS, PASS1_SUMS
+ vpxor PASS2_SUMS, PASS2_SUMS, PASS2_SUMS
+ vpxor PASS3_SUMS, PASS3_SUMS, PASS3_SUMS
+
+ sub $0x40, MESSAGE_LEN
+ jl .Lloop4_done
+.Lloop4:
+ vmovdqu (MESSAGE), T3
+ vmovdqu 0x00(KEY), K2
+ vmovdqu 0x10(KEY), K3
+ _nh_2xstride K0, K1, K2, K3
+
+ vmovdqu 0x20(MESSAGE), T3
+ vmovdqu 0x20(KEY), K0
+ vmovdqu 0x30(KEY), K1
+ _nh_2xstride K2, K3, K0, K1
+
+ add $0x40, MESSAGE
+ add $0x40, KEY
+ sub $0x40, MESSAGE_LEN
+ jge .Lloop4
+
+.Lloop4_done:
+ and $0x3f, MESSAGE_LEN
+ jz .Ldone
+
+ cmp $0x20, MESSAGE_LEN
+ jl .Llast
+
+ // 2 or 3 strides remain; do 2 more.
+ vmovdqu (MESSAGE), T3
+ vmovdqu 0x00(KEY), K2
+ vmovdqu 0x10(KEY), K3
+ _nh_2xstride K0, K1, K2, K3
+ add $0x20, MESSAGE
+ add $0x20, KEY
+ sub $0x20, MESSAGE_LEN
+ jz .Ldone
+ vmovdqa K2, K0
+ vmovdqa K3, K1
+.Llast:
+ // Last stride. Zero the high 128 bits of the message and keys so they
+ // don't affect the result when processing them like 2 strides.
+ vmovdqu (MESSAGE), T3_XMM
+ vmovdqa K0_XMM, K0_XMM
+ vmovdqa K1_XMM, K1_XMM
+ vmovdqu 0x00(KEY), K2_XMM
+ vmovdqu 0x10(KEY), K3_XMM
+ _nh_2xstride K0, K1, K2, K3
+
+.Ldone:
+ // Sum the accumulators for each pass, then store the sums to 'hash'
+
+ // PASS0_SUMS is (0A 0B 0C 0D)
+ // PASS1_SUMS is (1A 1B 1C 1D)
+ // PASS2_SUMS is (2A 2B 2C 2D)
+ // PASS3_SUMS is (3A 3B 3C 3D)
+ // We need the horizontal sums:
+ // (0A + 0B + 0C + 0D,
+ // 1A + 1B + 1C + 1D,
+ // 2A + 2B + 2C + 2D,
+ // 3A + 3B + 3C + 3D)
+ //
+
+ vpunpcklqdq PASS1_SUMS, PASS0_SUMS, T0 // T0 = (0A 1A 0C 1C)
+ vpunpckhqdq PASS1_SUMS, PASS0_SUMS, T1 // T1 = (0B 1B 0D 1D)
+ vpunpcklqdq PASS3_SUMS, PASS2_SUMS, T2 // T2 = (2A 3A 2C 3C)
+ vpunpckhqdq PASS3_SUMS, PASS2_SUMS, T3 // T3 = (2B 3B 2D 3D)
+
+ vinserti128 $0x1, T2_XMM, T0, T4 // T4 = (0A 1A 2A 3A)
+ vinserti128 $0x1, T3_XMM, T1, T5 // T5 = (0B 1B 2B 3B)
+ vperm2i128 $0x31, T2, T0, T0 // T0 = (0C 1C 2C 3C)
+ vperm2i128 $0x31, T3, T1, T1 // T1 = (0D 1D 2D 3D)
+
+ vpaddq T5, T4, T4
+ vpaddq T1, T0, T0
+ vpaddq T4, T0, T0
+ vmovdqu T0, (HASH)
+ vzeroupper
+ RET
+SYM_FUNC_END(nh_avx2)
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * NH - ε-almost-universal hash function, x86_64 SSE2 accelerated
+ *
+ * Copyright 2018 Google LLC
+ *
+ * Author: Eric Biggers <ebiggers@google.com>
+ */
+
+#include <linux/linkage.h>
+
+#define PASS0_SUMS %xmm0
+#define PASS1_SUMS %xmm1
+#define PASS2_SUMS %xmm2
+#define PASS3_SUMS %xmm3
+#define K0 %xmm4
+#define K1 %xmm5
+#define K2 %xmm6
+#define K3 %xmm7
+#define T0 %xmm8
+#define T1 %xmm9
+#define T2 %xmm10
+#define T3 %xmm11
+#define T4 %xmm12
+#define T5 %xmm13
+#define T6 %xmm14
+#define T7 %xmm15
+#define KEY %rdi
+#define MESSAGE %rsi
+#define MESSAGE_LEN %rdx
+#define HASH %rcx
+
+.macro _nh_stride k0, k1, k2, k3, offset
+
+ // Load next message stride
+ movdqu \offset(MESSAGE), T1
+
+ // Load next key stride
+ movdqu \offset(KEY), \k3
+
+ // Add message words to key words
+ movdqa T1, T2
+ movdqa T1, T3
+ paddd T1, \k0 // reuse k0 to avoid a move
+ paddd \k1, T1
+ paddd \k2, T2
+ paddd \k3, T3
+
+ // Multiply 32x32 => 64 and accumulate
+ pshufd $0x10, \k0, T4
+ pshufd $0x32, \k0, \k0
+ pshufd $0x10, T1, T5
+ pshufd $0x32, T1, T1
+ pshufd $0x10, T2, T6
+ pshufd $0x32, T2, T2
+ pshufd $0x10, T3, T7
+ pshufd $0x32, T3, T3
+ pmuludq T4, \k0
+ pmuludq T5, T1
+ pmuludq T6, T2
+ pmuludq T7, T3
+ paddq \k0, PASS0_SUMS
+ paddq T1, PASS1_SUMS
+ paddq T2, PASS2_SUMS
+ paddq T3, PASS3_SUMS
+.endm
+
+/*
+ * void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
+ * __le64 hash[NH_NUM_PASSES])
+ *
+ * It's guaranteed that message_len % 16 == 0.
+ */
+SYM_FUNC_START(nh_sse2)
+
+ movdqu 0x00(KEY), K0
+ movdqu 0x10(KEY), K1
+ movdqu 0x20(KEY), K2
+ add $0x30, KEY
+ pxor PASS0_SUMS, PASS0_SUMS
+ pxor PASS1_SUMS, PASS1_SUMS
+ pxor PASS2_SUMS, PASS2_SUMS
+ pxor PASS3_SUMS, PASS3_SUMS
+
+ sub $0x40, MESSAGE_LEN
+ jl .Lloop4_done
+.Lloop4:
+ _nh_stride K0, K1, K2, K3, 0x00
+ _nh_stride K1, K2, K3, K0, 0x10
+ _nh_stride K2, K3, K0, K1, 0x20
+ _nh_stride K3, K0, K1, K2, 0x30
+ add $0x40, KEY
+ add $0x40, MESSAGE
+ sub $0x40, MESSAGE_LEN
+ jge .Lloop4
+
+.Lloop4_done:
+ and $0x3f, MESSAGE_LEN
+ jz .Ldone
+ _nh_stride K0, K1, K2, K3, 0x00
+
+ sub $0x10, MESSAGE_LEN
+ jz .Ldone
+ _nh_stride K1, K2, K3, K0, 0x10
+
+ sub $0x10, MESSAGE_LEN
+ jz .Ldone
+ _nh_stride K2, K3, K0, K1, 0x20
+
+.Ldone:
+ // Sum the accumulators for each pass, then store the sums to 'hash'
+ movdqa PASS0_SUMS, T0
+ movdqa PASS2_SUMS, T1
+ punpcklqdq PASS1_SUMS, T0 // => (PASS0_SUM_A PASS1_SUM_A)
+ punpcklqdq PASS3_SUMS, T1 // => (PASS2_SUM_A PASS3_SUM_A)
+ punpckhqdq PASS1_SUMS, PASS0_SUMS // => (PASS0_SUM_B PASS1_SUM_B)
+ punpckhqdq PASS3_SUMS, PASS2_SUMS // => (PASS2_SUM_B PASS3_SUM_B)
+ paddq PASS0_SUMS, T0
+ paddq PASS2_SUMS, T1
+ movdqu T0, 0x00(HASH)
+ movdqu T1, 0x10(HASH)
+ RET
+SYM_FUNC_END(nh_sse2)
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * x86_64 accelerated implementation of NH
+ *
+ * Copyright 2018 Google LLC
+ */
+
+#include <asm/fpu/api.h>
+#include <linux/static_call.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sse2);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_avx2);
+
+asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
+ __le64 hash[NH_NUM_PASSES]);
+asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
+ __le64 hash[NH_NUM_PASSES]);
+
+static bool nh_arch(const u32 *key, const u8 *message, size_t message_len,
+ __le64 hash[NH_NUM_PASSES])
+{
+ if (message_len >= 64 && static_branch_likely(&have_sse2) &&
+ irq_fpu_usable()) {
+ kernel_fpu_begin();
+ if (static_branch_likely(&have_avx2))
+ nh_avx2(key, message, message_len, hash);
+ else
+ nh_sse2(key, message, message_len, hash);
+ kernel_fpu_end();
+ return true;
+ }
+ return false;
+}
+
+#define nh_mod_init_arch nh_mod_init_arch
+static void nh_mod_init_arch(void)
+{
+ if (boot_cpu_has(X86_FEATURE_XMM2)) {
+ static_branch_enable(&have_sse2);
+ if (boot_cpu_has(X86_FEATURE_AVX2) &&
+ cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
+ NULL))
+ static_branch_enable(&have_avx2);
+ }
+}
projects / linux-2.6-microblaze.git / commitdiff