arch/x86/crypto/blake2s-glue.c

   1 // SPDX-License-Identifier: GPL-2.0 OR MIT
   2 /*
   3  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
   4  */
   5
   6 #include <crypto/internal/blake2s.h>
   7 #include <crypto/internal/simd.h>
   8 #include <crypto/internal/hash.h>
   9
  10 #include <linux/types.h>
  11 #include <linux/jump_label.h>
  12 #include <linux/kernel.h>
  13 #include <linux/module.h>
  14 #include <linux/sizes.h>
  15
  16 #include <asm/cpufeature.h>
  17 #include <asm/fpu/api.h>
  18 #include <asm/processor.h>
  19 #include <asm/simd.h>
  20
  21 asmlinkage void blake2s_compress_ssse3(struct blake2s_state *state,
  22                                        const u8 *block, const size_t nblocks,
  23                                        const u32 inc);
  24 asmlinkage void blake2s_compress_avx512(struct blake2s_state *state,
  25                                         const u8 *block, const size_t nblocks,
  26                                         const u32 inc);
  27
  28 static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_ssse3);
  29 static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_avx512);
  30
  31 void blake2s_compress_arch(struct blake2s_state *state,
  32                            const u8 *block, size_t nblocks,
  33                            const u32 inc)
  34 {
  35         /* SIMD disables preemption, so relax after processing each page. */
  36         BUILD_BUG_ON(SZ_4K / BLAKE2S_BLOCK_SIZE < 8);
  37
  38         if (!static_branch_likely(&blake2s_use_ssse3) || !crypto_simd_usable()) {
  39                 blake2s_compress_generic(state, block, nblocks, inc);
  40                 return;
  41         }
  42
  43         do {
  44                 const size_t blocks = min_t(size_t, nblocks,
  45                                             SZ_4K / BLAKE2S_BLOCK_SIZE);
  46
  47                 kernel_fpu_begin();
  48                 if (IS_ENABLED(CONFIG_AS_AVX512) &&
  49                     static_branch_likely(&blake2s_use_avx512))
  50                         blake2s_compress_avx512(state, block, blocks, inc);
  51                 else
  52                         blake2s_compress_ssse3(state, block, blocks, inc);
  53                 kernel_fpu_end();
  54
  55                 nblocks -= blocks;
  56                 block += blocks * BLAKE2S_BLOCK_SIZE;
  57         } while (nblocks);
  58 }
  59 EXPORT_SYMBOL(blake2s_compress_arch);
  60
  61 static int crypto_blake2s_update_x86(struct shash_desc *desc,
  62                                      const u8 *in, unsigned int inlen)
  63 {
  64         return crypto_blake2s_update(desc, in, inlen, blake2s_compress_arch);
  65 }
  66
  67 static int crypto_blake2s_final_x86(struct shash_desc *desc, u8 *out)
  68 {
  69         return crypto_blake2s_final(desc, out, blake2s_compress_arch);
  70 }
  71
  72 #define BLAKE2S_ALG(name, driver_name, digest_size)                     \
  73         {                                                               \
  74                 .base.cra_name          = name,                         \
  75                 .base.cra_driver_name   = driver_name,                  \
  76                 .base.cra_priority      = 200,                          \
  77                 .base.cra_flags         = CRYPTO_ALG_OPTIONAL_KEY,      \
  78                 .base.cra_blocksize     = BLAKE2S_BLOCK_SIZE,           \
  79                 .base.cra_ctxsize       = sizeof(struct blake2s_tfm_ctx), \
  80                 .base.cra_module        = THIS_MODULE,                  \
  81                 .digestsize             = digest_size,                  \
  82                 .setkey                 = crypto_blake2s_setkey,        \
  83                 .init                   = crypto_blake2s_init,          \
  84                 .update                 = crypto_blake2s_update_x86,    \
  85                 .final                  = crypto_blake2s_final_x86,     \
  86                 .descsize               = sizeof(struct blake2s_state), \
  87         }
  88
  89 static struct shash_alg blake2s_algs[] = {
  90         BLAKE2S_ALG("blake2s-128", "blake2s-128-x86", BLAKE2S_128_HASH_SIZE),
  91         BLAKE2S_ALG("blake2s-160", "blake2s-160-x86", BLAKE2S_160_HASH_SIZE),
  92         BLAKE2S_ALG("blake2s-224", "blake2s-224-x86", BLAKE2S_224_HASH_SIZE),
  93         BLAKE2S_ALG("blake2s-256", "blake2s-256-x86", BLAKE2S_256_HASH_SIZE),
  94 };
  95
  96 static int __init blake2s_mod_init(void)
  97 {
  98         if (!boot_cpu_has(X86_FEATURE_SSSE3))
  99                 return 0;
 100
 101         static_branch_enable(&blake2s_use_ssse3);
 102
 103         if (IS_ENABLED(CONFIG_AS_AVX512) &&
 104             boot_cpu_has(X86_FEATURE_AVX) &&
 105             boot_cpu_has(X86_FEATURE_AVX2) &&
 106             boot_cpu_has(X86_FEATURE_AVX512F) &&
 107             boot_cpu_has(X86_FEATURE_AVX512VL) &&
 108             cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM |
 109                               XFEATURE_MASK_AVX512, NULL))
 110                 static_branch_enable(&blake2s_use_avx512);
 111
 112         return IS_REACHABLE(CONFIG_CRYPTO_HASH) ?
 113                 crypto_register_shashes(blake2s_algs,
 114                                         ARRAY_SIZE(blake2s_algs)) : 0;
 115 }
 116
 117 static void __exit blake2s_mod_exit(void)
 118 {
 119         if (IS_REACHABLE(CONFIG_CRYPTO_HASH) && boot_cpu_has(X86_FEATURE_SSSE3))
 120                 crypto_unregister_shashes(blake2s_algs, ARRAY_SIZE(blake2s_algs));
 121 }
 122
 123 module_init(blake2s_mod_init);
 124 module_exit(blake2s_mod_exit);
 125
 126 MODULE_ALIAS_CRYPTO("blake2s-128");
 127 MODULE_ALIAS_CRYPTO("blake2s-128-x86");
 128 MODULE_ALIAS_CRYPTO("blake2s-160");
 129 MODULE_ALIAS_CRYPTO("blake2s-160-x86");
 130 MODULE_ALIAS_CRYPTO("blake2s-224");
 131 MODULE_ALIAS_CRYPTO("blake2s-224-x86");
 132 MODULE_ALIAS_CRYPTO("blake2s-256");
 133 MODULE_ALIAS_CRYPTO("blake2s-256-x86");
 134 MODULE_LICENSE("GPL v2");