arch/x86/crypto/camellia_aesni_avx2_glue.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Glue Code for x86_64/AVX2/AES-NI assembler optimized version of Camellia
   4  *
   5  * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
   6  */
   7
   8 #include <asm/crypto/camellia.h>
   9 #include <asm/crypto/glue_helper.h>
  10 #include <crypto/algapi.h>
  11 #include <crypto/internal/simd.h>
  12 #include <linux/crypto.h>
  13 #include <linux/err.h>
  14 #include <linux/module.h>
  15 #include <linux/types.h>
  16
  17 #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16
  18 #define CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS 32
  19
  20 /* 32-way AVX2/AES-NI parallel cipher functions */
  21 asmlinkage void camellia_ecb_enc_32way(const void *ctx, u8 *dst, const u8 *src);
  22 asmlinkage void camellia_ecb_dec_32way(const void *ctx, u8 *dst, const u8 *src);
  23
  24 asmlinkage void camellia_cbc_dec_32way(const void *ctx, u8 *dst, const u8 *src);
  25
  26 static const struct common_glue_ctx camellia_enc = {
  27         .num_funcs = 4,
  28         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  29
  30         .funcs = { {
  31                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
  32                 .fn_u = { .ecb = camellia_ecb_enc_32way }
  33         }, {
  34                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  35                 .fn_u = { .ecb = camellia_ecb_enc_16way }
  36         }, {
  37                 .num_blocks = 2,
  38                 .fn_u = { .ecb = camellia_enc_blk_2way }
  39         }, {
  40                 .num_blocks = 1,
  41                 .fn_u = { .ecb = camellia_enc_blk }
  42         } }
  43 };
  44
  45 static const struct common_glue_ctx camellia_dec = {
  46         .num_funcs = 4,
  47         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  48
  49         .funcs = { {
  50                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
  51                 .fn_u = { .ecb = camellia_ecb_dec_32way }
  52         }, {
  53                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  54                 .fn_u = { .ecb = camellia_ecb_dec_16way }
  55         }, {
  56                 .num_blocks = 2,
  57                 .fn_u = { .ecb = camellia_dec_blk_2way }
  58         }, {
  59                 .num_blocks = 1,
  60                 .fn_u = { .ecb = camellia_dec_blk }
  61         } }
  62 };
  63
  64 static const struct common_glue_ctx camellia_dec_cbc = {
  65         .num_funcs = 4,
  66         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  67
  68         .funcs = { {
  69                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
  70                 .fn_u = { .cbc = camellia_cbc_dec_32way }
  71         }, {
  72                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  73                 .fn_u = { .cbc = camellia_cbc_dec_16way }
  74         }, {
  75                 .num_blocks = 2,
  76                 .fn_u = { .cbc = camellia_decrypt_cbc_2way }
  77         }, {
  78                 .num_blocks = 1,
  79                 .fn_u = { .cbc = camellia_dec_blk }
  80         } }
  81 };
  82
  83 static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key,
  84                            unsigned int keylen)
  85 {
  86         return __camellia_setkey(crypto_skcipher_ctx(tfm), key, keylen);
  87 }
  88
  89 static int ecb_encrypt(struct skcipher_request *req)
  90 {
  91         return glue_ecb_req_128bit(&camellia_enc, req);
  92 }
  93
  94 static int ecb_decrypt(struct skcipher_request *req)
  95 {
  96         return glue_ecb_req_128bit(&camellia_dec, req);
  97 }
  98
  99 static int cbc_encrypt(struct skcipher_request *req)
 100 {
 101         return glue_cbc_encrypt_req_128bit(camellia_enc_blk, req);
 102 }
 103
 104 static int cbc_decrypt(struct skcipher_request *req)
 105 {
 106         return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req);
 107 }
 108
 109 static struct skcipher_alg camellia_algs[] = {
 110         {
 111                 .base.cra_name          = "__ecb(camellia)",
 112                 .base.cra_driver_name   = "__ecb-camellia-aesni-avx2",
 113                 .base.cra_priority      = 500,
 114                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 115                 .base.cra_blocksize     = CAMELLIA_BLOCK_SIZE,
 116                 .base.cra_ctxsize       = sizeof(struct camellia_ctx),
 117                 .base.cra_module        = THIS_MODULE,
 118                 .min_keysize            = CAMELLIA_MIN_KEY_SIZE,
 119                 .max_keysize            = CAMELLIA_MAX_KEY_SIZE,
 120                 .setkey                 = camellia_setkey,
 121                 .encrypt                = ecb_encrypt,
 122                 .decrypt                = ecb_decrypt,
 123         }, {
 124                 .base.cra_name          = "__cbc(camellia)",
 125                 .base.cra_driver_name   = "__cbc-camellia-aesni-avx2",
 126                 .base.cra_priority      = 500,
 127                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 128                 .base.cra_blocksize     = CAMELLIA_BLOCK_SIZE,
 129                 .base.cra_ctxsize       = sizeof(struct camellia_ctx),
 130                 .base.cra_module        = THIS_MODULE,
 131                 .min_keysize            = CAMELLIA_MIN_KEY_SIZE,
 132                 .max_keysize            = CAMELLIA_MAX_KEY_SIZE,
 133                 .ivsize                 = CAMELLIA_BLOCK_SIZE,
 134                 .setkey                 = camellia_setkey,
 135                 .encrypt                = cbc_encrypt,
 136                 .decrypt                = cbc_decrypt,
 137         },
 138 };
 139
 140 static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)];
 141
 142 static int __init camellia_aesni_init(void)
 143 {
 144         const char *feature_name;
 145
 146         if (!boot_cpu_has(X86_FEATURE_AVX) ||
 147             !boot_cpu_has(X86_FEATURE_AVX2) ||
 148             !boot_cpu_has(X86_FEATURE_AES) ||
 149             !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
 150                 pr_info("AVX2 or AES-NI instructions are not detected.\n");
 151                 return -ENODEV;
 152         }
 153
 154         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
 155                                 &feature_name)) {
 156                 pr_info("CPU feature '%s' is not supported.\n", feature_name);
 157                 return -ENODEV;
 158         }
 159
 160         return simd_register_skciphers_compat(camellia_algs,
 161                                               ARRAY_SIZE(camellia_algs),
 162                                               camellia_simd_algs);
 163 }
 164
 165 static void __exit camellia_aesni_fini(void)
 166 {
 167         simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs),
 168                                   camellia_simd_algs);
 169 }
 170
 171 module_init(camellia_aesni_init);
 172 module_exit(camellia_aesni_fini);
 173
 174 MODULE_LICENSE("GPL");
 175 MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX2 optimized");
 176 MODULE_ALIAS_CRYPTO("camellia");
 177 MODULE_ALIAS_CRYPTO("camellia-asm");