arch/x86/crypto/camellia_aesni_avx_glue.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Glue Code for x86_64/AVX/AES-NI assembler optimized version of Camellia
   4  *
   5  * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
   6  */
   7
   8 #include <asm/crypto/camellia.h>
   9 #include <asm/crypto/glue_helper.h>
  10 #include <crypto/algapi.h>
  11 #include <crypto/internal/simd.h>
  12 #include <crypto/xts.h>
  13 #include <linux/crypto.h>
  14 #include <linux/err.h>
  15 #include <linux/module.h>
  16 #include <linux/types.h>
  17
  18 #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16
  19
  20 /* 16-way parallel cipher functions (avx/aes-ni) */
  21 asmlinkage void camellia_ecb_enc_16way(const void *ctx, u8 *dst, const u8 *src);
  22 EXPORT_SYMBOL_GPL(camellia_ecb_enc_16way);
  23
  24 asmlinkage void camellia_ecb_dec_16way(const void *ctx, u8 *dst, const u8 *src);
  25 EXPORT_SYMBOL_GPL(camellia_ecb_dec_16way);
  26
  27 asmlinkage void camellia_cbc_dec_16way(const void *ctx, u8 *dst, const u8 *src);
  28 EXPORT_SYMBOL_GPL(camellia_cbc_dec_16way);
  29
  30 asmlinkage void camellia_ctr_16way(const void *ctx, u8 *dst, const u8 *src,
  31                                    le128 *iv);
  32 EXPORT_SYMBOL_GPL(camellia_ctr_16way);
  33
  34 asmlinkage void camellia_xts_enc_16way(const void *ctx, u8 *dst, const u8 *src,
  35                                        le128 *iv);
  36 EXPORT_SYMBOL_GPL(camellia_xts_enc_16way);
  37
  38 asmlinkage void camellia_xts_dec_16way(const void *ctx, u8 *dst, const u8 *src,
  39                                        le128 *iv);
  40 EXPORT_SYMBOL_GPL(camellia_xts_dec_16way);
  41
  42 void camellia_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
  43 {
  44         glue_xts_crypt_128bit_one(ctx, dst, src, iv, camellia_enc_blk);
  45 }
  46 EXPORT_SYMBOL_GPL(camellia_xts_enc);
  47
  48 void camellia_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
  49 {
  50         glue_xts_crypt_128bit_one(ctx, dst, src, iv, camellia_dec_blk);
  51 }
  52 EXPORT_SYMBOL_GPL(camellia_xts_dec);
  53
  54 static const struct common_glue_ctx camellia_enc = {
  55         .num_funcs = 3,
  56         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  57
  58         .funcs = { {
  59                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  60                 .fn_u = { .ecb = camellia_ecb_enc_16way }
  61         }, {
  62                 .num_blocks = 2,
  63                 .fn_u = { .ecb = camellia_enc_blk_2way }
  64         }, {
  65                 .num_blocks = 1,
  66                 .fn_u = { .ecb = camellia_enc_blk }
  67         } }
  68 };
  69
  70 static const struct common_glue_ctx camellia_ctr = {
  71         .num_funcs = 3,
  72         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  73
  74         .funcs = { {
  75                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  76                 .fn_u = { .ctr = camellia_ctr_16way }
  77         }, {
  78                 .num_blocks = 2,
  79                 .fn_u = { .ctr = camellia_crypt_ctr_2way }
  80         }, {
  81                 .num_blocks = 1,
  82                 .fn_u = { .ctr = camellia_crypt_ctr }
  83         } }
  84 };
  85
  86 static const struct common_glue_ctx camellia_enc_xts = {
  87         .num_funcs = 2,
  88         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  89
  90         .funcs = { {
  91                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  92                 .fn_u = { .xts = camellia_xts_enc_16way }
  93         }, {
  94                 .num_blocks = 1,
  95                 .fn_u = { .xts = camellia_xts_enc }
  96         } }
  97 };
  98
  99 static const struct common_glue_ctx camellia_dec = {
 100         .num_funcs = 3,
 101         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 102
 103         .funcs = { {
 104                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 105                 .fn_u = { .ecb = camellia_ecb_dec_16way }
 106         }, {
 107                 .num_blocks = 2,
 108                 .fn_u = { .ecb = camellia_dec_blk_2way }
 109         }, {
 110                 .num_blocks = 1,
 111                 .fn_u = { .ecb = camellia_dec_blk }
 112         } }
 113 };
 114
 115 static const struct common_glue_ctx camellia_dec_cbc = {
 116         .num_funcs = 3,
 117         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 118
 119         .funcs = { {
 120                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 121                 .fn_u = { .cbc = camellia_cbc_dec_16way }
 122         }, {
 123                 .num_blocks = 2,
 124                 .fn_u = { .cbc = camellia_decrypt_cbc_2way }
 125         }, {
 126                 .num_blocks = 1,
 127                 .fn_u = { .cbc = camellia_dec_blk }
 128         } }
 129 };
 130
 131 static const struct common_glue_ctx camellia_dec_xts = {
 132         .num_funcs = 2,
 133         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 134
 135         .funcs = { {
 136                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 137                 .fn_u = { .xts = camellia_xts_dec_16way }
 138         }, {
 139                 .num_blocks = 1,
 140                 .fn_u = { .xts = camellia_xts_dec }
 141         } }
 142 };
 143
 144 static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key,
 145                            unsigned int keylen)
 146 {
 147         return __camellia_setkey(crypto_skcipher_ctx(tfm), key, keylen);
 148 }
 149
 150 static int ecb_encrypt(struct skcipher_request *req)
 151 {
 152         return glue_ecb_req_128bit(&camellia_enc, req);
 153 }
 154
 155 static int ecb_decrypt(struct skcipher_request *req)
 156 {
 157         return glue_ecb_req_128bit(&camellia_dec, req);
 158 }
 159
 160 static int cbc_encrypt(struct skcipher_request *req)
 161 {
 162         return glue_cbc_encrypt_req_128bit(camellia_enc_blk, req);
 163 }
 164
 165 static int cbc_decrypt(struct skcipher_request *req)
 166 {
 167         return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req);
 168 }
 169
 170 static int ctr_crypt(struct skcipher_request *req)
 171 {
 172         return glue_ctr_req_128bit(&camellia_ctr, req);
 173 }
 174
 175 int xts_camellia_setkey(struct crypto_skcipher *tfm, const u8 *key,
 176                         unsigned int keylen)
 177 {
 178         struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 179         int err;
 180
 181         err = xts_verify_key(tfm, key, keylen);
 182         if (err)
 183                 return err;
 184
 185         /* first half of xts-key is for crypt */
 186         err = __camellia_setkey(&ctx->crypt_ctx, key, keylen / 2);
 187         if (err)
 188                 return err;
 189
 190         /* second half of xts-key is for tweak */
 191         return __camellia_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2);
 192 }
 193 EXPORT_SYMBOL_GPL(xts_camellia_setkey);
 194
 195 static int xts_encrypt(struct skcipher_request *req)
 196 {
 197         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 198         struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 199
 200         return glue_xts_req_128bit(&camellia_enc_xts, req, camellia_enc_blk,
 201                                    &ctx->tweak_ctx, &ctx->crypt_ctx, false);
 202 }
 203
 204 static int xts_decrypt(struct skcipher_request *req)
 205 {
 206         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 207         struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 208
 209         return glue_xts_req_128bit(&camellia_dec_xts, req, camellia_enc_blk,
 210                                    &ctx->tweak_ctx, &ctx->crypt_ctx, true);
 211 }
 212
 213 static struct skcipher_alg camellia_algs[] = {
 214         {
 215                 .base.cra_name          = "__ecb(camellia)",
 216                 .base.cra_driver_name   = "__ecb-camellia-aesni",
 217                 .base.cra_priority      = 400,
 218                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 219                 .base.cra_blocksize     = CAMELLIA_BLOCK_SIZE,
 220                 .base.cra_ctxsize       = sizeof(struct camellia_ctx),
 221                 .base.cra_module        = THIS_MODULE,
 222                 .min_keysize            = CAMELLIA_MIN_KEY_SIZE,
 223                 .max_keysize            = CAMELLIA_MAX_KEY_SIZE,
 224                 .setkey                 = camellia_setkey,
 225                 .encrypt                = ecb_encrypt,
 226                 .decrypt                = ecb_decrypt,
 227         }, {
 228                 .base.cra_name          = "__cbc(camellia)",
 229                 .base.cra_driver_name   = "__cbc-camellia-aesni",
 230                 .base.cra_priority      = 400,
 231                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 232                 .base.cra_blocksize     = CAMELLIA_BLOCK_SIZE,
 233                 .base.cra_ctxsize       = sizeof(struct camellia_ctx),
 234                 .base.cra_module        = THIS_MODULE,
 235                 .min_keysize            = CAMELLIA_MIN_KEY_SIZE,
 236                 .max_keysize            = CAMELLIA_MAX_KEY_SIZE,
 237                 .ivsize                 = CAMELLIA_BLOCK_SIZE,
 238                 .setkey                 = camellia_setkey,
 239                 .encrypt                = cbc_encrypt,
 240                 .decrypt                = cbc_decrypt,
 241         }, {
 242                 .base.cra_name          = "__ctr(camellia)",
 243                 .base.cra_driver_name   = "__ctr-camellia-aesni",
 244                 .base.cra_priority      = 400,
 245                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 246                 .base.cra_blocksize     = 1,
 247                 .base.cra_ctxsize       = sizeof(struct camellia_ctx),
 248                 .base.cra_module        = THIS_MODULE,
 249                 .min_keysize            = CAMELLIA_MIN_KEY_SIZE,
 250                 .max_keysize            = CAMELLIA_MAX_KEY_SIZE,
 251                 .ivsize                 = CAMELLIA_BLOCK_SIZE,
 252                 .chunksize              = CAMELLIA_BLOCK_SIZE,
 253                 .setkey                 = camellia_setkey,
 254                 .encrypt                = ctr_crypt,
 255                 .decrypt                = ctr_crypt,
 256         }, {
 257                 .base.cra_name          = "__xts(camellia)",
 258                 .base.cra_driver_name   = "__xts-camellia-aesni",
 259                 .base.cra_priority      = 400,
 260                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 261                 .base.cra_blocksize     = CAMELLIA_BLOCK_SIZE,
 262                 .base.cra_ctxsize       = sizeof(struct camellia_xts_ctx),
 263                 .base.cra_module        = THIS_MODULE,
 264                 .min_keysize            = 2 * CAMELLIA_MIN_KEY_SIZE,
 265                 .max_keysize            = 2 * CAMELLIA_MAX_KEY_SIZE,
 266                 .ivsize                 = CAMELLIA_BLOCK_SIZE,
 267                 .setkey                 = xts_camellia_setkey,
 268                 .encrypt                = xts_encrypt,
 269                 .decrypt                = xts_decrypt,
 270         },
 271 };
 272
 273 static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)];
 274
 275 static int __init camellia_aesni_init(void)
 276 {
 277         const char *feature_name;
 278
 279         if (!boot_cpu_has(X86_FEATURE_AVX) ||
 280             !boot_cpu_has(X86_FEATURE_AES) ||
 281             !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
 282                 pr_info("AVX or AES-NI instructions are not detected.\n");
 283                 return -ENODEV;
 284         }
 285
 286         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
 287                                 &feature_name)) {
 288                 pr_info("CPU feature '%s' is not supported.\n", feature_name);
 289                 return -ENODEV;
 290         }
 291
 292         return simd_register_skciphers_compat(camellia_algs,
 293                                               ARRAY_SIZE(camellia_algs),
 294                                               camellia_simd_algs);
 295 }
 296
 297 static void __exit camellia_aesni_fini(void)
 298 {
 299         simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs),
 300                                   camellia_simd_algs);
 301 }
 302
 303 module_init(camellia_aesni_init);
 304 module_exit(camellia_aesni_fini);
 305
 306 MODULE_LICENSE("GPL");
 307 MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX optimized");
 308 MODULE_ALIAS_CRYPTO("camellia");
 309 MODULE_ALIAS_CRYPTO("camellia-asm");