arch/x86/crypto/aesni-intel_glue.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Support for Intel AES-NI instructions. This file contains glue
   4  * code, the real AES implementation is in intel-aes_asm.S.
   5  *
   6  * Copyright (C) 2008, Intel Corp.
   7  *    Author: Huang Ying <ying.huang@intel.com>
   8  *
   9  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
  10  * interface for 64-bit kernels.
  11  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
  12  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  13  *             Tadeusz Struk (tadeusz.struk@intel.com)
  14  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
  15  *    Copyright (c) 2010, Intel Corporation.
  16  */
  17
  18 #include <linux/hardirq.h>
  19 #include <linux/types.h>
  20 #include <linux/module.h>
  21 #include <linux/err.h>
  22 #include <crypto/algapi.h>
  23 #include <crypto/aes.h>
  24 #include <crypto/ctr.h>
  25 #include <crypto/b128ops.h>
  26 #include <crypto/gcm.h>
  27 #include <crypto/xts.h>
  28 #include <asm/cpu_device_id.h>
  29 #include <asm/simd.h>
  30 #include <crypto/scatterwalk.h>
  31 #include <crypto/internal/aead.h>
  32 #include <crypto/internal/simd.h>
  33 #include <crypto/internal/skcipher.h>
  34 #include <linux/workqueue.h>
  35 #include <linux/spinlock.h>
  36 #ifdef CONFIG_X86_64
  37 #include <asm/crypto/glue_helper.h>
  38 #endif
  39
  40
  41 #define AESNI_ALIGN     16
  42 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
  43 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE - 1))
  44 #define RFC4106_HASH_SUBKEY_SIZE 16
  45 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
  46 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
  47 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
  48
  49 /* This data is stored at the end of the crypto_tfm struct.
  50  * It's a type of per "session" data storage location.
  51  * This needs to be 16 byte aligned.
  52  */
  53 struct aesni_rfc4106_gcm_ctx {
  54         u8 hash_subkey[16] AESNI_ALIGN_ATTR;
  55         struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
  56         u8 nonce[4];
  57 };
  58
  59 struct generic_gcmaes_ctx {
  60         u8 hash_subkey[16] AESNI_ALIGN_ATTR;
  61         struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
  62 };
  63
  64 struct aesni_xts_ctx {
  65         u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
  66         u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
  67 };
  68
  69 #define GCM_BLOCK_LEN 16
  70
  71 struct gcm_context_data {
  72         /* init, update and finalize context data */
  73         u8 aad_hash[GCM_BLOCK_LEN];
  74         u64 aad_length;
  75         u64 in_length;
  76         u8 partial_block_enc_key[GCM_BLOCK_LEN];
  77         u8 orig_IV[GCM_BLOCK_LEN];
  78         u8 current_counter[GCM_BLOCK_LEN];
  79         u64 partial_block_len;
  80         u64 unused;
  81         u8 hash_keys[GCM_BLOCK_LEN * 16];
  82 };
  83
  84 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
  85                              unsigned int key_len);
  86 asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in);
  87 asmlinkage void aesni_dec(const void *ctx, u8 *out, const u8 *in);
  88 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
  89                               const u8 *in, unsigned int len);
  90 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
  91                               const u8 *in, unsigned int len);
  92 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
  93                               const u8 *in, unsigned int len, u8 *iv);
  94 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
  95                               const u8 *in, unsigned int len, u8 *iv);
  96
  97 #define AVX_GEN2_OPTSIZE 640
  98 #define AVX_GEN4_OPTSIZE 4096
  99
 100 #ifdef CONFIG_X86_64
 101
 102 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
 103                               const u8 *in, unsigned int len, u8 *iv);
 104 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
 105                               const u8 *in, unsigned int len, u8 *iv);
 106
 107 asmlinkage void aesni_xts_crypt8(const struct crypto_aes_ctx *ctx, u8 *out,
 108                                  const u8 *in, bool enc, le128 *iv);
 109
 110 /* asmlinkage void aesni_gcm_enc()
 111  * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
 112  * struct gcm_context_data.  May be uninitialized.
 113  * u8 *out, Ciphertext output. Encrypt in-place is allowed.
 114  * const u8 *in, Plaintext input
 115  * unsigned long plaintext_len, Length of data in bytes for encryption.
 116  * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
 117  *         16-byte aligned pointer.
 118  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 119  * const u8 *aad, Additional Authentication Data (AAD)
 120  * unsigned long aad_len, Length of AAD in bytes.
 121  * u8 *auth_tag, Authenticated Tag output.
 122  * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
 123  *          Valid values are 16 (most likely), 12 or 8.
 124  */
 125 asmlinkage void aesni_gcm_enc(void *ctx,
 126                         struct gcm_context_data *gdata, u8 *out,
 127                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 128                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 129                         u8 *auth_tag, unsigned long auth_tag_len);
 130
 131 /* asmlinkage void aesni_gcm_dec()
 132  * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
 133  * struct gcm_context_data.  May be uninitialized.
 134  * u8 *out, Plaintext output. Decrypt in-place is allowed.
 135  * const u8 *in, Ciphertext input
 136  * unsigned long ciphertext_len, Length of data in bytes for decryption.
 137  * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
 138  *         16-byte aligned pointer.
 139  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 140  * const u8 *aad, Additional Authentication Data (AAD)
 141  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
 142  * to be 8 or 12 bytes
 143  * u8 *auth_tag, Authenticated Tag output.
 144  * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
 145  * Valid values are 16 (most likely), 12 or 8.
 146  */
 147 asmlinkage void aesni_gcm_dec(void *ctx,
 148                         struct gcm_context_data *gdata, u8 *out,
 149                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 150                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 151                         u8 *auth_tag, unsigned long auth_tag_len);
 152
 153 /* Scatter / Gather routines, with args similar to above */
 154 asmlinkage void aesni_gcm_init(void *ctx,
 155                                struct gcm_context_data *gdata,
 156                                u8 *iv,
 157                                u8 *hash_subkey, const u8 *aad,
 158                                unsigned long aad_len);
 159 asmlinkage void aesni_gcm_enc_update(void *ctx,
 160                                      struct gcm_context_data *gdata, u8 *out,
 161                                      const u8 *in, unsigned long plaintext_len);
 162 asmlinkage void aesni_gcm_dec_update(void *ctx,
 163                                      struct gcm_context_data *gdata, u8 *out,
 164                                      const u8 *in,
 165                                      unsigned long ciphertext_len);
 166 asmlinkage void aesni_gcm_finalize(void *ctx,
 167                                    struct gcm_context_data *gdata,
 168                                    u8 *auth_tag, unsigned long auth_tag_len);
 169
 170 static const struct aesni_gcm_tfm_s {
 171         void (*init)(void *ctx, struct gcm_context_data *gdata, u8 *iv,
 172                      u8 *hash_subkey, const u8 *aad, unsigned long aad_len);
 173         void (*enc_update)(void *ctx, struct gcm_context_data *gdata, u8 *out,
 174                            const u8 *in, unsigned long plaintext_len);
 175         void (*dec_update)(void *ctx, struct gcm_context_data *gdata, u8 *out,
 176                            const u8 *in, unsigned long ciphertext_len);
 177         void (*finalize)(void *ctx, struct gcm_context_data *gdata,
 178                          u8 *auth_tag, unsigned long auth_tag_len);
 179 } *aesni_gcm_tfm;
 180
 181 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_sse = {
 182         .init = &aesni_gcm_init,
 183         .enc_update = &aesni_gcm_enc_update,
 184         .dec_update = &aesni_gcm_dec_update,
 185         .finalize = &aesni_gcm_finalize,
 186 };
 187
 188 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
 189                 void *keys, u8 *out, unsigned int num_bytes);
 190 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
 191                 void *keys, u8 *out, unsigned int num_bytes);
 192 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
 193                 void *keys, u8 *out, unsigned int num_bytes);
 194 /*
 195  * asmlinkage void aesni_gcm_init_avx_gen2()
 196  * gcm_data *my_ctx_data, context data
 197  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 198  */
 199 asmlinkage void aesni_gcm_init_avx_gen2(void *my_ctx_data,
 200                                         struct gcm_context_data *gdata,
 201                                         u8 *iv,
 202                                         u8 *hash_subkey,
 203                                         const u8 *aad,
 204                                         unsigned long aad_len);
 205
 206 asmlinkage void aesni_gcm_enc_update_avx_gen2(void *ctx,
 207                                      struct gcm_context_data *gdata, u8 *out,
 208                                      const u8 *in, unsigned long plaintext_len);
 209 asmlinkage void aesni_gcm_dec_update_avx_gen2(void *ctx,
 210                                      struct gcm_context_data *gdata, u8 *out,
 211                                      const u8 *in,
 212                                      unsigned long ciphertext_len);
 213 asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx,
 214                                    struct gcm_context_data *gdata,
 215                                    u8 *auth_tag, unsigned long auth_tag_len);
 216
 217 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx,
 218                                 struct gcm_context_data *gdata, u8 *out,
 219                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 220                         const u8 *aad, unsigned long aad_len,
 221                         u8 *auth_tag, unsigned long auth_tag_len);
 222
 223 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx,
 224                                 struct gcm_context_data *gdata, u8 *out,
 225                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 226                         const u8 *aad, unsigned long aad_len,
 227                         u8 *auth_tag, unsigned long auth_tag_len);
 228
 229 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen2 = {
 230         .init = &aesni_gcm_init_avx_gen2,
 231         .enc_update = &aesni_gcm_enc_update_avx_gen2,
 232         .dec_update = &aesni_gcm_dec_update_avx_gen2,
 233         .finalize = &aesni_gcm_finalize_avx_gen2,
 234 };
 235
 236 /*
 237  * asmlinkage void aesni_gcm_init_avx_gen4()
 238  * gcm_data *my_ctx_data, context data
 239  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 240  */
 241 asmlinkage void aesni_gcm_init_avx_gen4(void *my_ctx_data,
 242                                         struct gcm_context_data *gdata,
 243                                         u8 *iv,
 244                                         u8 *hash_subkey,
 245                                         const u8 *aad,
 246                                         unsigned long aad_len);
 247
 248 asmlinkage void aesni_gcm_enc_update_avx_gen4(void *ctx,
 249                                      struct gcm_context_data *gdata, u8 *out,
 250                                      const u8 *in, unsigned long plaintext_len);
 251 asmlinkage void aesni_gcm_dec_update_avx_gen4(void *ctx,
 252                                      struct gcm_context_data *gdata, u8 *out,
 253                                      const u8 *in,
 254                                      unsigned long ciphertext_len);
 255 asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx,
 256                                    struct gcm_context_data *gdata,
 257                                    u8 *auth_tag, unsigned long auth_tag_len);
 258
 259 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx,
 260                                 struct gcm_context_data *gdata, u8 *out,
 261                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 262                         const u8 *aad, unsigned long aad_len,
 263                         u8 *auth_tag, unsigned long auth_tag_len);
 264
 265 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx,
 266                                 struct gcm_context_data *gdata, u8 *out,
 267                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 268                         const u8 *aad, unsigned long aad_len,
 269                         u8 *auth_tag, unsigned long auth_tag_len);
 270
 271 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen4 = {
 272         .init = &aesni_gcm_init_avx_gen4,
 273         .enc_update = &aesni_gcm_enc_update_avx_gen4,
 274         .dec_update = &aesni_gcm_dec_update_avx_gen4,
 275         .finalize = &aesni_gcm_finalize_avx_gen4,
 276 };
 277
 278 static inline struct
 279 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
 280 {
 281         unsigned long align = AESNI_ALIGN;
 282
 283         if (align <= crypto_tfm_ctx_alignment())
 284                 align = 1;
 285         return PTR_ALIGN(crypto_aead_ctx(tfm), align);
 286 }
 287
 288 static inline struct
 289 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
 290 {
 291         unsigned long align = AESNI_ALIGN;
 292
 293         if (align <= crypto_tfm_ctx_alignment())
 294                 align = 1;
 295         return PTR_ALIGN(crypto_aead_ctx(tfm), align);
 296 }
 297 #endif
 298
 299 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
 300 {
 301         unsigned long addr = (unsigned long)raw_ctx;
 302         unsigned long align = AESNI_ALIGN;
 303
 304         if (align <= crypto_tfm_ctx_alignment())
 305                 align = 1;
 306         return (struct crypto_aes_ctx *)ALIGN(addr, align);
 307 }
 308
 309 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
 310                               const u8 *in_key, unsigned int key_len)
 311 {
 312         struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
 313         int err;
 314
 315         if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
 316             key_len != AES_KEYSIZE_256)
 317                 return -EINVAL;
 318
 319         if (!crypto_simd_usable())
 320                 err = aes_expandkey(ctx, in_key, key_len);
 321         else {
 322                 kernel_fpu_begin();
 323                 err = aesni_set_key(ctx, in_key, key_len);
 324                 kernel_fpu_end();
 325         }
 326
 327         return err;
 328 }
 329
 330 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 331                        unsigned int key_len)
 332 {
 333         return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
 334 }
 335
 336 static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 337 {
 338         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 339
 340         if (!crypto_simd_usable()) {
 341                 aes_encrypt(ctx, dst, src);
 342         } else {
 343                 kernel_fpu_begin();
 344                 aesni_enc(ctx, dst, src);
 345                 kernel_fpu_end();
 346         }
 347 }
 348
 349 static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 350 {
 351         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 352
 353         if (!crypto_simd_usable()) {
 354                 aes_decrypt(ctx, dst, src);
 355         } else {
 356                 kernel_fpu_begin();
 357                 aesni_dec(ctx, dst, src);
 358                 kernel_fpu_end();
 359         }
 360 }
 361
 362 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
 363                                  unsigned int len)
 364 {
 365         return aes_set_key_common(crypto_skcipher_tfm(tfm),
 366                                   crypto_skcipher_ctx(tfm), key, len);
 367 }
 368
 369 static int ecb_encrypt(struct skcipher_request *req)
 370 {
 371         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 372         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 373         struct skcipher_walk walk;
 374         unsigned int nbytes;
 375         int err;
 376
 377         err = skcipher_walk_virt(&walk, req, true);
 378
 379         kernel_fpu_begin();
 380         while ((nbytes = walk.nbytes)) {
 381                 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 382                               nbytes & AES_BLOCK_MASK);
 383                 nbytes &= AES_BLOCK_SIZE - 1;
 384                 err = skcipher_walk_done(&walk, nbytes);
 385         }
 386         kernel_fpu_end();
 387
 388         return err;
 389 }
 390
 391 static int ecb_decrypt(struct skcipher_request *req)
 392 {
 393         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 394         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 395         struct skcipher_walk walk;
 396         unsigned int nbytes;
 397         int err;
 398
 399         err = skcipher_walk_virt(&walk, req, true);
 400
 401         kernel_fpu_begin();
 402         while ((nbytes = walk.nbytes)) {
 403                 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 404                               nbytes & AES_BLOCK_MASK);
 405                 nbytes &= AES_BLOCK_SIZE - 1;
 406                 err = skcipher_walk_done(&walk, nbytes);
 407         }
 408         kernel_fpu_end();
 409
 410         return err;
 411 }
 412
 413 static int cbc_encrypt(struct skcipher_request *req)
 414 {
 415         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 416         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 417         struct skcipher_walk walk;
 418         unsigned int nbytes;
 419         int err;
 420
 421         err = skcipher_walk_virt(&walk, req, true);
 422
 423         kernel_fpu_begin();
 424         while ((nbytes = walk.nbytes)) {
 425                 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 426                               nbytes & AES_BLOCK_MASK, walk.iv);
 427                 nbytes &= AES_BLOCK_SIZE - 1;
 428                 err = skcipher_walk_done(&walk, nbytes);
 429         }
 430         kernel_fpu_end();
 431
 432         return err;
 433 }
 434
 435 static int cbc_decrypt(struct skcipher_request *req)
 436 {
 437         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 438         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 439         struct skcipher_walk walk;
 440         unsigned int nbytes;
 441         int err;
 442
 443         err = skcipher_walk_virt(&walk, req, true);
 444
 445         kernel_fpu_begin();
 446         while ((nbytes = walk.nbytes)) {
 447                 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 448                               nbytes & AES_BLOCK_MASK, walk.iv);
 449                 nbytes &= AES_BLOCK_SIZE - 1;
 450                 err = skcipher_walk_done(&walk, nbytes);
 451         }
 452         kernel_fpu_end();
 453
 454         return err;
 455 }
 456
 457 #ifdef CONFIG_X86_64
 458 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
 459                             struct skcipher_walk *walk)
 460 {
 461         u8 *ctrblk = walk->iv;
 462         u8 keystream[AES_BLOCK_SIZE];
 463         u8 *src = walk->src.virt.addr;
 464         u8 *dst = walk->dst.virt.addr;
 465         unsigned int nbytes = walk->nbytes;
 466
 467         aesni_enc(ctx, keystream, ctrblk);
 468         crypto_xor_cpy(dst, keystream, src, nbytes);
 469
 470         crypto_inc(ctrblk, AES_BLOCK_SIZE);
 471 }
 472
 473 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
 474                               const u8 *in, unsigned int len, u8 *iv)
 475 {
 476         /*
 477          * based on key length, override with the by8 version
 478          * of ctr mode encryption/decryption for improved performance
 479          * aes_set_key_common() ensures that key length is one of
 480          * {128,192,256}
 481          */
 482         if (ctx->key_length == AES_KEYSIZE_128)
 483                 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
 484         else if (ctx->key_length == AES_KEYSIZE_192)
 485                 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
 486         else
 487                 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
 488 }
 489
 490 static int ctr_crypt(struct skcipher_request *req)
 491 {
 492         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 493         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 494         struct skcipher_walk walk;
 495         unsigned int nbytes;
 496         int err;
 497
 498         err = skcipher_walk_virt(&walk, req, true);
 499
 500         kernel_fpu_begin();
 501         while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
 502                 aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 503                                       nbytes & AES_BLOCK_MASK, walk.iv);
 504                 nbytes &= AES_BLOCK_SIZE - 1;
 505                 err = skcipher_walk_done(&walk, nbytes);
 506         }
 507         if (walk.nbytes) {
 508                 ctr_crypt_final(ctx, &walk);
 509                 err = skcipher_walk_done(&walk, 0);
 510         }
 511         kernel_fpu_end();
 512
 513         return err;
 514 }
 515
 516 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
 517                             unsigned int keylen)
 518 {
 519         struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 520         int err;
 521
 522         err = xts_verify_key(tfm, key, keylen);
 523         if (err)
 524                 return err;
 525
 526         keylen /= 2;
 527
 528         /* first half of xts-key is for crypt */
 529         err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx,
 530                                  key, keylen);
 531         if (err)
 532                 return err;
 533
 534         /* second half of xts-key is for tweak */
 535         return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx,
 536                                   key + keylen, keylen);
 537 }
 538
 539
 540 static void aesni_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
 541 {
 542         glue_xts_crypt_128bit_one(ctx, dst, src, iv, aesni_enc);
 543 }
 544
 545 static void aesni_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
 546 {
 547         glue_xts_crypt_128bit_one(ctx, dst, src, iv, aesni_dec);
 548 }
 549
 550 static void aesni_xts_enc8(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
 551 {
 552         aesni_xts_crypt8(ctx, dst, src, true, iv);
 553 }
 554
 555 static void aesni_xts_dec8(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
 556 {
 557         aesni_xts_crypt8(ctx, dst, src, false, iv);
 558 }
 559
 560 static const struct common_glue_ctx aesni_enc_xts = {
 561         .num_funcs = 2,
 562         .fpu_blocks_limit = 1,
 563
 564         .funcs = { {
 565                 .num_blocks = 8,
 566                 .fn_u = { .xts = aesni_xts_enc8 }
 567         }, {
 568                 .num_blocks = 1,
 569                 .fn_u = { .xts = aesni_xts_enc }
 570         } }
 571 };
 572
 573 static const struct common_glue_ctx aesni_dec_xts = {
 574         .num_funcs = 2,
 575         .fpu_blocks_limit = 1,
 576
 577         .funcs = { {
 578                 .num_blocks = 8,
 579                 .fn_u = { .xts = aesni_xts_dec8 }
 580         }, {
 581                 .num_blocks = 1,
 582                 .fn_u = { .xts = aesni_xts_dec }
 583         } }
 584 };
 585
 586 static int xts_encrypt(struct skcipher_request *req)
 587 {
 588         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 589         struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 590
 591         return glue_xts_req_128bit(&aesni_enc_xts, req, aesni_enc,
 592                                    aes_ctx(ctx->raw_tweak_ctx),
 593                                    aes_ctx(ctx->raw_crypt_ctx),
 594                                    false);
 595 }
 596
 597 static int xts_decrypt(struct skcipher_request *req)
 598 {
 599         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 600         struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 601
 602         return glue_xts_req_128bit(&aesni_dec_xts, req, aesni_enc,
 603                                    aes_ctx(ctx->raw_tweak_ctx),
 604                                    aes_ctx(ctx->raw_crypt_ctx),
 605                                    true);
 606 }
 607
 608 static int
 609 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
 610 {
 611         struct crypto_aes_ctx ctx;
 612         int ret;
 613
 614         ret = aes_expandkey(&ctx, key, key_len);
 615         if (ret)
 616                 return ret;
 617
 618         /* Clear the data in the hash sub key container to zero.*/
 619         /* We want to cipher all zeros to create the hash sub key. */
 620         memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
 621
 622         aes_encrypt(&ctx, hash_subkey, hash_subkey);
 623
 624         memzero_explicit(&ctx, sizeof(ctx));
 625         return 0;
 626 }
 627
 628 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
 629                                   unsigned int key_len)
 630 {
 631         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
 632
 633         if (key_len < 4)
 634                 return -EINVAL;
 635
 636         /*Account for 4 byte nonce at the end.*/
 637         key_len -= 4;
 638
 639         memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
 640
 641         return aes_set_key_common(crypto_aead_tfm(aead),
 642                                   &ctx->aes_key_expanded, key, key_len) ?:
 643                rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
 644 }
 645
 646 /* This is the Integrity Check Value (aka the authentication tag) length and can
 647  * be 8, 12 or 16 bytes long. */
 648 static int common_rfc4106_set_authsize(struct crypto_aead *aead,
 649                                        unsigned int authsize)
 650 {
 651         switch (authsize) {
 652         case 8:
 653         case 12:
 654         case 16:
 655                 break;
 656         default:
 657                 return -EINVAL;
 658         }
 659
 660         return 0;
 661 }
 662
 663 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
 664                                        unsigned int authsize)
 665 {
 666         switch (authsize) {
 667         case 4:
 668         case 8:
 669         case 12:
 670         case 13:
 671         case 14:
 672         case 15:
 673         case 16:
 674                 break;
 675         default:
 676                 return -EINVAL;
 677         }
 678
 679         return 0;
 680 }
 681
 682 static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req,
 683                               unsigned int assoclen, u8 *hash_subkey,
 684                               u8 *iv, void *aes_ctx)
 685 {
 686         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 687         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
 688         const struct aesni_gcm_tfm_s *gcm_tfm = aesni_gcm_tfm;
 689         struct gcm_context_data data AESNI_ALIGN_ATTR;
 690         struct scatter_walk dst_sg_walk = {};
 691         unsigned long left = req->cryptlen;
 692         unsigned long len, srclen, dstlen;
 693         struct scatter_walk assoc_sg_walk;
 694         struct scatter_walk src_sg_walk;
 695         struct scatterlist src_start[2];
 696         struct scatterlist dst_start[2];
 697         struct scatterlist *src_sg;
 698         struct scatterlist *dst_sg;
 699         u8 *src, *dst, *assoc;
 700         u8 *assocmem = NULL;
 701         u8 authTag[16];
 702
 703         if (!enc)
 704                 left -= auth_tag_len;
 705
 706         if (left < AVX_GEN4_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen4)
 707                 gcm_tfm = &aesni_gcm_tfm_avx_gen2;
 708         if (left < AVX_GEN2_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen2)
 709                 gcm_tfm = &aesni_gcm_tfm_sse;
 710
 711         /* Linearize assoc, if not already linear */
 712         if (req->src->length >= assoclen && req->src->length &&
 713                 (!PageHighMem(sg_page(req->src)) ||
 714                         req->src->offset + req->src->length <= PAGE_SIZE)) {
 715                 scatterwalk_start(&assoc_sg_walk, req->src);
 716                 assoc = scatterwalk_map(&assoc_sg_walk);
 717         } else {
 718                 /* assoc can be any length, so must be on heap */
 719                 assocmem = kmalloc(assoclen, GFP_ATOMIC);
 720                 if (unlikely(!assocmem))
 721                         return -ENOMEM;
 722                 assoc = assocmem;
 723
 724                 scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0);
 725         }
 726
 727         if (left) {
 728                 src_sg = scatterwalk_ffwd(src_start, req->src, req->assoclen);
 729                 scatterwalk_start(&src_sg_walk, src_sg);
 730                 if (req->src != req->dst) {
 731                         dst_sg = scatterwalk_ffwd(dst_start, req->dst,
 732                                                   req->assoclen);
 733                         scatterwalk_start(&dst_sg_walk, dst_sg);
 734                 }
 735         }
 736
 737         kernel_fpu_begin();
 738         gcm_tfm->init(aes_ctx, &data, iv,
 739                 hash_subkey, assoc, assoclen);
 740         if (req->src != req->dst) {
 741                 while (left) {
 742                         src = scatterwalk_map(&src_sg_walk);
 743                         dst = scatterwalk_map(&dst_sg_walk);
 744                         srclen = scatterwalk_clamp(&src_sg_walk, left);
 745                         dstlen = scatterwalk_clamp(&dst_sg_walk, left);
 746                         len = min(srclen, dstlen);
 747                         if (len) {
 748                                 if (enc)
 749                                         gcm_tfm->enc_update(aes_ctx, &data,
 750                                                              dst, src, len);
 751                                 else
 752                                         gcm_tfm->dec_update(aes_ctx, &data,
 753                                                              dst, src, len);
 754                         }
 755                         left -= len;
 756
 757                         scatterwalk_unmap(src);
 758                         scatterwalk_unmap(dst);
 759                         scatterwalk_advance(&src_sg_walk, len);
 760                         scatterwalk_advance(&dst_sg_walk, len);
 761                         scatterwalk_done(&src_sg_walk, 0, left);
 762                         scatterwalk_done(&dst_sg_walk, 1, left);
 763                 }
 764         } else {
 765                 while (left) {
 766                         dst = src = scatterwalk_map(&src_sg_walk);
 767                         len = scatterwalk_clamp(&src_sg_walk, left);
 768                         if (len) {
 769                                 if (enc)
 770                                         gcm_tfm->enc_update(aes_ctx, &data,
 771                                                              src, src, len);
 772                                 else
 773                                         gcm_tfm->dec_update(aes_ctx, &data,
 774                                                              src, src, len);
 775                         }
 776                         left -= len;
 777                         scatterwalk_unmap(src);
 778                         scatterwalk_advance(&src_sg_walk, len);
 779                         scatterwalk_done(&src_sg_walk, 1, left);
 780                 }
 781         }
 782         gcm_tfm->finalize(aes_ctx, &data, authTag, auth_tag_len);
 783         kernel_fpu_end();
 784
 785         if (!assocmem)
 786                 scatterwalk_unmap(assoc);
 787         else
 788                 kfree(assocmem);
 789
 790         if (!enc) {
 791                 u8 authTagMsg[16];
 792
 793                 /* Copy out original authTag */
 794                 scatterwalk_map_and_copy(authTagMsg, req->src,
 795                                          req->assoclen + req->cryptlen -
 796                                          auth_tag_len,
 797                                          auth_tag_len, 0);
 798
 799                 /* Compare generated tag with passed in tag. */
 800                 return crypto_memneq(authTagMsg, authTag, auth_tag_len) ?
 801                         -EBADMSG : 0;
 802         }
 803
 804         /* Copy in the authTag */
 805         scatterwalk_map_and_copy(authTag, req->dst,
 806                                  req->assoclen + req->cryptlen,
 807                                  auth_tag_len, 1);
 808
 809         return 0;
 810 }
 811
 812 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
 813                           u8 *hash_subkey, u8 *iv, void *aes_ctx)
 814 {
 815         return gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv,
 816                                 aes_ctx);
 817 }
 818
 819 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
 820                           u8 *hash_subkey, u8 *iv, void *aes_ctx)
 821 {
 822         return gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv,
 823                                 aes_ctx);
 824 }
 825
 826 static int helper_rfc4106_encrypt(struct aead_request *req)
 827 {
 828         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 829         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 830         void *aes_ctx = &(ctx->aes_key_expanded);
 831         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
 832         unsigned int i;
 833         __be32 counter = cpu_to_be32(1);
 834
 835         /* Assuming we are supporting rfc4106 64-bit extended */
 836         /* sequence numbers We need to have the AAD length equal */
 837         /* to 16 or 20 bytes */
 838         if (unlikely(req->assoclen != 16 && req->assoclen != 20))
 839                 return -EINVAL;
 840
 841         /* IV below built */
 842         for (i = 0; i < 4; i++)
 843                 *(iv+i) = ctx->nonce[i];
 844         for (i = 0; i < 8; i++)
 845                 *(iv+4+i) = req->iv[i];
 846         *((__be32 *)(iv+12)) = counter;
 847
 848         return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
 849                               aes_ctx);
 850 }
 851
 852 static int helper_rfc4106_decrypt(struct aead_request *req)
 853 {
 854         __be32 counter = cpu_to_be32(1);
 855         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 856         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 857         void *aes_ctx = &(ctx->aes_key_expanded);
 858         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
 859         unsigned int i;
 860
 861         if (unlikely(req->assoclen != 16 && req->assoclen != 20))
 862                 return -EINVAL;
 863
 864         /* Assuming we are supporting rfc4106 64-bit extended */
 865         /* sequence numbers We need to have the AAD length */
 866         /* equal to 16 or 20 bytes */
 867
 868         /* IV below built */
 869         for (i = 0; i < 4; i++)
 870                 *(iv+i) = ctx->nonce[i];
 871         for (i = 0; i < 8; i++)
 872                 *(iv+4+i) = req->iv[i];
 873         *((__be32 *)(iv+12)) = counter;
 874
 875         return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
 876                               aes_ctx);
 877 }
 878 #endif
 879
 880 static struct crypto_alg aesni_cipher_alg = {
 881         .cra_name               = "aes",
 882         .cra_driver_name        = "aes-aesni",
 883         .cra_priority           = 300,
 884         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
 885         .cra_blocksize          = AES_BLOCK_SIZE,
 886         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
 887         .cra_module             = THIS_MODULE,
 888         .cra_u  = {
 889                 .cipher = {
 890                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
 891                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
 892                         .cia_setkey             = aes_set_key,
 893                         .cia_encrypt            = aesni_encrypt,
 894                         .cia_decrypt            = aesni_decrypt
 895                 }
 896         }
 897 };
 898
 899 static struct skcipher_alg aesni_skciphers[] = {
 900         {
 901                 .base = {
 902                         .cra_name               = "__ecb(aes)",
 903                         .cra_driver_name        = "__ecb-aes-aesni",
 904                         .cra_priority           = 400,
 905                         .cra_flags              = CRYPTO_ALG_INTERNAL,
 906                         .cra_blocksize          = AES_BLOCK_SIZE,
 907                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
 908                         .cra_module             = THIS_MODULE,
 909                 },
 910                 .min_keysize    = AES_MIN_KEY_SIZE,
 911                 .max_keysize    = AES_MAX_KEY_SIZE,
 912                 .setkey         = aesni_skcipher_setkey,
 913                 .encrypt        = ecb_encrypt,
 914                 .decrypt        = ecb_decrypt,
 915         }, {
 916                 .base = {
 917                         .cra_name               = "__cbc(aes)",
 918                         .cra_driver_name        = "__cbc-aes-aesni",
 919                         .cra_priority           = 400,
 920                         .cra_flags              = CRYPTO_ALG_INTERNAL,
 921                         .cra_blocksize          = AES_BLOCK_SIZE,
 922                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
 923                         .cra_module             = THIS_MODULE,
 924                 },
 925                 .min_keysize    = AES_MIN_KEY_SIZE,
 926                 .max_keysize    = AES_MAX_KEY_SIZE,
 927                 .ivsize         = AES_BLOCK_SIZE,
 928                 .setkey         = aesni_skcipher_setkey,
 929                 .encrypt        = cbc_encrypt,
 930                 .decrypt        = cbc_decrypt,
 931 #ifdef CONFIG_X86_64
 932         }, {
 933                 .base = {
 934                         .cra_name               = "__ctr(aes)",
 935                         .cra_driver_name        = "__ctr-aes-aesni",
 936                         .cra_priority           = 400,
 937                         .cra_flags              = CRYPTO_ALG_INTERNAL,
 938                         .cra_blocksize          = 1,
 939                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
 940                         .cra_module             = THIS_MODULE,
 941                 },
 942                 .min_keysize    = AES_MIN_KEY_SIZE,
 943                 .max_keysize    = AES_MAX_KEY_SIZE,
 944                 .ivsize         = AES_BLOCK_SIZE,
 945                 .chunksize      = AES_BLOCK_SIZE,
 946                 .setkey         = aesni_skcipher_setkey,
 947                 .encrypt        = ctr_crypt,
 948                 .decrypt        = ctr_crypt,
 949         }, {
 950                 .base = {
 951                         .cra_name               = "__xts(aes)",
 952                         .cra_driver_name        = "__xts-aes-aesni",
 953                         .cra_priority           = 401,
 954                         .cra_flags              = CRYPTO_ALG_INTERNAL,
 955                         .cra_blocksize          = AES_BLOCK_SIZE,
 956                         .cra_ctxsize            = XTS_AES_CTX_SIZE,
 957                         .cra_module             = THIS_MODULE,
 958                 },
 959                 .min_keysize    = 2 * AES_MIN_KEY_SIZE,
 960                 .max_keysize    = 2 * AES_MAX_KEY_SIZE,
 961                 .ivsize         = AES_BLOCK_SIZE,
 962                 .setkey         = xts_aesni_setkey,
 963                 .encrypt        = xts_encrypt,
 964                 .decrypt        = xts_decrypt,
 965 #endif
 966         }
 967 };
 968
 969 static
 970 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
 971
 972 #ifdef CONFIG_X86_64
 973 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
 974                                   unsigned int key_len)
 975 {
 976         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead);
 977
 978         return aes_set_key_common(crypto_aead_tfm(aead),
 979                                   &ctx->aes_key_expanded, key, key_len) ?:
 980                rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
 981 }
 982
 983 static int generic_gcmaes_encrypt(struct aead_request *req)
 984 {
 985         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 986         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
 987         void *aes_ctx = &(ctx->aes_key_expanded);
 988         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
 989         __be32 counter = cpu_to_be32(1);
 990
 991         memcpy(iv, req->iv, 12);
 992         *((__be32 *)(iv+12)) = counter;
 993
 994         return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
 995                               aes_ctx);
 996 }
 997
 998 static int generic_gcmaes_decrypt(struct aead_request *req)
 999 {
1000         __be32 counter = cpu_to_be32(1);
1001         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1002         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1003         void *aes_ctx = &(ctx->aes_key_expanded);
1004         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1005
1006         memcpy(iv, req->iv, 12);
1007         *((__be32 *)(iv+12)) = counter;
1008
1009         return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
1010                               aes_ctx);
1011 }
1012
1013 static struct aead_alg aesni_aeads[] = { {
1014         .setkey                 = common_rfc4106_set_key,
1015         .setauthsize            = common_rfc4106_set_authsize,
1016         .encrypt                = helper_rfc4106_encrypt,
1017         .decrypt                = helper_rfc4106_decrypt,
1018         .ivsize                 = GCM_RFC4106_IV_SIZE,
1019         .maxauthsize            = 16,
1020         .base = {
1021                 .cra_name               = "__rfc4106(gcm(aes))",
1022                 .cra_driver_name        = "__rfc4106-gcm-aesni",
1023                 .cra_priority           = 400,
1024                 .cra_flags              = CRYPTO_ALG_INTERNAL,
1025                 .cra_blocksize          = 1,
1026                 .cra_ctxsize            = sizeof(struct aesni_rfc4106_gcm_ctx),
1027                 .cra_alignmask          = AESNI_ALIGN - 1,
1028                 .cra_module             = THIS_MODULE,
1029         },
1030 }, {
1031         .setkey                 = generic_gcmaes_set_key,
1032         .setauthsize            = generic_gcmaes_set_authsize,
1033         .encrypt                = generic_gcmaes_encrypt,
1034         .decrypt                = generic_gcmaes_decrypt,
1035         .ivsize                 = GCM_AES_IV_SIZE,
1036         .maxauthsize            = 16,
1037         .base = {
1038                 .cra_name               = "__gcm(aes)",
1039                 .cra_driver_name        = "__generic-gcm-aesni",
1040                 .cra_priority           = 400,
1041                 .cra_flags              = CRYPTO_ALG_INTERNAL,
1042                 .cra_blocksize          = 1,
1043                 .cra_ctxsize            = sizeof(struct generic_gcmaes_ctx),
1044                 .cra_alignmask          = AESNI_ALIGN - 1,
1045                 .cra_module             = THIS_MODULE,
1046         },
1047 } };
1048 #else
1049 static struct aead_alg aesni_aeads[0];
1050 #endif
1051
1052 static struct simd_aead_alg *aesni_simd_aeads[ARRAY_SIZE(aesni_aeads)];
1053
1054 static const struct x86_cpu_id aesni_cpu_id[] = {
1055         X86_MATCH_FEATURE(X86_FEATURE_AES, NULL),
1056         {}
1057 };
1058 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1059
1060 static int __init aesni_init(void)
1061 {
1062         int err;
1063
1064         if (!x86_match_cpu(aesni_cpu_id))
1065                 return -ENODEV;
1066 #ifdef CONFIG_X86_64
1067         if (boot_cpu_has(X86_FEATURE_AVX2)) {
1068                 pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1069                 aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen4;
1070         } else
1071         if (boot_cpu_has(X86_FEATURE_AVX)) {
1072                 pr_info("AVX version of gcm_enc/dec engaged.\n");
1073                 aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen2;
1074         } else {
1075                 pr_info("SSE version of gcm_enc/dec engaged.\n");
1076                 aesni_gcm_tfm = &aesni_gcm_tfm_sse;
1077         }
1078         aesni_ctr_enc_tfm = aesni_ctr_enc;
1079         if (boot_cpu_has(X86_FEATURE_AVX)) {
1080                 /* optimize performance of ctr mode encryption transform */
1081                 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
1082                 pr_info("AES CTR mode by8 optimization enabled\n");
1083         }
1084 #endif
1085
1086         err = crypto_register_alg(&aesni_cipher_alg);
1087         if (err)
1088                 return err;
1089
1090         err = simd_register_skciphers_compat(aesni_skciphers,
1091                                              ARRAY_SIZE(aesni_skciphers),
1092                                              aesni_simd_skciphers);
1093         if (err)
1094                 goto unregister_cipher;
1095
1096         err = simd_register_aeads_compat(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1097                                          aesni_simd_aeads);
1098         if (err)
1099                 goto unregister_skciphers;
1100
1101         return 0;
1102
1103 unregister_skciphers:
1104         simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1105                                   aesni_simd_skciphers);
1106 unregister_cipher:
1107         crypto_unregister_alg(&aesni_cipher_alg);
1108         return err;
1109 }
1110
1111 static void __exit aesni_exit(void)
1112 {
1113         simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1114                               aesni_simd_aeads);
1115         simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1116                                   aesni_simd_skciphers);
1117         crypto_unregister_alg(&aesni_cipher_alg);
1118 }
1119
1120 late_initcall(aesni_init);
1121 module_exit(aesni_exit);
1122
1123 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1124 MODULE_LICENSE("GPL");
1125 MODULE_ALIAS_CRYPTO("aes");