arch/arm64/crypto/aes-glue.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES
   4  *
   5  * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
   6  */
   7
   8 #include <asm/neon.h>
   9 #include <asm/hwcap.h>
  10 #include <asm/simd.h>
  11 #include <crypto/aes.h>
  12 #include <crypto/ctr.h>
  13 #include <crypto/sha2.h>
  14 #include <crypto/internal/hash.h>
  15 #include <crypto/internal/simd.h>
  16 #include <crypto/internal/skcipher.h>
  17 #include <crypto/scatterwalk.h>
  18 #include <linux/module.h>
  19 #include <linux/cpufeature.h>
  20 #include <crypto/xts.h>
  21
  22 #include "aes-ce-setkey.h"
  23
  24 #ifdef USE_V8_CRYPTO_EXTENSIONS
  25 #define MODE                    "ce"
  26 #define PRIO                    300
  27 #define aes_expandkey           ce_aes_expandkey
  28 #define aes_ecb_encrypt         ce_aes_ecb_encrypt
  29 #define aes_ecb_decrypt         ce_aes_ecb_decrypt
  30 #define aes_cbc_encrypt         ce_aes_cbc_encrypt
  31 #define aes_cbc_decrypt         ce_aes_cbc_decrypt
  32 #define aes_cbc_cts_encrypt     ce_aes_cbc_cts_encrypt
  33 #define aes_cbc_cts_decrypt     ce_aes_cbc_cts_decrypt
  34 #define aes_essiv_cbc_encrypt   ce_aes_essiv_cbc_encrypt
  35 #define aes_essiv_cbc_decrypt   ce_aes_essiv_cbc_decrypt
  36 #define aes_ctr_encrypt         ce_aes_ctr_encrypt
  37 #define aes_xctr_encrypt        ce_aes_xctr_encrypt
  38 #define aes_xts_encrypt         ce_aes_xts_encrypt
  39 #define aes_xts_decrypt         ce_aes_xts_decrypt
  40 #define aes_mac_update          ce_aes_mac_update
  41 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS/XCTR using ARMv8 Crypto Extensions");
  42 #else
  43 #define MODE                    "neon"
  44 #define PRIO                    200
  45 #define aes_ecb_encrypt         neon_aes_ecb_encrypt
  46 #define aes_ecb_decrypt         neon_aes_ecb_decrypt
  47 #define aes_cbc_encrypt         neon_aes_cbc_encrypt
  48 #define aes_cbc_decrypt         neon_aes_cbc_decrypt
  49 #define aes_cbc_cts_encrypt     neon_aes_cbc_cts_encrypt
  50 #define aes_cbc_cts_decrypt     neon_aes_cbc_cts_decrypt
  51 #define aes_essiv_cbc_encrypt   neon_aes_essiv_cbc_encrypt
  52 #define aes_essiv_cbc_decrypt   neon_aes_essiv_cbc_decrypt
  53 #define aes_ctr_encrypt         neon_aes_ctr_encrypt
  54 #define aes_xctr_encrypt        neon_aes_xctr_encrypt
  55 #define aes_xts_encrypt         neon_aes_xts_encrypt
  56 #define aes_xts_decrypt         neon_aes_xts_decrypt
  57 #define aes_mac_update          neon_aes_mac_update
  58 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS/XCTR using ARMv8 NEON");
  59 #endif
  60 #if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS)
  61 MODULE_ALIAS_CRYPTO("ecb(aes)");
  62 MODULE_ALIAS_CRYPTO("cbc(aes)");
  63 MODULE_ALIAS_CRYPTO("ctr(aes)");
  64 MODULE_ALIAS_CRYPTO("xts(aes)");
  65 MODULE_ALIAS_CRYPTO("xctr(aes)");
  66 #endif
  67 MODULE_ALIAS_CRYPTO("cts(cbc(aes))");
  68 MODULE_ALIAS_CRYPTO("essiv(cbc(aes),sha256)");
  69 MODULE_ALIAS_CRYPTO("cmac(aes)");
  70 MODULE_ALIAS_CRYPTO("xcbc(aes)");
  71 MODULE_ALIAS_CRYPTO("cbcmac(aes)");
  72
  73 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
  74 MODULE_LICENSE("GPL v2");
  75
  76 /* defined in aes-modes.S */
  77 asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
  78                                 int rounds, int blocks);
  79 asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[],
  80                                 int rounds, int blocks);
  81
  82 asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
  83                                 int rounds, int blocks, u8 iv[]);
  84 asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[],
  85                                 int rounds, int blocks, u8 iv[]);
  86
  87 asmlinkage void aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
  88                                 int rounds, int bytes, u8 const iv[]);
  89 asmlinkage void aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
  90                                 int rounds, int bytes, u8 const iv[]);
  91
  92 asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
  93                                 int rounds, int bytes, u8 ctr[]);
  94
  95 asmlinkage void aes_xctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
  96                                  int rounds, int bytes, u8 ctr[], int byte_ctr);
  97
  98 asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[],
  99                                 int rounds, int bytes, u32 const rk2[], u8 iv[],
 100                                 int first);
 101 asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[],
 102                                 int rounds, int bytes, u32 const rk2[], u8 iv[],
 103                                 int first);
 104
 105 asmlinkage void aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[],
 106                                       int rounds, int blocks, u8 iv[],
 107                                       u32 const rk2[]);
 108 asmlinkage void aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[],
 109                                       int rounds, int blocks, u8 iv[],
 110                                       u32 const rk2[]);
 111
 112 asmlinkage int aes_mac_update(u8 const in[], u32 const rk[], int rounds,
 113                               int blocks, u8 dg[], int enc_before,
 114                               int enc_after);
 115
 116 struct crypto_aes_xts_ctx {
 117         struct crypto_aes_ctx key1;
 118         struct crypto_aes_ctx __aligned(8) key2;
 119 };
 120
 121 struct crypto_aes_essiv_cbc_ctx {
 122         struct crypto_aes_ctx key1;
 123         struct crypto_aes_ctx __aligned(8) key2;
 124         struct crypto_shash *hash;
 125 };
 126
 127 struct mac_tfm_ctx {
 128         struct crypto_aes_ctx key;
 129         u8 __aligned(8) consts[];
 130 };
 131
 132 struct mac_desc_ctx {
 133         unsigned int len;
 134         u8 dg[AES_BLOCK_SIZE];
 135 };
 136
 137 static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
 138                                unsigned int key_len)
 139 {
 140         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 141
 142         return aes_expandkey(ctx, in_key, key_len);
 143 }
 144
 145 static int __maybe_unused xts_set_key(struct crypto_skcipher *tfm,
 146                                       const u8 *in_key, unsigned int key_len)
 147 {
 148         struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 149         int ret;
 150
 151         ret = xts_verify_key(tfm, in_key, key_len);
 152         if (ret)
 153                 return ret;
 154
 155         ret = aes_expandkey(&ctx->key1, in_key, key_len / 2);
 156         if (!ret)
 157                 ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2],
 158                                     key_len / 2);
 159         return ret;
 160 }
 161
 162 static int __maybe_unused essiv_cbc_set_key(struct crypto_skcipher *tfm,
 163                                             const u8 *in_key,
 164                                             unsigned int key_len)
 165 {
 166         struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 167         u8 digest[SHA256_DIGEST_SIZE];
 168         int ret;
 169
 170         ret = aes_expandkey(&ctx->key1, in_key, key_len);
 171         if (ret)
 172                 return ret;
 173
 174         crypto_shash_tfm_digest(ctx->hash, in_key, key_len, digest);
 175
 176         return aes_expandkey(&ctx->key2, digest, sizeof(digest));
 177 }
 178
 179 static int __maybe_unused ecb_encrypt(struct skcipher_request *req)
 180 {
 181         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 182         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 183         int err, rounds = 6 + ctx->key_length / 4;
 184         struct skcipher_walk walk;
 185         unsigned int blocks;
 186
 187         err = skcipher_walk_virt(&walk, req, false);
 188
 189         while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
 190                 kernel_neon_begin();
 191                 aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 192                                 ctx->key_enc, rounds, blocks);
 193                 kernel_neon_end();
 194                 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 195         }
 196         return err;
 197 }
 198
 199 static int __maybe_unused ecb_decrypt(struct skcipher_request *req)
 200 {
 201         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 202         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 203         int err, rounds = 6 + ctx->key_length / 4;
 204         struct skcipher_walk walk;
 205         unsigned int blocks;
 206
 207         err = skcipher_walk_virt(&walk, req, false);
 208
 209         while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
 210                 kernel_neon_begin();
 211                 aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 212                                 ctx->key_dec, rounds, blocks);
 213                 kernel_neon_end();
 214                 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 215         }
 216         return err;
 217 }
 218
 219 static int cbc_encrypt_walk(struct skcipher_request *req,
 220                             struct skcipher_walk *walk)
 221 {
 222         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 223         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 224         int err = 0, rounds = 6 + ctx->key_length / 4;
 225         unsigned int blocks;
 226
 227         while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
 228                 kernel_neon_begin();
 229                 aes_cbc_encrypt(walk->dst.virt.addr, walk->src.virt.addr,
 230                                 ctx->key_enc, rounds, blocks, walk->iv);
 231                 kernel_neon_end();
 232                 err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
 233         }
 234         return err;
 235 }
 236
 237 static int __maybe_unused cbc_encrypt(struct skcipher_request *req)
 238 {
 239         struct skcipher_walk walk;
 240         int err;
 241
 242         err = skcipher_walk_virt(&walk, req, false);
 243         if (err)
 244                 return err;
 245         return cbc_encrypt_walk(req, &walk);
 246 }
 247
 248 static int cbc_decrypt_walk(struct skcipher_request *req,
 249                             struct skcipher_walk *walk)
 250 {
 251         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 252         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 253         int err = 0, rounds = 6 + ctx->key_length / 4;
 254         unsigned int blocks;
 255
 256         while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
 257                 kernel_neon_begin();
 258                 aes_cbc_decrypt(walk->dst.virt.addr, walk->src.virt.addr,
 259                                 ctx->key_dec, rounds, blocks, walk->iv);
 260                 kernel_neon_end();
 261                 err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
 262         }
 263         return err;
 264 }
 265
 266 static int __maybe_unused cbc_decrypt(struct skcipher_request *req)
 267 {
 268         struct skcipher_walk walk;
 269         int err;
 270
 271         err = skcipher_walk_virt(&walk, req, false);
 272         if (err)
 273                 return err;
 274         return cbc_decrypt_walk(req, &walk);
 275 }
 276
 277 static int cts_cbc_encrypt(struct skcipher_request *req)
 278 {
 279         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 280         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 281         int err, rounds = 6 + ctx->key_length / 4;
 282         int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
 283         struct scatterlist *src = req->src, *dst = req->dst;
 284         struct scatterlist sg_src[2], sg_dst[2];
 285         struct skcipher_request subreq;
 286         struct skcipher_walk walk;
 287
 288         skcipher_request_set_tfm(&subreq, tfm);
 289         skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
 290                                       NULL, NULL);
 291
 292         if (req->cryptlen <= AES_BLOCK_SIZE) {
 293                 if (req->cryptlen < AES_BLOCK_SIZE)
 294                         return -EINVAL;
 295                 cbc_blocks = 1;
 296         }
 297
 298         if (cbc_blocks > 0) {
 299                 skcipher_request_set_crypt(&subreq, req->src, req->dst,
 300                                            cbc_blocks * AES_BLOCK_SIZE,
 301                                            req->iv);
 302
 303                 err = skcipher_walk_virt(&walk, &subreq, false) ?:
 304                       cbc_encrypt_walk(&subreq, &walk);
 305                 if (err)
 306                         return err;
 307
 308                 if (req->cryptlen == AES_BLOCK_SIZE)
 309                         return 0;
 310
 311                 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
 312                 if (req->dst != req->src)
 313                         dst = scatterwalk_ffwd(sg_dst, req->dst,
 314                                                subreq.cryptlen);
 315         }
 316
 317         /* handle ciphertext stealing */
 318         skcipher_request_set_crypt(&subreq, src, dst,
 319                                    req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
 320                                    req->iv);
 321
 322         err = skcipher_walk_virt(&walk, &subreq, false);
 323         if (err)
 324                 return err;
 325
 326         kernel_neon_begin();
 327         aes_cbc_cts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 328                             ctx->key_enc, rounds, walk.nbytes, walk.iv);
 329         kernel_neon_end();
 330
 331         return skcipher_walk_done(&walk, 0);
 332 }
 333
 334 static int cts_cbc_decrypt(struct skcipher_request *req)
 335 {
 336         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 337         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 338         int err, rounds = 6 + ctx->key_length / 4;
 339         int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
 340         struct scatterlist *src = req->src, *dst = req->dst;
 341         struct scatterlist sg_src[2], sg_dst[2];
 342         struct skcipher_request subreq;
 343         struct skcipher_walk walk;
 344
 345         skcipher_request_set_tfm(&subreq, tfm);
 346         skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
 347                                       NULL, NULL);
 348
 349         if (req->cryptlen <= AES_BLOCK_SIZE) {
 350                 if (req->cryptlen < AES_BLOCK_SIZE)
 351                         return -EINVAL;
 352                 cbc_blocks = 1;
 353         }
 354
 355         if (cbc_blocks > 0) {
 356                 skcipher_request_set_crypt(&subreq, req->src, req->dst,
 357                                            cbc_blocks * AES_BLOCK_SIZE,
 358                                            req->iv);
 359
 360                 err = skcipher_walk_virt(&walk, &subreq, false) ?:
 361                       cbc_decrypt_walk(&subreq, &walk);
 362                 if (err)
 363                         return err;
 364
 365                 if (req->cryptlen == AES_BLOCK_SIZE)
 366                         return 0;
 367
 368                 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
 369                 if (req->dst != req->src)
 370                         dst = scatterwalk_ffwd(sg_dst, req->dst,
 371                                                subreq.cryptlen);
 372         }
 373
 374         /* handle ciphertext stealing */
 375         skcipher_request_set_crypt(&subreq, src, dst,
 376                                    req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
 377                                    req->iv);
 378
 379         err = skcipher_walk_virt(&walk, &subreq, false);
 380         if (err)
 381                 return err;
 382
 383         kernel_neon_begin();
 384         aes_cbc_cts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 385                             ctx->key_dec, rounds, walk.nbytes, walk.iv);
 386         kernel_neon_end();
 387
 388         return skcipher_walk_done(&walk, 0);
 389 }
 390
 391 static int __maybe_unused essiv_cbc_init_tfm(struct crypto_skcipher *tfm)
 392 {
 393         struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 394
 395         ctx->hash = crypto_alloc_shash("sha256", 0, 0);
 396
 397         return PTR_ERR_OR_ZERO(ctx->hash);
 398 }
 399
 400 static void __maybe_unused essiv_cbc_exit_tfm(struct crypto_skcipher *tfm)
 401 {
 402         struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 403
 404         crypto_free_shash(ctx->hash);
 405 }
 406
 407 static int __maybe_unused essiv_cbc_encrypt(struct skcipher_request *req)
 408 {
 409         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 410         struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 411         int err, rounds = 6 + ctx->key1.key_length / 4;
 412         struct skcipher_walk walk;
 413         unsigned int blocks;
 414
 415         err = skcipher_walk_virt(&walk, req, false);
 416
 417         blocks = walk.nbytes / AES_BLOCK_SIZE;
 418         if (blocks) {
 419                 kernel_neon_begin();
 420                 aes_essiv_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 421                                       ctx->key1.key_enc, rounds, blocks,
 422                                       req->iv, ctx->key2.key_enc);
 423                 kernel_neon_end();
 424                 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 425         }
 426         return err ?: cbc_encrypt_walk(req, &walk);
 427 }
 428
 429 static int __maybe_unused essiv_cbc_decrypt(struct skcipher_request *req)
 430 {
 431         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 432         struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 433         int err, rounds = 6 + ctx->key1.key_length / 4;
 434         struct skcipher_walk walk;
 435         unsigned int blocks;
 436
 437         err = skcipher_walk_virt(&walk, req, false);
 438
 439         blocks = walk.nbytes / AES_BLOCK_SIZE;
 440         if (blocks) {
 441                 kernel_neon_begin();
 442                 aes_essiv_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 443                                       ctx->key1.key_dec, rounds, blocks,
 444                                       req->iv, ctx->key2.key_enc);
 445                 kernel_neon_end();
 446                 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 447         }
 448         return err ?: cbc_decrypt_walk(req, &walk);
 449 }
 450
 451 static int __maybe_unused xctr_encrypt(struct skcipher_request *req)
 452 {
 453         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 454         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 455         int err, rounds = 6 + ctx->key_length / 4;
 456         struct skcipher_walk walk;
 457         unsigned int byte_ctr = 0;
 458
 459         err = skcipher_walk_virt(&walk, req, false);
 460
 461         while (walk.nbytes > 0) {
 462                 const u8 *src = walk.src.virt.addr;
 463                 unsigned int nbytes = walk.nbytes;
 464                 u8 *dst = walk.dst.virt.addr;
 465                 u8 buf[AES_BLOCK_SIZE];
 466
 467                 /*
 468                  * If given less than 16 bytes, we must copy the partial block
 469                  * into a temporary buffer of 16 bytes to avoid out of bounds
 470                  * reads and writes.  Furthermore, this code is somewhat unusual
 471                  * in that it expects the end of the data to be at the end of
 472                  * the temporary buffer, rather than the start of the data at
 473                  * the start of the temporary buffer.
 474                  */
 475                 if (unlikely(nbytes < AES_BLOCK_SIZE))
 476                         src = dst = memcpy(buf + sizeof(buf) - nbytes,
 477                                            src, nbytes);
 478                 else if (nbytes < walk.total)
 479                         nbytes &= ~(AES_BLOCK_SIZE - 1);
 480
 481                 kernel_neon_begin();
 482                 aes_xctr_encrypt(dst, src, ctx->key_enc, rounds, nbytes,
 483                                                  walk.iv, byte_ctr);
 484                 kernel_neon_end();
 485
 486                 if (unlikely(nbytes < AES_BLOCK_SIZE))
 487                         memcpy(walk.dst.virt.addr,
 488                                buf + sizeof(buf) - nbytes, nbytes);
 489                 byte_ctr += nbytes;
 490
 491                 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
 492         }
 493
 494         return err;
 495 }
 496
 497 static int __maybe_unused ctr_encrypt(struct skcipher_request *req)
 498 {
 499         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 500         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 501         int err, rounds = 6 + ctx->key_length / 4;
 502         struct skcipher_walk walk;
 503
 504         err = skcipher_walk_virt(&walk, req, false);
 505
 506         while (walk.nbytes > 0) {
 507                 const u8 *src = walk.src.virt.addr;
 508                 unsigned int nbytes = walk.nbytes;
 509                 u8 *dst = walk.dst.virt.addr;
 510                 u8 buf[AES_BLOCK_SIZE];
 511
 512                 /*
 513                  * If given less than 16 bytes, we must copy the partial block
 514                  * into a temporary buffer of 16 bytes to avoid out of bounds
 515                  * reads and writes.  Furthermore, this code is somewhat unusual
 516                  * in that it expects the end of the data to be at the end of
 517                  * the temporary buffer, rather than the start of the data at
 518                  * the start of the temporary buffer.
 519                  */
 520                 if (unlikely(nbytes < AES_BLOCK_SIZE))
 521                         src = dst = memcpy(buf + sizeof(buf) - nbytes,
 522                                            src, nbytes);
 523                 else if (nbytes < walk.total)
 524                         nbytes &= ~(AES_BLOCK_SIZE - 1);
 525
 526                 kernel_neon_begin();
 527                 aes_ctr_encrypt(dst, src, ctx->key_enc, rounds, nbytes,
 528                                 walk.iv);
 529                 kernel_neon_end();
 530
 531                 if (unlikely(nbytes < AES_BLOCK_SIZE))
 532                         memcpy(walk.dst.virt.addr,
 533                                buf + sizeof(buf) - nbytes, nbytes);
 534
 535                 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
 536         }
 537
 538         return err;
 539 }
 540
 541 static int __maybe_unused xts_encrypt(struct skcipher_request *req)
 542 {
 543         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 544         struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 545         int err, first, rounds = 6 + ctx->key1.key_length / 4;
 546         int tail = req->cryptlen % AES_BLOCK_SIZE;
 547         struct scatterlist sg_src[2], sg_dst[2];
 548         struct skcipher_request subreq;
 549         struct scatterlist *src, *dst;
 550         struct skcipher_walk walk;
 551
 552         if (req->cryptlen < AES_BLOCK_SIZE)
 553                 return -EINVAL;
 554
 555         err = skcipher_walk_virt(&walk, req, false);
 556
 557         if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
 558                 int xts_blocks = DIV_ROUND_UP(req->cryptlen,
 559                                               AES_BLOCK_SIZE) - 2;
 560
 561                 skcipher_walk_abort(&walk);
 562
 563                 skcipher_request_set_tfm(&subreq, tfm);
 564                 skcipher_request_set_callback(&subreq,
 565                                               skcipher_request_flags(req),
 566                                               NULL, NULL);
 567                 skcipher_request_set_crypt(&subreq, req->src, req->dst,
 568                                            xts_blocks * AES_BLOCK_SIZE,
 569                                            req->iv);
 570                 req = &subreq;
 571                 err = skcipher_walk_virt(&walk, req, false);
 572         } else {
 573                 tail = 0;
 574         }
 575
 576         for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
 577                 int nbytes = walk.nbytes;
 578
 579                 if (walk.nbytes < walk.total)
 580                         nbytes &= ~(AES_BLOCK_SIZE - 1);
 581
 582                 kernel_neon_begin();
 583                 aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 584                                 ctx->key1.key_enc, rounds, nbytes,
 585                                 ctx->key2.key_enc, walk.iv, first);
 586                 kernel_neon_end();
 587                 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
 588         }
 589
 590         if (err || likely(!tail))
 591                 return err;
 592
 593         dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
 594         if (req->dst != req->src)
 595                 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
 596
 597         skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
 598                                    req->iv);
 599
 600         err = skcipher_walk_virt(&walk, &subreq, false);
 601         if (err)
 602                 return err;
 603
 604         kernel_neon_begin();
 605         aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 606                         ctx->key1.key_enc, rounds, walk.nbytes,
 607                         ctx->key2.key_enc, walk.iv, first);
 608         kernel_neon_end();
 609
 610         return skcipher_walk_done(&walk, 0);
 611 }
 612
 613 static int __maybe_unused xts_decrypt(struct skcipher_request *req)
 614 {
 615         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 616         struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 617         int err, first, rounds = 6 + ctx->key1.key_length / 4;
 618         int tail = req->cryptlen % AES_BLOCK_SIZE;
 619         struct scatterlist sg_src[2], sg_dst[2];
 620         struct skcipher_request subreq;
 621         struct scatterlist *src, *dst;
 622         struct skcipher_walk walk;
 623
 624         if (req->cryptlen < AES_BLOCK_SIZE)
 625                 return -EINVAL;
 626
 627         err = skcipher_walk_virt(&walk, req, false);
 628
 629         if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
 630                 int xts_blocks = DIV_ROUND_UP(req->cryptlen,
 631                                               AES_BLOCK_SIZE) - 2;
 632
 633                 skcipher_walk_abort(&walk);
 634
 635                 skcipher_request_set_tfm(&subreq, tfm);
 636                 skcipher_request_set_callback(&subreq,
 637                                               skcipher_request_flags(req),
 638                                               NULL, NULL);
 639                 skcipher_request_set_crypt(&subreq, req->src, req->dst,
 640                                            xts_blocks * AES_BLOCK_SIZE,
 641                                            req->iv);
 642                 req = &subreq;
 643                 err = skcipher_walk_virt(&walk, req, false);
 644         } else {
 645                 tail = 0;
 646         }
 647
 648         for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
 649                 int nbytes = walk.nbytes;
 650
 651                 if (walk.nbytes < walk.total)
 652                         nbytes &= ~(AES_BLOCK_SIZE - 1);
 653
 654                 kernel_neon_begin();
 655                 aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 656                                 ctx->key1.key_dec, rounds, nbytes,
 657                                 ctx->key2.key_enc, walk.iv, first);
 658                 kernel_neon_end();
 659                 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
 660         }
 661
 662         if (err || likely(!tail))
 663                 return err;
 664
 665         dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
 666         if (req->dst != req->src)
 667                 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
 668
 669         skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
 670                                    req->iv);
 671
 672         err = skcipher_walk_virt(&walk, &subreq, false);
 673         if (err)
 674                 return err;
 675
 676
 677         kernel_neon_begin();
 678         aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 679                         ctx->key1.key_dec, rounds, walk.nbytes,
 680                         ctx->key2.key_enc, walk.iv, first);
 681         kernel_neon_end();
 682
 683         return skcipher_walk_done(&walk, 0);
 684 }
 685
 686 static struct skcipher_alg aes_algs[] = { {
 687 #if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS)
 688         .base = {
 689                 .cra_name               = "ecb(aes)",
 690                 .cra_driver_name        = "ecb-aes-" MODE,
 691                 .cra_priority           = PRIO,
 692                 .cra_blocksize          = AES_BLOCK_SIZE,
 693                 .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 694                 .cra_module             = THIS_MODULE,
 695         },
 696         .min_keysize    = AES_MIN_KEY_SIZE,
 697         .max_keysize    = AES_MAX_KEY_SIZE,
 698         .setkey         = skcipher_aes_setkey,
 699         .encrypt        = ecb_encrypt,
 700         .decrypt        = ecb_decrypt,
 701 }, {
 702         .base = {
 703                 .cra_name               = "cbc(aes)",
 704                 .cra_driver_name        = "cbc-aes-" MODE,
 705                 .cra_priority           = PRIO,
 706                 .cra_blocksize          = AES_BLOCK_SIZE,
 707                 .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 708                 .cra_module             = THIS_MODULE,
 709         },
 710         .min_keysize    = AES_MIN_KEY_SIZE,
 711         .max_keysize    = AES_MAX_KEY_SIZE,
 712         .ivsize         = AES_BLOCK_SIZE,
 713         .setkey         = skcipher_aes_setkey,
 714         .encrypt        = cbc_encrypt,
 715         .decrypt        = cbc_decrypt,
 716 }, {
 717         .base = {
 718                 .cra_name               = "ctr(aes)",
 719                 .cra_driver_name        = "ctr-aes-" MODE,
 720                 .cra_priority           = PRIO,
 721                 .cra_blocksize          = 1,
 722                 .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 723                 .cra_module             = THIS_MODULE,
 724         },
 725         .min_keysize    = AES_MIN_KEY_SIZE,
 726         .max_keysize    = AES_MAX_KEY_SIZE,
 727         .ivsize         = AES_BLOCK_SIZE,
 728         .chunksize      = AES_BLOCK_SIZE,
 729         .setkey         = skcipher_aes_setkey,
 730         .encrypt        = ctr_encrypt,
 731         .decrypt        = ctr_encrypt,
 732 }, {
 733         .base = {
 734                 .cra_name               = "xctr(aes)",
 735                 .cra_driver_name        = "xctr-aes-" MODE,
 736                 .cra_priority           = PRIO,
 737                 .cra_blocksize          = 1,
 738                 .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 739                 .cra_module             = THIS_MODULE,
 740         },
 741         .min_keysize    = AES_MIN_KEY_SIZE,
 742         .max_keysize    = AES_MAX_KEY_SIZE,
 743         .ivsize         = AES_BLOCK_SIZE,
 744         .chunksize      = AES_BLOCK_SIZE,
 745         .setkey         = skcipher_aes_setkey,
 746         .encrypt        = xctr_encrypt,
 747         .decrypt        = xctr_encrypt,
 748 }, {
 749         .base = {
 750                 .cra_name               = "xts(aes)",
 751                 .cra_driver_name        = "xts-aes-" MODE,
 752                 .cra_priority           = PRIO,
 753                 .cra_blocksize          = AES_BLOCK_SIZE,
 754                 .cra_ctxsize            = sizeof(struct crypto_aes_xts_ctx),
 755                 .cra_module             = THIS_MODULE,
 756         },
 757         .min_keysize    = 2 * AES_MIN_KEY_SIZE,
 758         .max_keysize    = 2 * AES_MAX_KEY_SIZE,
 759         .ivsize         = AES_BLOCK_SIZE,
 760         .walksize       = 2 * AES_BLOCK_SIZE,
 761         .setkey         = xts_set_key,
 762         .encrypt        = xts_encrypt,
 763         .decrypt        = xts_decrypt,
 764 }, {
 765 #endif
 766         .base = {
 767                 .cra_name               = "cts(cbc(aes))",
 768                 .cra_driver_name        = "cts-cbc-aes-" MODE,
 769                 .cra_priority           = PRIO,
 770                 .cra_blocksize          = AES_BLOCK_SIZE,
 771                 .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 772                 .cra_module             = THIS_MODULE,
 773         },
 774         .min_keysize    = AES_MIN_KEY_SIZE,
 775         .max_keysize    = AES_MAX_KEY_SIZE,
 776         .ivsize         = AES_BLOCK_SIZE,
 777         .walksize       = 2 * AES_BLOCK_SIZE,
 778         .setkey         = skcipher_aes_setkey,
 779         .encrypt        = cts_cbc_encrypt,
 780         .decrypt        = cts_cbc_decrypt,
 781 }, {
 782         .base = {
 783                 .cra_name               = "essiv(cbc(aes),sha256)",
 784                 .cra_driver_name        = "essiv-cbc-aes-sha256-" MODE,
 785                 .cra_priority           = PRIO + 1,
 786                 .cra_blocksize          = AES_BLOCK_SIZE,
 787                 .cra_ctxsize            = sizeof(struct crypto_aes_essiv_cbc_ctx),
 788                 .cra_module             = THIS_MODULE,
 789         },
 790         .min_keysize    = AES_MIN_KEY_SIZE,
 791         .max_keysize    = AES_MAX_KEY_SIZE,
 792         .ivsize         = AES_BLOCK_SIZE,
 793         .setkey         = essiv_cbc_set_key,
 794         .encrypt        = essiv_cbc_encrypt,
 795         .decrypt        = essiv_cbc_decrypt,
 796         .init           = essiv_cbc_init_tfm,
 797         .exit           = essiv_cbc_exit_tfm,
 798 } };
 799
 800 static int cbcmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
 801                          unsigned int key_len)
 802 {
 803         struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
 804
 805         return aes_expandkey(&ctx->key, in_key, key_len);
 806 }
 807
 808 static void cmac_gf128_mul_by_x(be128 *y, const be128 *x)
 809 {
 810         u64 a = be64_to_cpu(x->a);
 811         u64 b = be64_to_cpu(x->b);
 812
 813         y->a = cpu_to_be64((a << 1) | (b >> 63));
 814         y->b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0));
 815 }
 816
 817 static int cmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
 818                        unsigned int key_len)
 819 {
 820         struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
 821         be128 *consts = (be128 *)ctx->consts;
 822         int rounds = 6 + key_len / 4;
 823         int err;
 824
 825         err = cbcmac_setkey(tfm, in_key, key_len);
 826         if (err)
 827                 return err;
 828
 829         /* encrypt the zero vector */
 830         kernel_neon_begin();
 831         aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, ctx->key.key_enc,
 832                         rounds, 1);
 833         kernel_neon_end();
 834
 835         cmac_gf128_mul_by_x(consts, consts);
 836         cmac_gf128_mul_by_x(consts + 1, consts);
 837
 838         return 0;
 839 }
 840
 841 static int xcbc_setkey(struct crypto_shash *tfm, const u8 *in_key,
 842                        unsigned int key_len)
 843 {
 844         static u8 const ks[3][AES_BLOCK_SIZE] = {
 845                 { [0 ... AES_BLOCK_SIZE - 1] = 0x1 },
 846                 { [0 ... AES_BLOCK_SIZE - 1] = 0x2 },
 847                 { [0 ... AES_BLOCK_SIZE - 1] = 0x3 },
 848         };
 849
 850         struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
 851         int rounds = 6 + key_len / 4;
 852         u8 key[AES_BLOCK_SIZE];
 853         int err;
 854
 855         err = cbcmac_setkey(tfm, in_key, key_len);
 856         if (err)
 857                 return err;
 858
 859         kernel_neon_begin();
 860         aes_ecb_encrypt(key, ks[0], ctx->key.key_enc, rounds, 1);
 861         aes_ecb_encrypt(ctx->consts, ks[1], ctx->key.key_enc, rounds, 2);
 862         kernel_neon_end();
 863
 864         return cbcmac_setkey(tfm, key, sizeof(key));
 865 }
 866
 867 static int mac_init(struct shash_desc *desc)
 868 {
 869         struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
 870
 871         memset(ctx->dg, 0, AES_BLOCK_SIZE);
 872         ctx->len = 0;
 873
 874         return 0;
 875 }
 876
 877 static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks,
 878                           u8 dg[], int enc_before, int enc_after)
 879 {
 880         int rounds = 6 + ctx->key_length / 4;
 881
 882         if (crypto_simd_usable()) {
 883                 int rem;
 884
 885                 do {
 886                         kernel_neon_begin();
 887                         rem = aes_mac_update(in, ctx->key_enc, rounds, blocks,
 888                                              dg, enc_before, enc_after);
 889                         kernel_neon_end();
 890                         in += (blocks - rem) * AES_BLOCK_SIZE;
 891                         blocks = rem;
 892                         enc_before = 0;
 893                 } while (blocks);
 894         } else {
 895                 if (enc_before)
 896                         aes_encrypt(ctx, dg, dg);
 897
 898                 while (blocks--) {
 899                         crypto_xor(dg, in, AES_BLOCK_SIZE);
 900                         in += AES_BLOCK_SIZE;
 901
 902                         if (blocks || enc_after)
 903                                 aes_encrypt(ctx, dg, dg);
 904                 }
 905         }
 906 }
 907
 908 static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len)
 909 {
 910         struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
 911         struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
 912
 913         while (len > 0) {
 914                 unsigned int l;
 915
 916                 if ((ctx->len % AES_BLOCK_SIZE) == 0 &&
 917                     (ctx->len + len) > AES_BLOCK_SIZE) {
 918
 919                         int blocks = len / AES_BLOCK_SIZE;
 920
 921                         len %= AES_BLOCK_SIZE;
 922
 923                         mac_do_update(&tctx->key, p, blocks, ctx->dg,
 924                                       (ctx->len != 0), (len != 0));
 925
 926                         p += blocks * AES_BLOCK_SIZE;
 927
 928                         if (!len) {
 929                                 ctx->len = AES_BLOCK_SIZE;
 930                                 break;
 931                         }
 932                         ctx->len = 0;
 933                 }
 934
 935                 l = min(len, AES_BLOCK_SIZE - ctx->len);
 936
 937                 if (l <= AES_BLOCK_SIZE) {
 938                         crypto_xor(ctx->dg + ctx->len, p, l);
 939                         ctx->len += l;
 940                         len -= l;
 941                         p += l;
 942                 }
 943         }
 944
 945         return 0;
 946 }
 947
 948 static int cbcmac_final(struct shash_desc *desc, u8 *out)
 949 {
 950         struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
 951         struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
 952
 953         mac_do_update(&tctx->key, NULL, 0, ctx->dg, (ctx->len != 0), 0);
 954
 955         memcpy(out, ctx->dg, AES_BLOCK_SIZE);
 956
 957         return 0;
 958 }
 959
 960 static int cmac_final(struct shash_desc *desc, u8 *out)
 961 {
 962         struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
 963         struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
 964         u8 *consts = tctx->consts;
 965
 966         if (ctx->len != AES_BLOCK_SIZE) {
 967                 ctx->dg[ctx->len] ^= 0x80;
 968                 consts += AES_BLOCK_SIZE;
 969         }
 970
 971         mac_do_update(&tctx->key, consts, 1, ctx->dg, 0, 1);
 972
 973         memcpy(out, ctx->dg, AES_BLOCK_SIZE);
 974
 975         return 0;
 976 }
 977
 978 static struct shash_alg mac_algs[] = { {
 979         .base.cra_name          = "cmac(aes)",
 980         .base.cra_driver_name   = "cmac-aes-" MODE,
 981         .base.cra_priority      = PRIO,
 982         .base.cra_blocksize     = AES_BLOCK_SIZE,
 983         .base.cra_ctxsize       = sizeof(struct mac_tfm_ctx) +
 984                                   2 * AES_BLOCK_SIZE,
 985         .base.cra_module        = THIS_MODULE,
 986
 987         .digestsize             = AES_BLOCK_SIZE,
 988         .init                   = mac_init,
 989         .update                 = mac_update,
 990         .final                  = cmac_final,
 991         .setkey                 = cmac_setkey,
 992         .descsize               = sizeof(struct mac_desc_ctx),
 993 }, {
 994         .base.cra_name          = "xcbc(aes)",
 995         .base.cra_driver_name   = "xcbc-aes-" MODE,
 996         .base.cra_priority      = PRIO,
 997         .base.cra_blocksize     = AES_BLOCK_SIZE,
 998         .base.cra_ctxsize       = sizeof(struct mac_tfm_ctx) +
 999                                   2 * AES_BLOCK_SIZE,
1000         .base.cra_module        = THIS_MODULE,
1001
1002         .digestsize             = AES_BLOCK_SIZE,
1003         .init                   = mac_init,
1004         .update                 = mac_update,
1005         .final                  = cmac_final,
1006         .setkey                 = xcbc_setkey,
1007         .descsize               = sizeof(struct mac_desc_ctx),
1008 }, {
1009         .base.cra_name          = "cbcmac(aes)",
1010         .base.cra_driver_name   = "cbcmac-aes-" MODE,
1011         .base.cra_priority      = PRIO,
1012         .base.cra_blocksize     = 1,
1013         .base.cra_ctxsize       = sizeof(struct mac_tfm_ctx),
1014         .base.cra_module        = THIS_MODULE,
1015
1016         .digestsize             = AES_BLOCK_SIZE,
1017         .init                   = mac_init,
1018         .update                 = mac_update,
1019         .final                  = cbcmac_final,
1020         .setkey                 = cbcmac_setkey,
1021         .descsize               = sizeof(struct mac_desc_ctx),
1022 } };
1023
1024 static void aes_exit(void)
1025 {
1026         crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs));
1027         crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
1028 }
1029
1030 static int __init aes_init(void)
1031 {
1032         int err;
1033
1034         err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
1035         if (err)
1036                 return err;
1037
1038         err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs));
1039         if (err)
1040                 goto unregister_ciphers;
1041
1042         return 0;
1043
1044 unregister_ciphers:
1045         crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
1046         return err;
1047 }
1048
1049 #ifdef USE_V8_CRYPTO_EXTENSIONS
1050 module_cpu_feature_match(AES, aes_init);
1051 #else
1052 module_init(aes_init);
1053 EXPORT_SYMBOL(neon_aes_ecb_encrypt);
1054 EXPORT_SYMBOL(neon_aes_cbc_encrypt);
1055 EXPORT_SYMBOL(neon_aes_ctr_encrypt);
1056 EXPORT_SYMBOL(neon_aes_xts_encrypt);
1057 EXPORT_SYMBOL(neon_aes_xts_decrypt);
1058 #endif
1059 module_exit(aes_exit);