drivers/crypto/caam/caamalg_qi.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Freescale FSL CAAM support for crypto API over QI backend.
   4  * Based on caamalg.c
   5  *
   6  * Copyright 2013-2016 Freescale Semiconductor, Inc.
   7  * Copyright 2016-2019 NXP
   8  */
   9
  10 #include "compat.h"
  11 #include "ctrl.h"
  12 #include "regs.h"
  13 #include "intern.h"
  14 #include "desc_constr.h"
  15 #include "error.h"
  16 #include "sg_sw_qm.h"
  17 #include "key_gen.h"
  18 #include "qi.h"
  19 #include "jr.h"
  20 #include "caamalg_desc.h"
  21 #include <crypto/xts.h>
  22 #include <asm/unaligned.h>
  23
  24 /*
  25  * crypto alg
  26  */
  27 #define CAAM_CRA_PRIORITY               2000
  28 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  29 #define CAAM_MAX_KEY_SIZE               (AES_MAX_KEY_SIZE + \
  30                                          SHA512_DIGEST_SIZE * 2)
  31
  32 #define DESC_MAX_USED_BYTES             (DESC_QI_AEAD_GIVENC_LEN + \
  33                                          CAAM_MAX_KEY_SIZE)
  34 #define DESC_MAX_USED_LEN               (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
  35
  36 struct caam_alg_entry {
  37         int class1_alg_type;
  38         int class2_alg_type;
  39         bool rfc3686;
  40         bool geniv;
  41         bool nodkp;
  42 };
  43
  44 struct caam_aead_alg {
  45         struct aead_alg aead;
  46         struct caam_alg_entry caam;
  47         bool registered;
  48 };
  49
  50 struct caam_skcipher_alg {
  51         struct skcipher_alg skcipher;
  52         struct caam_alg_entry caam;
  53         bool registered;
  54 };
  55
  56 /*
  57  * per-session context
  58  */
  59 struct caam_ctx {
  60         struct device *jrdev;
  61         u32 sh_desc_enc[DESC_MAX_USED_LEN];
  62         u32 sh_desc_dec[DESC_MAX_USED_LEN];
  63         u8 key[CAAM_MAX_KEY_SIZE];
  64         dma_addr_t key_dma;
  65         enum dma_data_direction dir;
  66         struct alginfo adata;
  67         struct alginfo cdata;
  68         unsigned int authsize;
  69         struct device *qidev;
  70         spinlock_t lock;        /* Protects multiple init of driver context */
  71         struct caam_drv_ctx *drv_ctx[NUM_OP];
  72         bool xts_key_fallback;
  73         struct crypto_skcipher *fallback;
  74 };
  75
  76 struct caam_skcipher_req_ctx {
  77         struct skcipher_request fallback_req;
  78 };
  79
  80 static int aead_set_sh_desc(struct crypto_aead *aead)
  81 {
  82         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
  83                                                  typeof(*alg), aead);
  84         struct caam_ctx *ctx = crypto_aead_ctx(aead);
  85         unsigned int ivsize = crypto_aead_ivsize(aead);
  86         u32 ctx1_iv_off = 0;
  87         u32 *nonce = NULL;
  88         unsigned int data_len[2];
  89         u32 inl_mask;
  90         const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
  91                                OP_ALG_AAI_CTR_MOD128);
  92         const bool is_rfc3686 = alg->caam.rfc3686;
  93         struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
  94
  95         if (!ctx->cdata.keylen || !ctx->authsize)
  96                 return 0;
  97
  98         /*
  99          * AES-CTR needs to load IV in CONTEXT1 reg
 100          * at an offset of 128bits (16bytes)
 101          * CONTEXT1[255:128] = IV
 102          */
 103         if (ctr_mode)
 104                 ctx1_iv_off = 16;
 105
 106         /*
 107          * RFC3686 specific:
 108          *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 109          */
 110         if (is_rfc3686) {
 111                 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 112                 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
 113                                 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
 114         }
 115
 116         /*
 117          * In case |user key| > |derived key|, using DKP<imm,imm> would result
 118          * in invalid opcodes (last bytes of user key) in the resulting
 119          * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
 120          * addresses are needed.
 121          */
 122         ctx->adata.key_virt = ctx->key;
 123         ctx->adata.key_dma = ctx->key_dma;
 124
 125         ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 126         ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 127
 128         data_len[0] = ctx->adata.keylen_pad;
 129         data_len[1] = ctx->cdata.keylen;
 130
 131         if (alg->caam.geniv)
 132                 goto skip_enc;
 133
 134         /* aead_encrypt shared descriptor */
 135         if (desc_inline_query(DESC_QI_AEAD_ENC_LEN +
 136                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 137                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 138                               ARRAY_SIZE(data_len)) < 0)
 139                 return -EINVAL;
 140
 141         ctx->adata.key_inline = !!(inl_mask & 1);
 142         ctx->cdata.key_inline = !!(inl_mask & 2);
 143
 144         cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata,
 145                                ivsize, ctx->authsize, is_rfc3686, nonce,
 146                                ctx1_iv_off, true, ctrlpriv->era);
 147
 148 skip_enc:
 149         /* aead_decrypt shared descriptor */
 150         if (desc_inline_query(DESC_QI_AEAD_DEC_LEN +
 151                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 152                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 153                               ARRAY_SIZE(data_len)) < 0)
 154                 return -EINVAL;
 155
 156         ctx->adata.key_inline = !!(inl_mask & 1);
 157         ctx->cdata.key_inline = !!(inl_mask & 2);
 158
 159         cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata,
 160                                ivsize, ctx->authsize, alg->caam.geniv,
 161                                is_rfc3686, nonce, ctx1_iv_off, true,
 162                                ctrlpriv->era);
 163
 164         if (!alg->caam.geniv)
 165                 goto skip_givenc;
 166
 167         /* aead_givencrypt shared descriptor */
 168         if (desc_inline_query(DESC_QI_AEAD_GIVENC_LEN +
 169                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 170                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 171                               ARRAY_SIZE(data_len)) < 0)
 172                 return -EINVAL;
 173
 174         ctx->adata.key_inline = !!(inl_mask & 1);
 175         ctx->cdata.key_inline = !!(inl_mask & 2);
 176
 177         cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata,
 178                                   ivsize, ctx->authsize, is_rfc3686, nonce,
 179                                   ctx1_iv_off, true, ctrlpriv->era);
 180
 181 skip_givenc:
 182         return 0;
 183 }
 184
 185 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 186 {
 187         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 188
 189         ctx->authsize = authsize;
 190         aead_set_sh_desc(authenc);
 191
 192         return 0;
 193 }
 194
 195 static int aead_setkey(struct crypto_aead *aead, const u8 *key,
 196                        unsigned int keylen)
 197 {
 198         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 199         struct device *jrdev = ctx->jrdev;
 200         struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 201         struct crypto_authenc_keys keys;
 202         int ret = 0;
 203
 204         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 205                 goto badkey;
 206
 207         dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
 208                 keys.authkeylen + keys.enckeylen, keys.enckeylen,
 209                 keys.authkeylen);
 210         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 211                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 212
 213         /*
 214          * If DKP is supported, use it in the shared descriptor to generate
 215          * the split key.
 216          */
 217         if (ctrlpriv->era >= 6) {
 218                 ctx->adata.keylen = keys.authkeylen;
 219                 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
 220                                                       OP_ALG_ALGSEL_MASK);
 221
 222                 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
 223                         goto badkey;
 224
 225                 memcpy(ctx->key, keys.authkey, keys.authkeylen);
 226                 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
 227                        keys.enckeylen);
 228                 dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
 229                                            ctx->adata.keylen_pad +
 230                                            keys.enckeylen, ctx->dir);
 231                 goto skip_split_key;
 232         }
 233
 234         ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey,
 235                             keys.authkeylen, CAAM_MAX_KEY_SIZE -
 236                             keys.enckeylen);
 237         if (ret)
 238                 goto badkey;
 239
 240         /* postpend encryption key to auth split key */
 241         memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
 242         dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
 243                                    ctx->adata.keylen_pad + keys.enckeylen,
 244                                    ctx->dir);
 245
 246         print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
 247                              DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 248                              ctx->adata.keylen_pad + keys.enckeylen, 1);
 249
 250 skip_split_key:
 251         ctx->cdata.keylen = keys.enckeylen;
 252
 253         ret = aead_set_sh_desc(aead);
 254         if (ret)
 255                 goto badkey;
 256
 257         /* Now update the driver contexts with the new shared descriptor */
 258         if (ctx->drv_ctx[ENCRYPT]) {
 259                 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
 260                                           ctx->sh_desc_enc);
 261                 if (ret) {
 262                         dev_err(jrdev, "driver enc context update failed\n");
 263                         goto badkey;
 264                 }
 265         }
 266
 267         if (ctx->drv_ctx[DECRYPT]) {
 268                 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
 269                                           ctx->sh_desc_dec);
 270                 if (ret) {
 271                         dev_err(jrdev, "driver dec context update failed\n");
 272                         goto badkey;
 273                 }
 274         }
 275
 276         memzero_explicit(&keys, sizeof(keys));
 277         return ret;
 278 badkey:
 279         memzero_explicit(&keys, sizeof(keys));
 280         return -EINVAL;
 281 }
 282
 283 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
 284                             unsigned int keylen)
 285 {
 286         struct crypto_authenc_keys keys;
 287         int err;
 288
 289         err = crypto_authenc_extractkeys(&keys, key, keylen);
 290         if (unlikely(err))
 291                 return err;
 292
 293         err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?:
 294               aead_setkey(aead, key, keylen);
 295
 296         memzero_explicit(&keys, sizeof(keys));
 297         return err;
 298 }
 299
 300 static int gcm_set_sh_desc(struct crypto_aead *aead)
 301 {
 302         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 303         unsigned int ivsize = crypto_aead_ivsize(aead);
 304         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 305                         ctx->cdata.keylen;
 306
 307         if (!ctx->cdata.keylen || !ctx->authsize)
 308                 return 0;
 309
 310         /*
 311          * Job Descriptor and Shared Descriptor
 312          * must fit into the 64-word Descriptor h/w Buffer
 313          */
 314         if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {
 315                 ctx->cdata.key_inline = true;
 316                 ctx->cdata.key_virt = ctx->key;
 317         } else {
 318                 ctx->cdata.key_inline = false;
 319                 ctx->cdata.key_dma = ctx->key_dma;
 320         }
 321
 322         cnstr_shdsc_gcm_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
 323                               ctx->authsize, true);
 324
 325         /*
 326          * Job Descriptor and Shared Descriptor
 327          * must fit into the 64-word Descriptor h/w Buffer
 328          */
 329         if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {
 330                 ctx->cdata.key_inline = true;
 331                 ctx->cdata.key_virt = ctx->key;
 332         } else {
 333                 ctx->cdata.key_inline = false;
 334                 ctx->cdata.key_dma = ctx->key_dma;
 335         }
 336
 337         cnstr_shdsc_gcm_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
 338                               ctx->authsize, true);
 339
 340         return 0;
 341 }
 342
 343 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 344 {
 345         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 346         int err;
 347
 348         err = crypto_gcm_check_authsize(authsize);
 349         if (err)
 350                 return err;
 351
 352         ctx->authsize = authsize;
 353         gcm_set_sh_desc(authenc);
 354
 355         return 0;
 356 }
 357
 358 static int gcm_setkey(struct crypto_aead *aead,
 359                       const u8 *key, unsigned int keylen)
 360 {
 361         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 362         struct device *jrdev = ctx->jrdev;
 363         int ret;
 364
 365         ret = aes_check_keylen(keylen);
 366         if (ret)
 367                 return ret;
 368
 369         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 370                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 371
 372         memcpy(ctx->key, key, keylen);
 373         dma_sync_single_for_device(jrdev->parent, ctx->key_dma, keylen,
 374                                    ctx->dir);
 375         ctx->cdata.keylen = keylen;
 376
 377         ret = gcm_set_sh_desc(aead);
 378         if (ret)
 379                 return ret;
 380
 381         /* Now update the driver contexts with the new shared descriptor */
 382         if (ctx->drv_ctx[ENCRYPT]) {
 383                 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
 384                                           ctx->sh_desc_enc);
 385                 if (ret) {
 386                         dev_err(jrdev, "driver enc context update failed\n");
 387                         return ret;
 388                 }
 389         }
 390
 391         if (ctx->drv_ctx[DECRYPT]) {
 392                 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
 393                                           ctx->sh_desc_dec);
 394                 if (ret) {
 395                         dev_err(jrdev, "driver dec context update failed\n");
 396                         return ret;
 397                 }
 398         }
 399
 400         return 0;
 401 }
 402
 403 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 404 {
 405         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 406         unsigned int ivsize = crypto_aead_ivsize(aead);
 407         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 408                         ctx->cdata.keylen;
 409
 410         if (!ctx->cdata.keylen || !ctx->authsize)
 411                 return 0;
 412
 413         ctx->cdata.key_virt = ctx->key;
 414
 415         /*
 416          * Job Descriptor and Shared Descriptor
 417          * must fit into the 64-word Descriptor h/w Buffer
 418          */
 419         if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {
 420                 ctx->cdata.key_inline = true;
 421         } else {
 422                 ctx->cdata.key_inline = false;
 423                 ctx->cdata.key_dma = ctx->key_dma;
 424         }
 425
 426         cnstr_shdsc_rfc4106_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
 427                                   ctx->authsize, true);
 428
 429         /*
 430          * Job Descriptor and Shared Descriptor
 431          * must fit into the 64-word Descriptor h/w Buffer
 432          */
 433         if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {
 434                 ctx->cdata.key_inline = true;
 435         } else {
 436                 ctx->cdata.key_inline = false;
 437                 ctx->cdata.key_dma = ctx->key_dma;
 438         }
 439
 440         cnstr_shdsc_rfc4106_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
 441                                   ctx->authsize, true);
 442
 443         return 0;
 444 }
 445
 446 static int rfc4106_setauthsize(struct crypto_aead *authenc,
 447                                unsigned int authsize)
 448 {
 449         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 450         int err;
 451
 452         err = crypto_rfc4106_check_authsize(authsize);
 453         if (err)
 454                 return err;
 455
 456         ctx->authsize = authsize;
 457         rfc4106_set_sh_desc(authenc);
 458
 459         return 0;
 460 }
 461
 462 static int rfc4106_setkey(struct crypto_aead *aead,
 463                           const u8 *key, unsigned int keylen)
 464 {
 465         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 466         struct device *jrdev = ctx->jrdev;
 467         int ret;
 468
 469         ret = aes_check_keylen(keylen - 4);
 470         if (ret)
 471                 return ret;
 472
 473         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 474                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 475
 476         memcpy(ctx->key, key, keylen);
 477         /*
 478          * The last four bytes of the key material are used as the salt value
 479          * in the nonce. Update the AES key length.
 480          */
 481         ctx->cdata.keylen = keylen - 4;
 482         dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
 483                                    ctx->cdata.keylen, ctx->dir);
 484
 485         ret = rfc4106_set_sh_desc(aead);
 486         if (ret)
 487                 return ret;
 488
 489         /* Now update the driver contexts with the new shared descriptor */
 490         if (ctx->drv_ctx[ENCRYPT]) {
 491                 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
 492                                           ctx->sh_desc_enc);
 493                 if (ret) {
 494                         dev_err(jrdev, "driver enc context update failed\n");
 495                         return ret;
 496                 }
 497         }
 498
 499         if (ctx->drv_ctx[DECRYPT]) {
 500                 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
 501                                           ctx->sh_desc_dec);
 502                 if (ret) {
 503                         dev_err(jrdev, "driver dec context update failed\n");
 504                         return ret;
 505                 }
 506         }
 507
 508         return 0;
 509 }
 510
 511 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
 512 {
 513         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 514         unsigned int ivsize = crypto_aead_ivsize(aead);
 515         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 516                         ctx->cdata.keylen;
 517
 518         if (!ctx->cdata.keylen || !ctx->authsize)
 519                 return 0;
 520
 521         ctx->cdata.key_virt = ctx->key;
 522
 523         /*
 524          * Job Descriptor and Shared Descriptor
 525          * must fit into the 64-word Descriptor h/w Buffer
 526          */
 527         if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {
 528                 ctx->cdata.key_inline = true;
 529         } else {
 530                 ctx->cdata.key_inline = false;
 531                 ctx->cdata.key_dma = ctx->key_dma;
 532         }
 533
 534         cnstr_shdsc_rfc4543_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
 535                                   ctx->authsize, true);
 536
 537         /*
 538          * Job Descriptor and Shared Descriptor
 539          * must fit into the 64-word Descriptor h/w Buffer
 540          */
 541         if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {
 542                 ctx->cdata.key_inline = true;
 543         } else {
 544                 ctx->cdata.key_inline = false;
 545                 ctx->cdata.key_dma = ctx->key_dma;
 546         }
 547
 548         cnstr_shdsc_rfc4543_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
 549                                   ctx->authsize, true);
 550
 551         return 0;
 552 }
 553
 554 static int rfc4543_setauthsize(struct crypto_aead *authenc,
 555                                unsigned int authsize)
 556 {
 557         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 558
 559         if (authsize != 16)
 560                 return -EINVAL;
 561
 562         ctx->authsize = authsize;
 563         rfc4543_set_sh_desc(authenc);
 564
 565         return 0;
 566 }
 567
 568 static int rfc4543_setkey(struct crypto_aead *aead,
 569                           const u8 *key, unsigned int keylen)
 570 {
 571         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 572         struct device *jrdev = ctx->jrdev;
 573         int ret;
 574
 575         ret = aes_check_keylen(keylen - 4);
 576         if (ret)
 577                 return ret;
 578
 579         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 580                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 581
 582         memcpy(ctx->key, key, keylen);
 583         /*
 584          * The last four bytes of the key material are used as the salt value
 585          * in the nonce. Update the AES key length.
 586          */
 587         ctx->cdata.keylen = keylen - 4;
 588         dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
 589                                    ctx->cdata.keylen, ctx->dir);
 590
 591         ret = rfc4543_set_sh_desc(aead);
 592         if (ret)
 593                 return ret;
 594
 595         /* Now update the driver contexts with the new shared descriptor */
 596         if (ctx->drv_ctx[ENCRYPT]) {
 597                 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
 598                                           ctx->sh_desc_enc);
 599                 if (ret) {
 600                         dev_err(jrdev, "driver enc context update failed\n");
 601                         return ret;
 602                 }
 603         }
 604
 605         if (ctx->drv_ctx[DECRYPT]) {
 606                 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
 607                                           ctx->sh_desc_dec);
 608                 if (ret) {
 609                         dev_err(jrdev, "driver dec context update failed\n");
 610                         return ret;
 611                 }
 612         }
 613
 614         return 0;
 615 }
 616
 617 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 618                            unsigned int keylen, const u32 ctx1_iv_off)
 619 {
 620         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
 621         struct caam_skcipher_alg *alg =
 622                 container_of(crypto_skcipher_alg(skcipher), typeof(*alg),
 623                              skcipher);
 624         struct device *jrdev = ctx->jrdev;
 625         unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
 626         const bool is_rfc3686 = alg->caam.rfc3686;
 627         int ret = 0;
 628
 629         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 630                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 631
 632         ctx->cdata.keylen = keylen;
 633         ctx->cdata.key_virt = key;
 634         ctx->cdata.key_inline = true;
 635
 636         /* skcipher encrypt, decrypt shared descriptors */
 637         cnstr_shdsc_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
 638                                    is_rfc3686, ctx1_iv_off);
 639         cnstr_shdsc_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
 640                                    is_rfc3686, ctx1_iv_off);
 641
 642         /* Now update the driver contexts with the new shared descriptor */
 643         if (ctx->drv_ctx[ENCRYPT]) {
 644                 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
 645                                           ctx->sh_desc_enc);
 646                 if (ret) {
 647                         dev_err(jrdev, "driver enc context update failed\n");
 648                         return -EINVAL;
 649                 }
 650         }
 651
 652         if (ctx->drv_ctx[DECRYPT]) {
 653                 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
 654                                           ctx->sh_desc_dec);
 655                 if (ret) {
 656                         dev_err(jrdev, "driver dec context update failed\n");
 657                         return -EINVAL;
 658                 }
 659         }
 660
 661         return ret;
 662 }
 663
 664 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
 665                                const u8 *key, unsigned int keylen)
 666 {
 667         int err;
 668
 669         err = aes_check_keylen(keylen);
 670         if (err)
 671                 return err;
 672
 673         return skcipher_setkey(skcipher, key, keylen, 0);
 674 }
 675
 676 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
 677                                    const u8 *key, unsigned int keylen)
 678 {
 679         u32 ctx1_iv_off;
 680         int err;
 681
 682         /*
 683          * RFC3686 specific:
 684          *      | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 685          *      | *key = {KEY, NONCE}
 686          */
 687         ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 688         keylen -= CTR_RFC3686_NONCE_SIZE;
 689
 690         err = aes_check_keylen(keylen);
 691         if (err)
 692                 return err;
 693
 694         return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
 695 }
 696
 697 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
 698                                const u8 *key, unsigned int keylen)
 699 {
 700         u32 ctx1_iv_off;
 701         int err;
 702
 703         /*
 704          * AES-CTR needs to load IV in CONTEXT1 reg
 705          * at an offset of 128bits (16bytes)
 706          * CONTEXT1[255:128] = IV
 707          */
 708         ctx1_iv_off = 16;
 709
 710         err = aes_check_keylen(keylen);
 711         if (err)
 712                 return err;
 713
 714         return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
 715 }
 716
 717 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
 718                                 const u8 *key, unsigned int keylen)
 719 {
 720         return verify_skcipher_des3_key(skcipher, key) ?:
 721                skcipher_setkey(skcipher, key, keylen, 0);
 722 }
 723
 724 static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
 725                                const u8 *key, unsigned int keylen)
 726 {
 727         return verify_skcipher_des_key(skcipher, key) ?:
 728                skcipher_setkey(skcipher, key, keylen, 0);
 729 }
 730
 731 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 732                                unsigned int keylen)
 733 {
 734         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
 735         struct device *jrdev = ctx->jrdev;
 736         struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 737         int ret = 0;
 738         int err;
 739
 740         err = xts_verify_key(skcipher, key, keylen);
 741         if (err) {
 742                 dev_dbg(jrdev, "key size mismatch\n");
 743                 return err;
 744         }
 745
 746         if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256)
 747                 ctx->xts_key_fallback = true;
 748
 749         if (ctrlpriv->era <= 8 || ctx->xts_key_fallback) {
 750                 err = crypto_skcipher_setkey(ctx->fallback, key, keylen);
 751                 if (err)
 752                         return err;
 753         }
 754
 755         ctx->cdata.keylen = keylen;
 756         ctx->cdata.key_virt = key;
 757         ctx->cdata.key_inline = true;
 758
 759         /* xts skcipher encrypt, decrypt shared descriptors */
 760         cnstr_shdsc_xts_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata);
 761         cnstr_shdsc_xts_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata);
 762
 763         /* Now update the driver contexts with the new shared descriptor */
 764         if (ctx->drv_ctx[ENCRYPT]) {
 765                 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
 766                                           ctx->sh_desc_enc);
 767                 if (ret) {
 768                         dev_err(jrdev, "driver enc context update failed\n");
 769                         return -EINVAL;
 770                 }
 771         }
 772
 773         if (ctx->drv_ctx[DECRYPT]) {
 774                 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
 775                                           ctx->sh_desc_dec);
 776                 if (ret) {
 777                         dev_err(jrdev, "driver dec context update failed\n");
 778                         return -EINVAL;
 779                 }
 780         }
 781
 782         return ret;
 783 }
 784
 785 /*
 786  * aead_edesc - s/w-extended aead descriptor
 787  * @src_nents: number of segments in input scatterlist
 788  * @dst_nents: number of segments in output scatterlist
 789  * @iv_dma: dma address of iv for checking continuity and link table
 790  * @qm_sg_bytes: length of dma mapped h/w link table
 791  * @qm_sg_dma: bus physical mapped address of h/w link table
 792  * @assoclen: associated data length, in CAAM endianness
 793  * @assoclen_dma: bus physical mapped address of req->assoclen
 794  * @drv_req: driver-specific request structure
 795  * @sgt: the h/w link table, followed by IV
 796  */
 797 struct aead_edesc {
 798         int src_nents;
 799         int dst_nents;
 800         dma_addr_t iv_dma;
 801         int qm_sg_bytes;
 802         dma_addr_t qm_sg_dma;
 803         unsigned int assoclen;
 804         dma_addr_t assoclen_dma;
 805         struct caam_drv_req drv_req;
 806         struct qm_sg_entry sgt[];
 807 };
 808
 809 /*
 810  * skcipher_edesc - s/w-extended skcipher descriptor
 811  * @src_nents: number of segments in input scatterlist
 812  * @dst_nents: number of segments in output scatterlist
 813  * @iv_dma: dma address of iv for checking continuity and link table
 814  * @qm_sg_bytes: length of dma mapped h/w link table
 815  * @qm_sg_dma: bus physical mapped address of h/w link table
 816  * @drv_req: driver-specific request structure
 817  * @sgt: the h/w link table, followed by IV
 818  */
 819 struct skcipher_edesc {
 820         int src_nents;
 821         int dst_nents;
 822         dma_addr_t iv_dma;
 823         int qm_sg_bytes;
 824         dma_addr_t qm_sg_dma;
 825         struct caam_drv_req drv_req;
 826         struct qm_sg_entry sgt[];
 827 };
 828
 829 static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx,
 830                                         enum optype type)
 831 {
 832         /*
 833          * This function is called on the fast path with values of 'type'
 834          * known at compile time. Invalid arguments are not expected and
 835          * thus no checks are made.
 836          */
 837         struct caam_drv_ctx *drv_ctx = ctx->drv_ctx[type];
 838         u32 *desc;
 839
 840         if (unlikely(!drv_ctx)) {
 841                 spin_lock(&ctx->lock);
 842
 843                 /* Read again to check if some other core init drv_ctx */
 844                 drv_ctx = ctx->drv_ctx[type];
 845                 if (!drv_ctx) {
 846                         int cpu;
 847
 848                         if (type == ENCRYPT)
 849                                 desc = ctx->sh_desc_enc;
 850                         else /* (type == DECRYPT) */
 851                                 desc = ctx->sh_desc_dec;
 852
 853                         cpu = smp_processor_id();
 854                         drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc);
 855                         if (!IS_ERR(drv_ctx))
 856                                 drv_ctx->op_type = type;
 857
 858                         ctx->drv_ctx[type] = drv_ctx;
 859                 }
 860
 861                 spin_unlock(&ctx->lock);
 862         }
 863
 864         return drv_ctx;
 865 }
 866
 867 static void caam_unmap(struct device *dev, struct scatterlist *src,
 868                        struct scatterlist *dst, int src_nents,
 869                        int dst_nents, dma_addr_t iv_dma, int ivsize,
 870                        enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
 871                        int qm_sg_bytes)
 872 {
 873         if (dst != src) {
 874                 if (src_nents)
 875                         dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 876                 if (dst_nents)
 877                         dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
 878         } else {
 879                 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 880         }
 881
 882         if (iv_dma)
 883                 dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
 884         if (qm_sg_bytes)
 885                 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
 886 }
 887
 888 static void aead_unmap(struct device *dev,
 889                        struct aead_edesc *edesc,
 890                        struct aead_request *req)
 891 {
 892         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 893         int ivsize = crypto_aead_ivsize(aead);
 894
 895         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
 896                    edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
 897                    edesc->qm_sg_bytes);
 898         dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
 899 }
 900
 901 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
 902                            struct skcipher_request *req)
 903 {
 904         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
 905         int ivsize = crypto_skcipher_ivsize(skcipher);
 906
 907         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
 908                    edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
 909                    edesc->qm_sg_bytes);
 910 }
 911
 912 static void aead_done(struct caam_drv_req *drv_req, u32 status)
 913 {
 914         struct device *qidev;
 915         struct aead_edesc *edesc;
 916         struct aead_request *aead_req = drv_req->app_ctx;
 917         struct crypto_aead *aead = crypto_aead_reqtfm(aead_req);
 918         struct caam_ctx *caam_ctx = crypto_aead_ctx(aead);
 919         int ecode = 0;
 920
 921         qidev = caam_ctx->qidev;
 922
 923         if (unlikely(status))
 924                 ecode = caam_jr_strstatus(qidev, status);
 925
 926         edesc = container_of(drv_req, typeof(*edesc), drv_req);
 927         aead_unmap(qidev, edesc, aead_req);
 928
 929         aead_request_complete(aead_req, ecode);
 930         qi_cache_free(edesc);
 931 }
 932
 933 /*
 934  * allocate and map the aead extended descriptor
 935  */
 936 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
 937                                            bool encrypt)
 938 {
 939         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 940         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 941         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 942                                                  typeof(*alg), aead);
 943         struct device *qidev = ctx->qidev;
 944         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 945                        GFP_KERNEL : GFP_ATOMIC;
 946         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
 947         int src_len, dst_len = 0;
 948         struct aead_edesc *edesc;
 949         dma_addr_t qm_sg_dma, iv_dma = 0;
 950         int ivsize = 0;
 951         unsigned int authsize = ctx->authsize;
 952         int qm_sg_index = 0, qm_sg_ents = 0, qm_sg_bytes;
 953         int in_len, out_len;
 954         struct qm_sg_entry *sg_table, *fd_sgt;
 955         struct caam_drv_ctx *drv_ctx;
 956
 957         drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);
 958         if (IS_ERR(drv_ctx))
 959                 return (struct aead_edesc *)drv_ctx;
 960
 961         /* allocate space for base edesc and hw desc commands, link tables */
 962         edesc = qi_cache_alloc(GFP_DMA | flags);
 963         if (unlikely(!edesc)) {
 964                 dev_err(qidev, "could not allocate extended descriptor\n");
 965                 return ERR_PTR(-ENOMEM);
 966         }
 967
 968         if (likely(req->src == req->dst)) {
 969                 src_len = req->assoclen + req->cryptlen +
 970                           (encrypt ? authsize : 0);
 971
 972                 src_nents = sg_nents_for_len(req->src, src_len);
 973                 if (unlikely(src_nents < 0)) {
 974                         dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
 975                                 src_len);
 976                         qi_cache_free(edesc);
 977                         return ERR_PTR(src_nents);
 978                 }
 979
 980                 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,
 981                                               DMA_BIDIRECTIONAL);
 982                 if (unlikely(!mapped_src_nents)) {
 983                         dev_err(qidev, "unable to map source\n");
 984                         qi_cache_free(edesc);
 985                         return ERR_PTR(-ENOMEM);
 986                 }
 987         } else {
 988                 src_len = req->assoclen + req->cryptlen;
 989                 dst_len = src_len + (encrypt ? authsize : (-authsize));
 990
 991                 src_nents = sg_nents_for_len(req->src, src_len);
 992                 if (unlikely(src_nents < 0)) {
 993                         dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
 994                                 src_len);
 995                         qi_cache_free(edesc);
 996                         return ERR_PTR(src_nents);
 997                 }
 998
 999                 dst_nents = sg_nents_for_len(req->dst, dst_len);
1000                 if (unlikely(dst_nents < 0)) {
1001                         dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n",
1002                                 dst_len);
1003                         qi_cache_free(edesc);
1004                         return ERR_PTR(dst_nents);
1005                 }
1006
1007                 if (src_nents) {
1008                         mapped_src_nents = dma_map_sg(qidev, req->src,
1009                                                       src_nents, DMA_TO_DEVICE);
1010                         if (unlikely(!mapped_src_nents)) {
1011                                 dev_err(qidev, "unable to map source\n");
1012                                 qi_cache_free(edesc);
1013                                 return ERR_PTR(-ENOMEM);
1014                         }
1015                 } else {
1016                         mapped_src_nents = 0;
1017                 }
1018
1019                 if (dst_nents) {
1020                         mapped_dst_nents = dma_map_sg(qidev, req->dst,
1021                                                       dst_nents,
1022                                                       DMA_FROM_DEVICE);
1023                         if (unlikely(!mapped_dst_nents)) {
1024                                 dev_err(qidev, "unable to map destination\n");
1025                                 dma_unmap_sg(qidev, req->src, src_nents,
1026                                              DMA_TO_DEVICE);
1027                                 qi_cache_free(edesc);
1028                                 return ERR_PTR(-ENOMEM);
1029                         }
1030                 } else {
1031                         mapped_dst_nents = 0;
1032                 }
1033         }
1034
1035         if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv)
1036                 ivsize = crypto_aead_ivsize(aead);
1037
1038         /*
1039          * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
1040          * Input is not contiguous.
1041          * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1042          * the end of the table by allocating more S/G entries. Logic:
1043          * if (src != dst && output S/G)
1044          *      pad output S/G, if needed
1045          * else if (src == dst && S/G)
1046          *      overlapping S/Gs; pad one of them
1047          * else if (input S/G) ...
1048          *      pad input S/G, if needed
1049          */
1050         qm_sg_ents = 1 + !!ivsize + mapped_src_nents;
1051         if (mapped_dst_nents > 1)
1052                 qm_sg_ents += pad_sg_nents(mapped_dst_nents);
1053         else if ((req->src == req->dst) && (mapped_src_nents > 1))
1054                 qm_sg_ents = max(pad_sg_nents(qm_sg_ents),
1055                                  1 + !!ivsize + pad_sg_nents(mapped_src_nents));
1056         else
1057                 qm_sg_ents = pad_sg_nents(qm_sg_ents);
1058
1059         sg_table = &edesc->sgt[0];
1060         qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
1061         if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
1062                      CAAM_QI_MEMCACHE_SIZE)) {
1063                 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
1064                         qm_sg_ents, ivsize);
1065                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1066                            0, DMA_NONE, 0, 0);
1067                 qi_cache_free(edesc);
1068                 return ERR_PTR(-ENOMEM);
1069         }
1070
1071         if (ivsize) {
1072                 u8 *iv = (u8 *)(sg_table + qm_sg_ents);
1073
1074                 /* Make sure IV is located in a DMAable area */
1075                 memcpy(iv, req->iv, ivsize);
1076
1077                 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE);
1078                 if (dma_mapping_error(qidev, iv_dma)) {
1079                         dev_err(qidev, "unable to map IV\n");
1080                         caam_unmap(qidev, req->src, req->dst, src_nents,
1081                                    dst_nents, 0, 0, DMA_NONE, 0, 0);
1082                         qi_cache_free(edesc);
1083                         return ERR_PTR(-ENOMEM);
1084                 }
1085         }
1086
1087         edesc->src_nents = src_nents;
1088         edesc->dst_nents = dst_nents;
1089         edesc->iv_dma = iv_dma;
1090         edesc->drv_req.app_ctx = req;
1091         edesc->drv_req.cbk = aead_done;
1092         edesc->drv_req.drv_ctx = drv_ctx;
1093
1094         edesc->assoclen = cpu_to_caam32(req->assoclen);
1095         edesc->assoclen_dma = dma_map_single(qidev, &edesc->assoclen, 4,
1096                                              DMA_TO_DEVICE);
1097         if (dma_mapping_error(qidev, edesc->assoclen_dma)) {
1098                 dev_err(qidev, "unable to map assoclen\n");
1099                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
1100                            iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
1101                 qi_cache_free(edesc);
1102                 return ERR_PTR(-ENOMEM);
1103         }
1104
1105         dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0);
1106         qm_sg_index++;
1107         if (ivsize) {
1108                 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
1109                 qm_sg_index++;
1110         }
1111         sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
1112         qm_sg_index += mapped_src_nents;
1113
1114         if (mapped_dst_nents > 1)
1115                 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
1116
1117         qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
1118         if (dma_mapping_error(qidev, qm_sg_dma)) {
1119                 dev_err(qidev, "unable to map S/G table\n");
1120                 dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
1121                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
1122                            iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
1123                 qi_cache_free(edesc);
1124                 return ERR_PTR(-ENOMEM);
1125         }
1126
1127         edesc->qm_sg_dma = qm_sg_dma;
1128         edesc->qm_sg_bytes = qm_sg_bytes;
1129
1130         out_len = req->assoclen + req->cryptlen +
1131                   (encrypt ? ctx->authsize : (-ctx->authsize));
1132         in_len = 4 + ivsize + req->assoclen + req->cryptlen;
1133
1134         fd_sgt = &edesc->drv_req.fd_sgt[0];
1135         dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0);
1136
1137         if (req->dst == req->src) {
1138                 if (mapped_src_nents == 1)
1139                         dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src),
1140                                          out_len, 0);
1141                 else
1142                         dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma +
1143                                              (1 + !!ivsize) * sizeof(*sg_table),
1144                                              out_len, 0);
1145         } else if (mapped_dst_nents <= 1) {
1146                 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len,
1147                                  0);
1148         } else {
1149                 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) *
1150                                      qm_sg_index, out_len, 0);
1151         }
1152
1153         return edesc;
1154 }
1155
1156 static inline int aead_crypt(struct aead_request *req, bool encrypt)
1157 {
1158         struct aead_edesc *edesc;
1159         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1160         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1161         int ret;
1162
1163         if (unlikely(caam_congested))
1164                 return -EAGAIN;
1165
1166         /* allocate extended descriptor */
1167         edesc = aead_edesc_alloc(req, encrypt);
1168         if (IS_ERR(edesc))
1169                 return PTR_ERR(edesc);
1170
1171         /* Create and submit job descriptor */
1172         ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req);
1173         if (!ret) {
1174                 ret = -EINPROGRESS;
1175         } else {
1176                 aead_unmap(ctx->qidev, edesc, req);
1177                 qi_cache_free(edesc);
1178         }
1179
1180         return ret;
1181 }
1182
1183 static int aead_encrypt(struct aead_request *req)
1184 {
1185         return aead_crypt(req, true);
1186 }
1187
1188 static int aead_decrypt(struct aead_request *req)
1189 {
1190         return aead_crypt(req, false);
1191 }
1192
1193 static int ipsec_gcm_encrypt(struct aead_request *req)
1194 {
1195         return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req,
1196                                            true);
1197 }
1198
1199 static int ipsec_gcm_decrypt(struct aead_request *req)
1200 {
1201         return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req,
1202                                            false);
1203 }
1204
1205 static void skcipher_done(struct caam_drv_req *drv_req, u32 status)
1206 {
1207         struct skcipher_edesc *edesc;
1208         struct skcipher_request *req = drv_req->app_ctx;
1209         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1210         struct caam_ctx *caam_ctx = crypto_skcipher_ctx(skcipher);
1211         struct device *qidev = caam_ctx->qidev;
1212         int ivsize = crypto_skcipher_ivsize(skcipher);
1213         int ecode = 0;
1214
1215         dev_dbg(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status);
1216
1217         edesc = container_of(drv_req, typeof(*edesc), drv_req);
1218
1219         if (status)
1220                 ecode = caam_jr_strstatus(qidev, status);
1221
1222         print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1223                              DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1224                              edesc->src_nents > 1 ? 100 : ivsize, 1);
1225         caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1226                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1227                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1228
1229         skcipher_unmap(qidev, edesc, req);
1230
1231         /*
1232          * The crypto API expects us to set the IV (req->iv) to the last
1233          * ciphertext block (CBC mode) or last counter (CTR mode).
1234          * This is used e.g. by the CTS mode.
1235          */
1236         if (!ecode)
1237                 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1238                        ivsize);
1239
1240         qi_cache_free(edesc);
1241         skcipher_request_complete(req, ecode);
1242 }
1243
1244 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
1245                                                    bool encrypt)
1246 {
1247         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1248         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1249         struct device *qidev = ctx->qidev;
1250         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1251                        GFP_KERNEL : GFP_ATOMIC;
1252         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1253         struct skcipher_edesc *edesc;
1254         dma_addr_t iv_dma;
1255         u8 *iv;
1256         int ivsize = crypto_skcipher_ivsize(skcipher);
1257         int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
1258         struct qm_sg_entry *sg_table, *fd_sgt;
1259         struct caam_drv_ctx *drv_ctx;
1260
1261         drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);
1262         if (IS_ERR(drv_ctx))
1263                 return (struct skcipher_edesc *)drv_ctx;
1264
1265         src_nents = sg_nents_for_len(req->src, req->cryptlen);
1266         if (unlikely(src_nents < 0)) {
1267                 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
1268                         req->cryptlen);
1269                 return ERR_PTR(src_nents);
1270         }
1271
1272         if (unlikely(req->src != req->dst)) {
1273                 dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
1274                 if (unlikely(dst_nents < 0)) {
1275                         dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n",
1276                                 req->cryptlen);
1277                         return ERR_PTR(dst_nents);
1278                 }
1279
1280                 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,
1281                                               DMA_TO_DEVICE);
1282                 if (unlikely(!mapped_src_nents)) {
1283                         dev_err(qidev, "unable to map source\n");
1284                         return ERR_PTR(-ENOMEM);
1285                 }
1286
1287                 mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents,
1288                                               DMA_FROM_DEVICE);
1289                 if (unlikely(!mapped_dst_nents)) {
1290                         dev_err(qidev, "unable to map destination\n");
1291                         dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE);
1292                         return ERR_PTR(-ENOMEM);
1293                 }
1294         } else {
1295                 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,
1296                                               DMA_BIDIRECTIONAL);
1297                 if (unlikely(!mapped_src_nents)) {
1298                         dev_err(qidev, "unable to map source\n");
1299                         return ERR_PTR(-ENOMEM);
1300                 }
1301         }
1302
1303         qm_sg_ents = 1 + mapped_src_nents;
1304         dst_sg_idx = qm_sg_ents;
1305
1306         /*
1307          * Input, output HW S/G tables: [IV, src][dst, IV]
1308          * IV entries point to the same buffer
1309          * If src == dst, S/G entries are reused (S/G tables overlap)
1310          *
1311          * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1312          * the end of the table by allocating more S/G entries.
1313          */
1314         if (req->src != req->dst)
1315                 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
1316         else
1317                 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
1318
1319         qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);
1320         if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
1321                      ivsize > CAAM_QI_MEMCACHE_SIZE)) {
1322                 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
1323                         qm_sg_ents, ivsize);
1324                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1325                            0, DMA_NONE, 0, 0);
1326                 return ERR_PTR(-ENOMEM);
1327         }
1328
1329         /* allocate space for base edesc, link tables and IV */
1330         edesc = qi_cache_alloc(GFP_DMA | flags);
1331         if (unlikely(!edesc)) {
1332                 dev_err(qidev, "could not allocate extended descriptor\n");
1333                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1334                            0, DMA_NONE, 0, 0);
1335                 return ERR_PTR(-ENOMEM);
1336         }
1337
1338         /* Make sure IV is located in a DMAable area */
1339         sg_table = &edesc->sgt[0];
1340         iv = (u8 *)(sg_table + qm_sg_ents);
1341         memcpy(iv, req->iv, ivsize);
1342
1343         iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL);
1344         if (dma_mapping_error(qidev, iv_dma)) {
1345                 dev_err(qidev, "unable to map IV\n");
1346                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1347                            0, DMA_NONE, 0, 0);
1348                 qi_cache_free(edesc);
1349                 return ERR_PTR(-ENOMEM);
1350         }
1351
1352         edesc->src_nents = src_nents;
1353         edesc->dst_nents = dst_nents;
1354         edesc->iv_dma = iv_dma;
1355         edesc->qm_sg_bytes = qm_sg_bytes;
1356         edesc->drv_req.app_ctx = req;
1357         edesc->drv_req.cbk = skcipher_done;
1358         edesc->drv_req.drv_ctx = drv_ctx;
1359
1360         dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
1361         sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
1362
1363         if (req->src != req->dst)
1364                 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
1365
1366         dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
1367                          ivsize, 0);
1368
1369         edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes,
1370                                           DMA_TO_DEVICE);
1371         if (dma_mapping_error(qidev, edesc->qm_sg_dma)) {
1372                 dev_err(qidev, "unable to map S/G table\n");
1373                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
1374                            iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
1375                 qi_cache_free(edesc);
1376                 return ERR_PTR(-ENOMEM);
1377         }
1378
1379         fd_sgt = &edesc->drv_req.fd_sgt[0];
1380
1381         dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma,
1382                                   ivsize + req->cryptlen, 0);
1383
1384         if (req->src == req->dst)
1385                 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma +
1386                                      sizeof(*sg_table), req->cryptlen + ivsize,
1387                                      0);
1388         else
1389                 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx *
1390                                      sizeof(*sg_table), req->cryptlen + ivsize,
1391                                      0);
1392
1393         return edesc;
1394 }
1395
1396 static inline bool xts_skcipher_ivsize(struct skcipher_request *req)
1397 {
1398         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1399         unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
1400
1401         return !!get_unaligned((u64 *)(req->iv + (ivsize / 2)));
1402 }
1403
1404 static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
1405 {
1406         struct skcipher_edesc *edesc;
1407         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1408         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1409         struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
1410         int ret;
1411
1412         /*
1413          * XTS is expected to return an error even for input length = 0
1414          * Note that the case input length < block size will be caught during
1415          * HW offloading and return an error.
1416          */
1417         if (!req->cryptlen && !ctx->fallback)
1418                 return 0;
1419
1420         if (ctx->fallback && ((ctrlpriv->era <= 8 && xts_skcipher_ivsize(req)) ||
1421                               ctx->xts_key_fallback)) {
1422                 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
1423
1424                 skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback);
1425                 skcipher_request_set_callback(&rctx->fallback_req,
1426                                               req->base.flags,
1427                                               req->base.complete,
1428                                               req->base.data);
1429                 skcipher_request_set_crypt(&rctx->fallback_req, req->src,
1430                                            req->dst, req->cryptlen, req->iv);
1431
1432                 return encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) :
1433                                  crypto_skcipher_decrypt(&rctx->fallback_req);
1434         }
1435
1436         if (unlikely(caam_congested))
1437                 return -EAGAIN;
1438
1439         /* allocate extended descriptor */
1440         edesc = skcipher_edesc_alloc(req, encrypt);
1441         if (IS_ERR(edesc))
1442                 return PTR_ERR(edesc);
1443
1444         ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req);
1445         if (!ret) {
1446                 ret = -EINPROGRESS;
1447         } else {
1448                 skcipher_unmap(ctx->qidev, edesc, req);
1449                 qi_cache_free(edesc);
1450         }
1451
1452         return ret;
1453 }
1454
1455 static int skcipher_encrypt(struct skcipher_request *req)
1456 {
1457         return skcipher_crypt(req, true);
1458 }
1459
1460 static int skcipher_decrypt(struct skcipher_request *req)
1461 {
1462         return skcipher_crypt(req, false);
1463 }
1464
1465 static struct caam_skcipher_alg driver_algs[] = {
1466         {
1467                 .skcipher = {
1468                         .base = {
1469                                 .cra_name = "cbc(aes)",
1470                                 .cra_driver_name = "cbc-aes-caam-qi",
1471                                 .cra_blocksize = AES_BLOCK_SIZE,
1472                         },
1473                         .setkey = aes_skcipher_setkey,
1474                         .encrypt = skcipher_encrypt,
1475                         .decrypt = skcipher_decrypt,
1476                         .min_keysize = AES_MIN_KEY_SIZE,
1477                         .max_keysize = AES_MAX_KEY_SIZE,
1478                         .ivsize = AES_BLOCK_SIZE,
1479                 },
1480                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1481         },
1482         {
1483                 .skcipher = {
1484                         .base = {
1485                                 .cra_name = "cbc(des3_ede)",
1486                                 .cra_driver_name = "cbc-3des-caam-qi",
1487                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1488                         },
1489                         .setkey = des3_skcipher_setkey,
1490                         .encrypt = skcipher_encrypt,
1491                         .decrypt = skcipher_decrypt,
1492                         .min_keysize = DES3_EDE_KEY_SIZE,
1493                         .max_keysize = DES3_EDE_KEY_SIZE,
1494                         .ivsize = DES3_EDE_BLOCK_SIZE,
1495                 },
1496                 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1497         },
1498         {
1499                 .skcipher = {
1500                         .base = {
1501                                 .cra_name = "cbc(des)",
1502                                 .cra_driver_name = "cbc-des-caam-qi",
1503                                 .cra_blocksize = DES_BLOCK_SIZE,
1504                         },
1505                         .setkey = des_skcipher_setkey,
1506                         .encrypt = skcipher_encrypt,
1507                         .decrypt = skcipher_decrypt,
1508                         .min_keysize = DES_KEY_SIZE,
1509                         .max_keysize = DES_KEY_SIZE,
1510                         .ivsize = DES_BLOCK_SIZE,
1511                 },
1512                 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1513         },
1514         {
1515                 .skcipher = {
1516                         .base = {
1517                                 .cra_name = "ctr(aes)",
1518                                 .cra_driver_name = "ctr-aes-caam-qi",
1519                                 .cra_blocksize = 1,
1520                         },
1521                         .setkey = ctr_skcipher_setkey,
1522                         .encrypt = skcipher_encrypt,
1523                         .decrypt = skcipher_decrypt,
1524                         .min_keysize = AES_MIN_KEY_SIZE,
1525                         .max_keysize = AES_MAX_KEY_SIZE,
1526                         .ivsize = AES_BLOCK_SIZE,
1527                         .chunksize = AES_BLOCK_SIZE,
1528                 },
1529                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1530                                         OP_ALG_AAI_CTR_MOD128,
1531         },
1532         {
1533                 .skcipher = {
1534                         .base = {
1535                                 .cra_name = "rfc3686(ctr(aes))",
1536                                 .cra_driver_name = "rfc3686-ctr-aes-caam-qi",
1537                                 .cra_blocksize = 1,
1538                         },
1539                         .setkey = rfc3686_skcipher_setkey,
1540                         .encrypt = skcipher_encrypt,
1541                         .decrypt = skcipher_decrypt,
1542                         .min_keysize = AES_MIN_KEY_SIZE +
1543                                        CTR_RFC3686_NONCE_SIZE,
1544                         .max_keysize = AES_MAX_KEY_SIZE +
1545                                        CTR_RFC3686_NONCE_SIZE,
1546                         .ivsize = CTR_RFC3686_IV_SIZE,
1547                         .chunksize = AES_BLOCK_SIZE,
1548                 },
1549                 .caam = {
1550                         .class1_alg_type = OP_ALG_ALGSEL_AES |
1551                                            OP_ALG_AAI_CTR_MOD128,
1552                         .rfc3686 = true,
1553                 },
1554         },
1555         {
1556                 .skcipher = {
1557                         .base = {
1558                                 .cra_name = "xts(aes)",
1559                                 .cra_driver_name = "xts-aes-caam-qi",
1560                                 .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
1561                                 .cra_blocksize = AES_BLOCK_SIZE,
1562                         },
1563                         .setkey = xts_skcipher_setkey,
1564                         .encrypt = skcipher_encrypt,
1565                         .decrypt = skcipher_decrypt,
1566                         .min_keysize = 2 * AES_MIN_KEY_SIZE,
1567                         .max_keysize = 2 * AES_MAX_KEY_SIZE,
1568                         .ivsize = AES_BLOCK_SIZE,
1569                 },
1570                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
1571         },
1572 };
1573
1574 static struct caam_aead_alg driver_aeads[] = {
1575         {
1576                 .aead = {
1577                         .base = {
1578                                 .cra_name = "rfc4106(gcm(aes))",
1579                                 .cra_driver_name = "rfc4106-gcm-aes-caam-qi",
1580                                 .cra_blocksize = 1,
1581                         },
1582                         .setkey = rfc4106_setkey,
1583                         .setauthsize = rfc4106_setauthsize,
1584                         .encrypt = ipsec_gcm_encrypt,
1585                         .decrypt = ipsec_gcm_decrypt,
1586                         .ivsize = 8,
1587                         .maxauthsize = AES_BLOCK_SIZE,
1588                 },
1589                 .caam = {
1590                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1591                         .nodkp = true,
1592                 },
1593         },
1594         {
1595                 .aead = {
1596                         .base = {
1597                                 .cra_name = "rfc4543(gcm(aes))",
1598                                 .cra_driver_name = "rfc4543-gcm-aes-caam-qi",
1599                                 .cra_blocksize = 1,
1600                         },
1601                         .setkey = rfc4543_setkey,
1602                         .setauthsize = rfc4543_setauthsize,
1603                         .encrypt = ipsec_gcm_encrypt,
1604                         .decrypt = ipsec_gcm_decrypt,
1605                         .ivsize = 8,
1606                         .maxauthsize = AES_BLOCK_SIZE,
1607                 },
1608                 .caam = {
1609                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1610                         .nodkp = true,
1611                 },
1612         },
1613         /* Galois Counter Mode */
1614         {
1615                 .aead = {
1616                         .base = {
1617                                 .cra_name = "gcm(aes)",
1618                                 .cra_driver_name = "gcm-aes-caam-qi",
1619                                 .cra_blocksize = 1,
1620                         },
1621                         .setkey = gcm_setkey,
1622                         .setauthsize = gcm_setauthsize,
1623                         .encrypt = aead_encrypt,
1624                         .decrypt = aead_decrypt,
1625                         .ivsize = 12,
1626                         .maxauthsize = AES_BLOCK_SIZE,
1627                 },
1628                 .caam = {
1629                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1630                         .nodkp = true,
1631                 }
1632         },
1633         /* single-pass ipsec_esp descriptor */
1634         {
1635                 .aead = {
1636                         .base = {
1637                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
1638                                 .cra_driver_name = "authenc-hmac-md5-"
1639                                                    "cbc-aes-caam-qi",
1640                                 .cra_blocksize = AES_BLOCK_SIZE,
1641                         },
1642                         .setkey = aead_setkey,
1643                         .setauthsize = aead_setauthsize,
1644                         .encrypt = aead_encrypt,
1645                         .decrypt = aead_decrypt,
1646                         .ivsize = AES_BLOCK_SIZE,
1647                         .maxauthsize = MD5_DIGEST_SIZE,
1648                 },
1649                 .caam = {
1650                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1651                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1652                                            OP_ALG_AAI_HMAC_PRECOMP,
1653                 }
1654         },
1655         {
1656                 .aead = {
1657                         .base = {
1658                                 .cra_name = "echainiv(authenc(hmac(md5),"
1659                                             "cbc(aes)))",
1660                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
1661                                                    "cbc-aes-caam-qi",
1662                                 .cra_blocksize = AES_BLOCK_SIZE,
1663                         },
1664                         .setkey = aead_setkey,
1665                         .setauthsize = aead_setauthsize,
1666                         .encrypt = aead_encrypt,
1667                         .decrypt = aead_decrypt,
1668                         .ivsize = AES_BLOCK_SIZE,
1669                         .maxauthsize = MD5_DIGEST_SIZE,
1670                 },
1671                 .caam = {
1672                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1673                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1674                                            OP_ALG_AAI_HMAC_PRECOMP,
1675                         .geniv = true,
1676                 }
1677         },
1678         {
1679                 .aead = {
1680                         .base = {
1681                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1682                                 .cra_driver_name = "authenc-hmac-sha1-"
1683                                                    "cbc-aes-caam-qi",
1684                                 .cra_blocksize = AES_BLOCK_SIZE,
1685                         },
1686                         .setkey = aead_setkey,
1687                         .setauthsize = aead_setauthsize,
1688                         .encrypt = aead_encrypt,
1689                         .decrypt = aead_decrypt,
1690                         .ivsize = AES_BLOCK_SIZE,
1691                         .maxauthsize = SHA1_DIGEST_SIZE,
1692                 },
1693                 .caam = {
1694                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1695                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1696                                            OP_ALG_AAI_HMAC_PRECOMP,
1697                 }
1698         },
1699         {
1700                 .aead = {
1701                         .base = {
1702                                 .cra_name = "echainiv(authenc(hmac(sha1),"
1703                                             "cbc(aes)))",
1704                                 .cra_driver_name = "echainiv-authenc-"
1705                                                    "hmac-sha1-cbc-aes-caam-qi",
1706                                 .cra_blocksize = AES_BLOCK_SIZE,
1707                         },
1708                         .setkey = aead_setkey,
1709                         .setauthsize = aead_setauthsize,
1710                         .encrypt = aead_encrypt,
1711                         .decrypt = aead_decrypt,
1712                         .ivsize = AES_BLOCK_SIZE,
1713                         .maxauthsize = SHA1_DIGEST_SIZE,
1714                 },
1715                 .caam = {
1716                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1717                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1718                                            OP_ALG_AAI_HMAC_PRECOMP,
1719                         .geniv = true,
1720                 },
1721         },
1722         {
1723                 .aead = {
1724                         .base = {
1725                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
1726                                 .cra_driver_name = "authenc-hmac-sha224-"
1727                                                    "cbc-aes-caam-qi",
1728                                 .cra_blocksize = AES_BLOCK_SIZE,
1729                         },
1730                         .setkey = aead_setkey,
1731                         .setauthsize = aead_setauthsize,
1732                         .encrypt = aead_encrypt,
1733                         .decrypt = aead_decrypt,
1734                         .ivsize = AES_BLOCK_SIZE,
1735                         .maxauthsize = SHA224_DIGEST_SIZE,
1736                 },
1737                 .caam = {
1738                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1739                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1740                                            OP_ALG_AAI_HMAC_PRECOMP,
1741                 }
1742         },
1743         {
1744                 .aead = {
1745                         .base = {
1746                                 .cra_name = "echainiv(authenc(hmac(sha224),"
1747                                             "cbc(aes)))",
1748                                 .cra_driver_name = "echainiv-authenc-"
1749                                                    "hmac-sha224-cbc-aes-caam-qi",
1750                                 .cra_blocksize = AES_BLOCK_SIZE,
1751                         },
1752                         .setkey = aead_setkey,
1753                         .setauthsize = aead_setauthsize,
1754                         .encrypt = aead_encrypt,
1755                         .decrypt = aead_decrypt,
1756                         .ivsize = AES_BLOCK_SIZE,
1757                         .maxauthsize = SHA224_DIGEST_SIZE,
1758                 },
1759                 .caam = {
1760                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1761                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1762                                            OP_ALG_AAI_HMAC_PRECOMP,
1763                         .geniv = true,
1764                 }
1765         },
1766         {
1767                 .aead = {
1768                         .base = {
1769                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1770                                 .cra_driver_name = "authenc-hmac-sha256-"
1771                                                    "cbc-aes-caam-qi",
1772                                 .cra_blocksize = AES_BLOCK_SIZE,
1773                         },
1774                         .setkey = aead_setkey,
1775                         .setauthsize = aead_setauthsize,
1776                         .encrypt = aead_encrypt,
1777                         .decrypt = aead_decrypt,
1778                         .ivsize = AES_BLOCK_SIZE,
1779                         .maxauthsize = SHA256_DIGEST_SIZE,
1780                 },
1781                 .caam = {
1782                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1783                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1784                                            OP_ALG_AAI_HMAC_PRECOMP,
1785                 }
1786         },
1787         {
1788                 .aead = {
1789                         .base = {
1790                                 .cra_name = "echainiv(authenc(hmac(sha256),"
1791                                             "cbc(aes)))",
1792                                 .cra_driver_name = "echainiv-authenc-"
1793                                                    "hmac-sha256-cbc-aes-"
1794                                                    "caam-qi",
1795                                 .cra_blocksize = AES_BLOCK_SIZE,
1796                         },
1797                         .setkey = aead_setkey,
1798                         .setauthsize = aead_setauthsize,
1799                         .encrypt = aead_encrypt,
1800                         .decrypt = aead_decrypt,
1801                         .ivsize = AES_BLOCK_SIZE,
1802                         .maxauthsize = SHA256_DIGEST_SIZE,
1803                 },
1804                 .caam = {
1805                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1806                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1807                                            OP_ALG_AAI_HMAC_PRECOMP,
1808                         .geniv = true,
1809                 }
1810         },
1811         {
1812                 .aead = {
1813                         .base = {
1814                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
1815                                 .cra_driver_name = "authenc-hmac-sha384-"
1816                                                    "cbc-aes-caam-qi",
1817                                 .cra_blocksize = AES_BLOCK_SIZE,
1818                         },
1819                         .setkey = aead_setkey,
1820                         .setauthsize = aead_setauthsize,
1821                         .encrypt = aead_encrypt,
1822                         .decrypt = aead_decrypt,
1823                         .ivsize = AES_BLOCK_SIZE,
1824                         .maxauthsize = SHA384_DIGEST_SIZE,
1825                 },
1826                 .caam = {
1827                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1828                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1829                                            OP_ALG_AAI_HMAC_PRECOMP,
1830                 }
1831         },
1832         {
1833                 .aead = {
1834                         .base = {
1835                                 .cra_name = "echainiv(authenc(hmac(sha384),"
1836                                             "cbc(aes)))",
1837                                 .cra_driver_name = "echainiv-authenc-"
1838                                                    "hmac-sha384-cbc-aes-"
1839                                                    "caam-qi",
1840                                 .cra_blocksize = AES_BLOCK_SIZE,
1841                         },
1842                         .setkey = aead_setkey,
1843                         .setauthsize = aead_setauthsize,
1844                         .encrypt = aead_encrypt,
1845                         .decrypt = aead_decrypt,
1846                         .ivsize = AES_BLOCK_SIZE,
1847                         .maxauthsize = SHA384_DIGEST_SIZE,
1848                 },
1849                 .caam = {
1850                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1851                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1852                                            OP_ALG_AAI_HMAC_PRECOMP,
1853                         .geniv = true,
1854                 }
1855         },
1856         {
1857                 .aead = {
1858                         .base = {
1859                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1860                                 .cra_driver_name = "authenc-hmac-sha512-"
1861                                                    "cbc-aes-caam-qi",
1862                                 .cra_blocksize = AES_BLOCK_SIZE,
1863                         },
1864                         .setkey = aead_setkey,
1865                         .setauthsize = aead_setauthsize,
1866                         .encrypt = aead_encrypt,
1867                         .decrypt = aead_decrypt,
1868                         .ivsize = AES_BLOCK_SIZE,
1869                         .maxauthsize = SHA512_DIGEST_SIZE,
1870                 },
1871                 .caam = {
1872                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1873                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1874                                            OP_ALG_AAI_HMAC_PRECOMP,
1875                 }
1876         },
1877         {
1878                 .aead = {
1879                         .base = {
1880                                 .cra_name = "echainiv(authenc(hmac(sha512),"
1881                                             "cbc(aes)))",
1882                                 .cra_driver_name = "echainiv-authenc-"
1883                                                    "hmac-sha512-cbc-aes-"
1884                                                    "caam-qi",
1885                                 .cra_blocksize = AES_BLOCK_SIZE,
1886                         },
1887                         .setkey = aead_setkey,
1888                         .setauthsize = aead_setauthsize,
1889                         .encrypt = aead_encrypt,
1890                         .decrypt = aead_decrypt,
1891                         .ivsize = AES_BLOCK_SIZE,
1892                         .maxauthsize = SHA512_DIGEST_SIZE,
1893                 },
1894                 .caam = {
1895                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1896                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1897                                            OP_ALG_AAI_HMAC_PRECOMP,
1898                         .geniv = true,
1899                 }
1900         },
1901         {
1902                 .aead = {
1903                         .base = {
1904                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
1905                                 .cra_driver_name = "authenc-hmac-md5-"
1906                                                    "cbc-des3_ede-caam-qi",
1907                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1908                         },
1909                         .setkey = des3_aead_setkey,
1910                         .setauthsize = aead_setauthsize,
1911                         .encrypt = aead_encrypt,
1912                         .decrypt = aead_decrypt,
1913                         .ivsize = DES3_EDE_BLOCK_SIZE,
1914                         .maxauthsize = MD5_DIGEST_SIZE,
1915                 },
1916                 .caam = {
1917                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1918                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1919                                            OP_ALG_AAI_HMAC_PRECOMP,
1920                 }
1921         },
1922         {
1923                 .aead = {
1924                         .base = {
1925                                 .cra_name = "echainiv(authenc(hmac(md5),"
1926                                             "cbc(des3_ede)))",
1927                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
1928                                                    "cbc-des3_ede-caam-qi",
1929                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1930                         },
1931                         .setkey = des3_aead_setkey,
1932                         .setauthsize = aead_setauthsize,
1933                         .encrypt = aead_encrypt,
1934                         .decrypt = aead_decrypt,
1935                         .ivsize = DES3_EDE_BLOCK_SIZE,
1936                         .maxauthsize = MD5_DIGEST_SIZE,
1937                 },
1938                 .caam = {
1939                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1940                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1941                                            OP_ALG_AAI_HMAC_PRECOMP,
1942                         .geniv = true,
1943                 }
1944         },
1945         {
1946                 .aead = {
1947                         .base = {
1948                                 .cra_name = "authenc(hmac(sha1),"
1949                                             "cbc(des3_ede))",
1950                                 .cra_driver_name = "authenc-hmac-sha1-"
1951                                                    "cbc-des3_ede-caam-qi",
1952                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1953                         },
1954                         .setkey = des3_aead_setkey,
1955                         .setauthsize = aead_setauthsize,
1956                         .encrypt = aead_encrypt,
1957                         .decrypt = aead_decrypt,
1958                         .ivsize = DES3_EDE_BLOCK_SIZE,
1959                         .maxauthsize = SHA1_DIGEST_SIZE,
1960                 },
1961                 .caam = {
1962                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1963                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1964                                            OP_ALG_AAI_HMAC_PRECOMP,
1965                 },
1966         },
1967         {
1968                 .aead = {
1969                         .base = {
1970                                 .cra_name = "echainiv(authenc(hmac(sha1),"
1971                                             "cbc(des3_ede)))",
1972                                 .cra_driver_name = "echainiv-authenc-"
1973                                                    "hmac-sha1-"
1974                                                    "cbc-des3_ede-caam-qi",
1975                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1976                         },
1977                         .setkey = des3_aead_setkey,
1978                         .setauthsize = aead_setauthsize,
1979                         .encrypt = aead_encrypt,
1980                         .decrypt = aead_decrypt,
1981                         .ivsize = DES3_EDE_BLOCK_SIZE,
1982                         .maxauthsize = SHA1_DIGEST_SIZE,
1983                 },
1984                 .caam = {
1985                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1986                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1987                                            OP_ALG_AAI_HMAC_PRECOMP,
1988                         .geniv = true,
1989                 }
1990         },
1991         {
1992                 .aead = {
1993                         .base = {
1994                                 .cra_name = "authenc(hmac(sha224),"
1995                                             "cbc(des3_ede))",
1996                                 .cra_driver_name = "authenc-hmac-sha224-"
1997                                                    "cbc-des3_ede-caam-qi",
1998                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1999                         },
2000                         .setkey = des3_aead_setkey,
2001                         .setauthsize = aead_setauthsize,
2002                         .encrypt = aead_encrypt,
2003                         .decrypt = aead_decrypt,
2004                         .ivsize = DES3_EDE_BLOCK_SIZE,
2005                         .maxauthsize = SHA224_DIGEST_SIZE,
2006                 },
2007                 .caam = {
2008                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2009                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2010                                            OP_ALG_AAI_HMAC_PRECOMP,
2011                 },
2012         },
2013         {
2014                 .aead = {
2015                         .base = {
2016                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2017                                             "cbc(des3_ede)))",
2018                                 .cra_driver_name = "echainiv-authenc-"
2019                                                    "hmac-sha224-"
2020                                                    "cbc-des3_ede-caam-qi",
2021                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2022                         },
2023                         .setkey = des3_aead_setkey,
2024                         .setauthsize = aead_setauthsize,
2025                         .encrypt = aead_encrypt,
2026                         .decrypt = aead_decrypt,
2027                         .ivsize = DES3_EDE_BLOCK_SIZE,
2028                         .maxauthsize = SHA224_DIGEST_SIZE,
2029                 },
2030                 .caam = {
2031                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2032                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2033                                            OP_ALG_AAI_HMAC_PRECOMP,
2034                         .geniv = true,
2035                 }
2036         },
2037         {
2038                 .aead = {
2039                         .base = {
2040                                 .cra_name = "authenc(hmac(sha256),"
2041                                             "cbc(des3_ede))",
2042                                 .cra_driver_name = "authenc-hmac-sha256-"
2043                                                    "cbc-des3_ede-caam-qi",
2044                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2045                         },
2046                         .setkey = des3_aead_setkey,
2047                         .setauthsize = aead_setauthsize,
2048                         .encrypt = aead_encrypt,
2049                         .decrypt = aead_decrypt,
2050                         .ivsize = DES3_EDE_BLOCK_SIZE,
2051                         .maxauthsize = SHA256_DIGEST_SIZE,
2052                 },
2053                 .caam = {
2054                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2055                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2056                                            OP_ALG_AAI_HMAC_PRECOMP,
2057                 },
2058         },
2059         {
2060                 .aead = {
2061                         .base = {
2062                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2063                                             "cbc(des3_ede)))",
2064                                 .cra_driver_name = "echainiv-authenc-"
2065                                                    "hmac-sha256-"
2066                                                    "cbc-des3_ede-caam-qi",
2067                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2068                         },
2069                         .setkey = des3_aead_setkey,
2070                         .setauthsize = aead_setauthsize,
2071                         .encrypt = aead_encrypt,
2072                         .decrypt = aead_decrypt,
2073                         .ivsize = DES3_EDE_BLOCK_SIZE,
2074                         .maxauthsize = SHA256_DIGEST_SIZE,
2075                 },
2076                 .caam = {
2077                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2078                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2079                                            OP_ALG_AAI_HMAC_PRECOMP,
2080                         .geniv = true,
2081                 }
2082         },
2083         {
2084                 .aead = {
2085                         .base = {
2086                                 .cra_name = "authenc(hmac(sha384),"
2087                                             "cbc(des3_ede))",
2088                                 .cra_driver_name = "authenc-hmac-sha384-"
2089                                                    "cbc-des3_ede-caam-qi",
2090                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2091                         },
2092                         .setkey = des3_aead_setkey,
2093                         .setauthsize = aead_setauthsize,
2094                         .encrypt = aead_encrypt,
2095                         .decrypt = aead_decrypt,
2096                         .ivsize = DES3_EDE_BLOCK_SIZE,
2097                         .maxauthsize = SHA384_DIGEST_SIZE,
2098                 },
2099                 .caam = {
2100                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2101                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2102                                            OP_ALG_AAI_HMAC_PRECOMP,
2103                 },
2104         },
2105         {
2106                 .aead = {
2107                         .base = {
2108                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2109                                             "cbc(des3_ede)))",
2110                                 .cra_driver_name = "echainiv-authenc-"
2111                                                    "hmac-sha384-"
2112                                                    "cbc-des3_ede-caam-qi",
2113                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2114                         },
2115                         .setkey = des3_aead_setkey,
2116                         .setauthsize = aead_setauthsize,
2117                         .encrypt = aead_encrypt,
2118                         .decrypt = aead_decrypt,
2119                         .ivsize = DES3_EDE_BLOCK_SIZE,
2120                         .maxauthsize = SHA384_DIGEST_SIZE,
2121                 },
2122                 .caam = {
2123                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2124                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2125                                            OP_ALG_AAI_HMAC_PRECOMP,
2126                         .geniv = true,
2127                 }
2128         },
2129         {
2130                 .aead = {
2131                         .base = {
2132                                 .cra_name = "authenc(hmac(sha512),"
2133                                             "cbc(des3_ede))",
2134                                 .cra_driver_name = "authenc-hmac-sha512-"
2135                                                    "cbc-des3_ede-caam-qi",
2136                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2137                         },
2138                         .setkey = des3_aead_setkey,
2139                         .setauthsize = aead_setauthsize,
2140                         .encrypt = aead_encrypt,
2141                         .decrypt = aead_decrypt,
2142                         .ivsize = DES3_EDE_BLOCK_SIZE,
2143                         .maxauthsize = SHA512_DIGEST_SIZE,
2144                 },
2145                 .caam = {
2146                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2147                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2148                                            OP_ALG_AAI_HMAC_PRECOMP,
2149                 },
2150         },
2151         {
2152                 .aead = {
2153                         .base = {
2154                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2155                                             "cbc(des3_ede)))",
2156                                 .cra_driver_name = "echainiv-authenc-"
2157                                                    "hmac-sha512-"
2158                                                    "cbc-des3_ede-caam-qi",
2159                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2160                         },
2161                         .setkey = des3_aead_setkey,
2162                         .setauthsize = aead_setauthsize,
2163                         .encrypt = aead_encrypt,
2164                         .decrypt = aead_decrypt,
2165                         .ivsize = DES3_EDE_BLOCK_SIZE,
2166                         .maxauthsize = SHA512_DIGEST_SIZE,
2167                 },
2168                 .caam = {
2169                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2170                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2171                                            OP_ALG_AAI_HMAC_PRECOMP,
2172                         .geniv = true,
2173                 }
2174         },
2175         {
2176                 .aead = {
2177                         .base = {
2178                                 .cra_name = "authenc(hmac(md5),cbc(des))",
2179                                 .cra_driver_name = "authenc-hmac-md5-"
2180                                                    "cbc-des-caam-qi",
2181                                 .cra_blocksize = DES_BLOCK_SIZE,
2182                         },
2183                         .setkey = aead_setkey,
2184                         .setauthsize = aead_setauthsize,
2185                         .encrypt = aead_encrypt,
2186                         .decrypt = aead_decrypt,
2187                         .ivsize = DES_BLOCK_SIZE,
2188                         .maxauthsize = MD5_DIGEST_SIZE,
2189                 },
2190                 .caam = {
2191                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2192                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2193                                            OP_ALG_AAI_HMAC_PRECOMP,
2194                 },
2195         },
2196         {
2197                 .aead = {
2198                         .base = {
2199                                 .cra_name = "echainiv(authenc(hmac(md5),"
2200                                             "cbc(des)))",
2201                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2202                                                    "cbc-des-caam-qi",
2203                                 .cra_blocksize = DES_BLOCK_SIZE,
2204                         },
2205                         .setkey = aead_setkey,
2206                         .setauthsize = aead_setauthsize,
2207                         .encrypt = aead_encrypt,
2208                         .decrypt = aead_decrypt,
2209                         .ivsize = DES_BLOCK_SIZE,
2210                         .maxauthsize = MD5_DIGEST_SIZE,
2211                 },
2212                 .caam = {
2213                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2214                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2215                                            OP_ALG_AAI_HMAC_PRECOMP,
2216                         .geniv = true,
2217                 }
2218         },
2219         {
2220                 .aead = {
2221                         .base = {
2222                                 .cra_name = "authenc(hmac(sha1),cbc(des))",
2223                                 .cra_driver_name = "authenc-hmac-sha1-"
2224                                                    "cbc-des-caam-qi",
2225                                 .cra_blocksize = DES_BLOCK_SIZE,
2226                         },
2227                         .setkey = aead_setkey,
2228                         .setauthsize = aead_setauthsize,
2229                         .encrypt = aead_encrypt,
2230                         .decrypt = aead_decrypt,
2231                         .ivsize = DES_BLOCK_SIZE,
2232                         .maxauthsize = SHA1_DIGEST_SIZE,
2233                 },
2234                 .caam = {
2235                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2236                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2237                                            OP_ALG_AAI_HMAC_PRECOMP,
2238                 },
2239         },
2240         {
2241                 .aead = {
2242                         .base = {
2243                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2244                                             "cbc(des)))",
2245                                 .cra_driver_name = "echainiv-authenc-"
2246                                                    "hmac-sha1-cbc-des-caam-qi",
2247                                 .cra_blocksize = DES_BLOCK_SIZE,
2248                         },
2249                         .setkey = aead_setkey,
2250                         .setauthsize = aead_setauthsize,
2251                         .encrypt = aead_encrypt,
2252                         .decrypt = aead_decrypt,
2253                         .ivsize = DES_BLOCK_SIZE,
2254                         .maxauthsize = SHA1_DIGEST_SIZE,
2255                 },
2256                 .caam = {
2257                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2258                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2259                                            OP_ALG_AAI_HMAC_PRECOMP,
2260                         .geniv = true,
2261                 }
2262         },
2263         {
2264                 .aead = {
2265                         .base = {
2266                                 .cra_name = "authenc(hmac(sha224),cbc(des))",
2267                                 .cra_driver_name = "authenc-hmac-sha224-"
2268                                                    "cbc-des-caam-qi",
2269                                 .cra_blocksize = DES_BLOCK_SIZE,
2270                         },
2271                         .setkey = aead_setkey,
2272                         .setauthsize = aead_setauthsize,
2273                         .encrypt = aead_encrypt,
2274                         .decrypt = aead_decrypt,
2275                         .ivsize = DES_BLOCK_SIZE,
2276                         .maxauthsize = SHA224_DIGEST_SIZE,
2277                 },
2278                 .caam = {
2279                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2280                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2281                                            OP_ALG_AAI_HMAC_PRECOMP,
2282                 },
2283         },
2284         {
2285                 .aead = {
2286                         .base = {
2287                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2288                                             "cbc(des)))",
2289                                 .cra_driver_name = "echainiv-authenc-"
2290                                                    "hmac-sha224-cbc-des-"
2291                                                    "caam-qi",
2292                                 .cra_blocksize = DES_BLOCK_SIZE,
2293                         },
2294                         .setkey = aead_setkey,
2295                         .setauthsize = aead_setauthsize,
2296                         .encrypt = aead_encrypt,
2297                         .decrypt = aead_decrypt,
2298                         .ivsize = DES_BLOCK_SIZE,
2299                         .maxauthsize = SHA224_DIGEST_SIZE,
2300                 },
2301                 .caam = {
2302                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2303                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2304                                            OP_ALG_AAI_HMAC_PRECOMP,
2305                         .geniv = true,
2306                 }
2307         },
2308         {
2309                 .aead = {
2310                         .base = {
2311                                 .cra_name = "authenc(hmac(sha256),cbc(des))",
2312                                 .cra_driver_name = "authenc-hmac-sha256-"
2313                                                    "cbc-des-caam-qi",
2314                                 .cra_blocksize = DES_BLOCK_SIZE,
2315                         },
2316                         .setkey = aead_setkey,
2317                         .setauthsize = aead_setauthsize,
2318                         .encrypt = aead_encrypt,
2319                         .decrypt = aead_decrypt,
2320                         .ivsize = DES_BLOCK_SIZE,
2321                         .maxauthsize = SHA256_DIGEST_SIZE,
2322                 },
2323                 .caam = {
2324                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2325                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2326                                            OP_ALG_AAI_HMAC_PRECOMP,
2327                 },
2328         },
2329         {
2330                 .aead = {
2331                         .base = {
2332                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2333                                             "cbc(des)))",
2334                                 .cra_driver_name = "echainiv-authenc-"
2335                                                    "hmac-sha256-cbc-des-"
2336                                                    "caam-qi",
2337                                 .cra_blocksize = DES_BLOCK_SIZE,
2338                         },
2339                         .setkey = aead_setkey,
2340                         .setauthsize = aead_setauthsize,
2341                         .encrypt = aead_encrypt,
2342                         .decrypt = aead_decrypt,
2343                         .ivsize = DES_BLOCK_SIZE,
2344                         .maxauthsize = SHA256_DIGEST_SIZE,
2345                 },
2346                 .caam = {
2347                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2348                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2349                                            OP_ALG_AAI_HMAC_PRECOMP,
2350                         .geniv = true,
2351                 },
2352         },
2353         {
2354                 .aead = {
2355                         .base = {
2356                                 .cra_name = "authenc(hmac(sha384),cbc(des))",
2357                                 .cra_driver_name = "authenc-hmac-sha384-"
2358                                                    "cbc-des-caam-qi",
2359                                 .cra_blocksize = DES_BLOCK_SIZE,
2360                         },
2361                         .setkey = aead_setkey,
2362                         .setauthsize = aead_setauthsize,
2363                         .encrypt = aead_encrypt,
2364                         .decrypt = aead_decrypt,
2365                         .ivsize = DES_BLOCK_SIZE,
2366                         .maxauthsize = SHA384_DIGEST_SIZE,
2367                 },
2368                 .caam = {
2369                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2370                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2371                                            OP_ALG_AAI_HMAC_PRECOMP,
2372                 },
2373         },
2374         {
2375                 .aead = {
2376                         .base = {
2377                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2378                                             "cbc(des)))",
2379                                 .cra_driver_name = "echainiv-authenc-"
2380                                                    "hmac-sha384-cbc-des-"
2381                                                    "caam-qi",
2382                                 .cra_blocksize = DES_BLOCK_SIZE,
2383                         },
2384                         .setkey = aead_setkey,
2385                         .setauthsize = aead_setauthsize,
2386                         .encrypt = aead_encrypt,
2387                         .decrypt = aead_decrypt,
2388                         .ivsize = DES_BLOCK_SIZE,
2389                         .maxauthsize = SHA384_DIGEST_SIZE,
2390                 },
2391                 .caam = {
2392                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2393                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2394                                            OP_ALG_AAI_HMAC_PRECOMP,
2395                         .geniv = true,
2396                 }
2397         },
2398         {
2399                 .aead = {
2400                         .base = {
2401                                 .cra_name = "authenc(hmac(sha512),cbc(des))",
2402                                 .cra_driver_name = "authenc-hmac-sha512-"
2403                                                    "cbc-des-caam-qi",
2404                                 .cra_blocksize = DES_BLOCK_SIZE,
2405                         },
2406                         .setkey = aead_setkey,
2407                         .setauthsize = aead_setauthsize,
2408                         .encrypt = aead_encrypt,
2409                         .decrypt = aead_decrypt,
2410                         .ivsize = DES_BLOCK_SIZE,
2411                         .maxauthsize = SHA512_DIGEST_SIZE,
2412                 },
2413                 .caam = {
2414                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2415                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2416                                            OP_ALG_AAI_HMAC_PRECOMP,
2417                 }
2418         },
2419         {
2420                 .aead = {
2421                         .base = {
2422                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2423                                             "cbc(des)))",
2424                                 .cra_driver_name = "echainiv-authenc-"
2425                                                    "hmac-sha512-cbc-des-"
2426                                                    "caam-qi",
2427                                 .cra_blocksize = DES_BLOCK_SIZE,
2428                         },
2429                         .setkey = aead_setkey,
2430                         .setauthsize = aead_setauthsize,
2431                         .encrypt = aead_encrypt,
2432                         .decrypt = aead_decrypt,
2433                         .ivsize = DES_BLOCK_SIZE,
2434                         .maxauthsize = SHA512_DIGEST_SIZE,
2435                 },
2436                 .caam = {
2437                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2438                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2439                                            OP_ALG_AAI_HMAC_PRECOMP,
2440                         .geniv = true,
2441                 }
2442         },
2443 };
2444
2445 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
2446                             bool uses_dkp)
2447 {
2448         struct caam_drv_private *priv;
2449         struct device *dev;
2450
2451         /*
2452          * distribute tfms across job rings to ensure in-order
2453          * crypto request processing per tfm
2454          */
2455         ctx->jrdev = caam_jr_alloc();
2456         if (IS_ERR(ctx->jrdev)) {
2457                 pr_err("Job Ring Device allocation for transform failed\n");
2458                 return PTR_ERR(ctx->jrdev);
2459         }
2460
2461         dev = ctx->jrdev->parent;
2462         priv = dev_get_drvdata(dev);
2463         if (priv->era >= 6 && uses_dkp)
2464                 ctx->dir = DMA_BIDIRECTIONAL;
2465         else
2466                 ctx->dir = DMA_TO_DEVICE;
2467
2468         ctx->key_dma = dma_map_single(dev, ctx->key, sizeof(ctx->key),
2469                                       ctx->dir);
2470         if (dma_mapping_error(dev, ctx->key_dma)) {
2471                 dev_err(dev, "unable to map key\n");
2472                 caam_jr_free(ctx->jrdev);
2473                 return -ENOMEM;
2474         }
2475
2476         /* copy descriptor header template value */
2477         ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
2478         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
2479
2480         ctx->qidev = dev;
2481
2482         spin_lock_init(&ctx->lock);
2483         ctx->drv_ctx[ENCRYPT] = NULL;
2484         ctx->drv_ctx[DECRYPT] = NULL;
2485
2486         return 0;
2487 }
2488
2489 static int caam_cra_init(struct crypto_skcipher *tfm)
2490 {
2491         struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
2492         struct caam_skcipher_alg *caam_alg =
2493                 container_of(alg, typeof(*caam_alg), skcipher);
2494         struct caam_ctx *ctx = crypto_skcipher_ctx(tfm);
2495         u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
2496         int ret = 0;
2497
2498         if (alg_aai == OP_ALG_AAI_XTS) {
2499                 const char *tfm_name = crypto_tfm_alg_name(&tfm->base);
2500                 struct crypto_skcipher *fallback;
2501
2502                 fallback = crypto_alloc_skcipher(tfm_name, 0,
2503                                                  CRYPTO_ALG_NEED_FALLBACK);
2504                 if (IS_ERR(fallback)) {
2505                         pr_err("Failed to allocate %s fallback: %ld\n",
2506                                tfm_name, PTR_ERR(fallback));
2507                         return PTR_ERR(fallback);
2508                 }
2509
2510                 ctx->fallback = fallback;
2511                 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx) +
2512                                             crypto_skcipher_reqsize(fallback));
2513         }
2514
2515         ret = caam_init_common(ctx, &caam_alg->caam, false);
2516         if (ret && ctx->fallback)
2517                 crypto_free_skcipher(ctx->fallback);
2518
2519         return ret;
2520 }
2521
2522 static int caam_aead_init(struct crypto_aead *tfm)
2523 {
2524         struct aead_alg *alg = crypto_aead_alg(tfm);
2525         struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg),
2526                                                       aead);
2527         struct caam_ctx *ctx = crypto_aead_ctx(tfm);
2528
2529         return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp);
2530 }
2531
2532 static void caam_exit_common(struct caam_ctx *ctx)
2533 {
2534         caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]);
2535         caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]);
2536
2537         dma_unmap_single(ctx->jrdev->parent, ctx->key_dma, sizeof(ctx->key),
2538                          ctx->dir);
2539
2540         caam_jr_free(ctx->jrdev);
2541 }
2542
2543 static void caam_cra_exit(struct crypto_skcipher *tfm)
2544 {
2545         struct caam_ctx *ctx = crypto_skcipher_ctx(tfm);
2546
2547         if (ctx->fallback)
2548                 crypto_free_skcipher(ctx->fallback);
2549         caam_exit_common(ctx);
2550 }
2551
2552 static void caam_aead_exit(struct crypto_aead *tfm)
2553 {
2554         caam_exit_common(crypto_aead_ctx(tfm));
2555 }
2556
2557 void caam_qi_algapi_exit(void)
2558 {
2559         int i;
2560
2561         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
2562                 struct caam_aead_alg *t_alg = driver_aeads + i;
2563
2564                 if (t_alg->registered)
2565                         crypto_unregister_aead(&t_alg->aead);
2566         }
2567
2568         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2569                 struct caam_skcipher_alg *t_alg = driver_algs + i;
2570
2571                 if (t_alg->registered)
2572                         crypto_unregister_skcipher(&t_alg->skcipher);
2573         }
2574 }
2575
2576 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
2577 {
2578         struct skcipher_alg *alg = &t_alg->skcipher;
2579
2580         alg->base.cra_module = THIS_MODULE;
2581         alg->base.cra_priority = CAAM_CRA_PRIORITY;
2582         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
2583         alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
2584                                 CRYPTO_ALG_KERN_DRIVER_ONLY);
2585
2586         alg->init = caam_cra_init;
2587         alg->exit = caam_cra_exit;
2588 }
2589
2590 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
2591 {
2592         struct aead_alg *alg = &t_alg->aead;
2593
2594         alg->base.cra_module = THIS_MODULE;
2595         alg->base.cra_priority = CAAM_CRA_PRIORITY;
2596         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
2597         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
2598                               CRYPTO_ALG_KERN_DRIVER_ONLY;
2599
2600         alg->init = caam_aead_init;
2601         alg->exit = caam_aead_exit;
2602 }
2603
2604 int caam_qi_algapi_init(struct device *ctrldev)
2605 {
2606         struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
2607         int i = 0, err = 0;
2608         u32 aes_vid, aes_inst, des_inst, md_vid, md_inst;
2609         unsigned int md_limit = SHA512_DIGEST_SIZE;
2610         bool registered = false;
2611
2612         /* Make sure this runs only on (DPAA 1.x) QI */
2613         if (!priv->qi_present || caam_dpaa2)
2614                 return 0;
2615
2616         /*
2617          * Register crypto algorithms the device supports.
2618          * First, detect presence and attributes of DES, AES, and MD blocks.
2619          */
2620         if (priv->era < 10) {
2621                 u32 cha_vid, cha_inst;
2622
2623                 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
2624                 aes_vid = cha_vid & CHA_ID_LS_AES_MASK;
2625                 md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
2626
2627                 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
2628                 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >>
2629                            CHA_ID_LS_DES_SHIFT;
2630                 aes_inst = cha_inst & CHA_ID_LS_AES_MASK;
2631                 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
2632         } else {
2633                 u32 aesa, mdha;
2634
2635                 aesa = rd_reg32(&priv->ctrl->vreg.aesa);
2636                 mdha = rd_reg32(&priv->ctrl->vreg.mdha);
2637
2638                 aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
2639                 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
2640
2641                 des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK;
2642                 aes_inst = aesa & CHA_VER_NUM_MASK;
2643                 md_inst = mdha & CHA_VER_NUM_MASK;
2644         }
2645
2646         /* If MD is present, limit digest size based on LP256 */
2647         if (md_inst && md_vid  == CHA_VER_VID_MD_LP256)
2648                 md_limit = SHA256_DIGEST_SIZE;
2649
2650         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2651                 struct caam_skcipher_alg *t_alg = driver_algs + i;
2652                 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
2653
2654                 /* Skip DES algorithms if not supported by device */
2655                 if (!des_inst &&
2656                     ((alg_sel == OP_ALG_ALGSEL_3DES) ||
2657                      (alg_sel == OP_ALG_ALGSEL_DES)))
2658                         continue;
2659
2660                 /* Skip AES algorithms if not supported by device */
2661                 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))
2662                         continue;
2663
2664                 caam_skcipher_alg_init(t_alg);
2665
2666                 err = crypto_register_skcipher(&t_alg->skcipher);
2667                 if (err) {
2668                         dev_warn(ctrldev, "%s alg registration failed\n",
2669                                  t_alg->skcipher.base.cra_driver_name);
2670                         continue;
2671                 }
2672
2673                 t_alg->registered = true;
2674                 registered = true;
2675         }
2676
2677         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
2678                 struct caam_aead_alg *t_alg = driver_aeads + i;
2679                 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
2680                                  OP_ALG_ALGSEL_MASK;
2681                 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
2682                                  OP_ALG_ALGSEL_MASK;
2683                 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
2684
2685                 /* Skip DES algorithms if not supported by device */
2686                 if (!des_inst &&
2687                     ((c1_alg_sel == OP_ALG_ALGSEL_3DES) ||
2688                      (c1_alg_sel == OP_ALG_ALGSEL_DES)))
2689                         continue;
2690
2691                 /* Skip AES algorithms if not supported by device */
2692                 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES))
2693                         continue;
2694
2695                 /*
2696                  * Check support for AES algorithms not available
2697                  * on LP devices.
2698                  */
2699                 if (aes_vid  == CHA_VER_VID_AES_LP && alg_aai == OP_ALG_AAI_GCM)
2700                         continue;
2701
2702                 /*
2703                  * Skip algorithms requiring message digests
2704                  * if MD or MD size is not supported by device.
2705                  */
2706                 if (c2_alg_sel &&
2707                     (!md_inst || (t_alg->aead.maxauthsize > md_limit)))
2708                         continue;
2709
2710                 caam_aead_alg_init(t_alg);
2711
2712                 err = crypto_register_aead(&t_alg->aead);
2713                 if (err) {
2714                         pr_warn("%s alg registration failed\n",
2715                                 t_alg->aead.base.cra_driver_name);
2716                         continue;
2717                 }
2718
2719                 t_alg->registered = true;
2720                 registered = true;
2721         }
2722
2723         if (registered)
2724                 dev_info(ctrldev, "algorithms registered in /proc/crypto\n");
2725
2726         return err;
2727 }