drivers/crypto/caam/caamalg_qi.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Freescale FSL CAAM support for crypto API over QI backend.
   4  * Based on caamalg.c
   5  *
   6  * Copyright 2013-2016 Freescale Semiconductor, Inc.
   7  * Copyright 2016-2019 NXP
   8  */
   9
  10 #include "compat.h"
  11 #include "ctrl.h"
  12 #include "regs.h"
  13 #include "intern.h"
  14 #include "desc_constr.h"
  15 #include "error.h"
  16 #include "sg_sw_qm.h"
  17 #include "key_gen.h"
  18 #include "qi.h"
  19 #include "jr.h"
  20 #include "caamalg_desc.h"
  21
  22 /*
  23  * crypto alg
  24  */
  25 #define CAAM_CRA_PRIORITY               2000
  26 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  27 #define CAAM_MAX_KEY_SIZE               (AES_MAX_KEY_SIZE + \
  28                                          SHA512_DIGEST_SIZE * 2)
  29
  30 #define DESC_MAX_USED_BYTES             (DESC_QI_AEAD_GIVENC_LEN + \
  31                                          CAAM_MAX_KEY_SIZE)
  32 #define DESC_MAX_USED_LEN               (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
  33
  34 struct caam_alg_entry {
  35         int class1_alg_type;
  36         int class2_alg_type;
  37         bool rfc3686;
  38         bool geniv;
  39         bool nodkp;
  40 };
  41
  42 struct caam_aead_alg {
  43         struct aead_alg aead;
  44         struct caam_alg_entry caam;
  45         bool registered;
  46 };
  47
  48 struct caam_skcipher_alg {
  49         struct skcipher_alg skcipher;
  50         struct caam_alg_entry caam;
  51         bool registered;
  52 };
  53
  54 /*
  55  * per-session context
  56  */
  57 struct caam_ctx {
  58         struct device *jrdev;
  59         u32 sh_desc_enc[DESC_MAX_USED_LEN];
  60         u32 sh_desc_dec[DESC_MAX_USED_LEN];
  61         u8 key[CAAM_MAX_KEY_SIZE];
  62         dma_addr_t key_dma;
  63         enum dma_data_direction dir;
  64         struct alginfo adata;
  65         struct alginfo cdata;
  66         unsigned int authsize;
  67         struct device *qidev;
  68         spinlock_t lock;        /* Protects multiple init of driver context */
  69         struct caam_drv_ctx *drv_ctx[NUM_OP];
  70 };
  71
  72 static int aead_set_sh_desc(struct crypto_aead *aead)
  73 {
  74         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
  75                                                  typeof(*alg), aead);
  76         struct caam_ctx *ctx = crypto_aead_ctx(aead);
  77         unsigned int ivsize = crypto_aead_ivsize(aead);
  78         u32 ctx1_iv_off = 0;
  79         u32 *nonce = NULL;
  80         unsigned int data_len[2];
  81         u32 inl_mask;
  82         const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
  83                                OP_ALG_AAI_CTR_MOD128);
  84         const bool is_rfc3686 = alg->caam.rfc3686;
  85         struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
  86
  87         if (!ctx->cdata.keylen || !ctx->authsize)
  88                 return 0;
  89
  90         /*
  91          * AES-CTR needs to load IV in CONTEXT1 reg
  92          * at an offset of 128bits (16bytes)
  93          * CONTEXT1[255:128] = IV
  94          */
  95         if (ctr_mode)
  96                 ctx1_iv_off = 16;
  97
  98         /*
  99          * RFC3686 specific:
 100          *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 101          */
 102         if (is_rfc3686) {
 103                 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 104                 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
 105                                 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
 106         }
 107
 108         /*
 109          * In case |user key| > |derived key|, using DKP<imm,imm> would result
 110          * in invalid opcodes (last bytes of user key) in the resulting
 111          * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
 112          * addresses are needed.
 113          */
 114         ctx->adata.key_virt = ctx->key;
 115         ctx->adata.key_dma = ctx->key_dma;
 116
 117         ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 118         ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 119
 120         data_len[0] = ctx->adata.keylen_pad;
 121         data_len[1] = ctx->cdata.keylen;
 122
 123         if (alg->caam.geniv)
 124                 goto skip_enc;
 125
 126         /* aead_encrypt shared descriptor */
 127         if (desc_inline_query(DESC_QI_AEAD_ENC_LEN +
 128                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 129                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 130                               ARRAY_SIZE(data_len)) < 0)
 131                 return -EINVAL;
 132
 133         ctx->adata.key_inline = !!(inl_mask & 1);
 134         ctx->cdata.key_inline = !!(inl_mask & 2);
 135
 136         cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata,
 137                                ivsize, ctx->authsize, is_rfc3686, nonce,
 138                                ctx1_iv_off, true, ctrlpriv->era);
 139
 140 skip_enc:
 141         /* aead_decrypt shared descriptor */
 142         if (desc_inline_query(DESC_QI_AEAD_DEC_LEN +
 143                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 144                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 145                               ARRAY_SIZE(data_len)) < 0)
 146                 return -EINVAL;
 147
 148         ctx->adata.key_inline = !!(inl_mask & 1);
 149         ctx->cdata.key_inline = !!(inl_mask & 2);
 150
 151         cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata,
 152                                ivsize, ctx->authsize, alg->caam.geniv,
 153                                is_rfc3686, nonce, ctx1_iv_off, true,
 154                                ctrlpriv->era);
 155
 156         if (!alg->caam.geniv)
 157                 goto skip_givenc;
 158
 159         /* aead_givencrypt shared descriptor */
 160         if (desc_inline_query(DESC_QI_AEAD_GIVENC_LEN +
 161                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 162                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 163                               ARRAY_SIZE(data_len)) < 0)
 164                 return -EINVAL;
 165
 166         ctx->adata.key_inline = !!(inl_mask & 1);
 167         ctx->cdata.key_inline = !!(inl_mask & 2);
 168
 169         cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata,
 170                                   ivsize, ctx->authsize, is_rfc3686, nonce,
 171                                   ctx1_iv_off, true, ctrlpriv->era);
 172
 173 skip_givenc:
 174         return 0;
 175 }
 176
 177 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 178 {
 179         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 180
 181         ctx->authsize = authsize;
 182         aead_set_sh_desc(authenc);
 183
 184         return 0;
 185 }
 186
 187 static int aead_setkey(struct crypto_aead *aead, const u8 *key,
 188                        unsigned int keylen)
 189 {
 190         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 191         struct device *jrdev = ctx->jrdev;
 192         struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 193         struct crypto_authenc_keys keys;
 194         int ret = 0;
 195
 196         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 197                 goto badkey;
 198
 199         dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
 200                 keys.authkeylen + keys.enckeylen, keys.enckeylen,
 201                 keys.authkeylen);
 202         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 203                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 204
 205         /*
 206          * If DKP is supported, use it in the shared descriptor to generate
 207          * the split key.
 208          */
 209         if (ctrlpriv->era >= 6) {
 210                 ctx->adata.keylen = keys.authkeylen;
 211                 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
 212                                                       OP_ALG_ALGSEL_MASK);
 213
 214                 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
 215                         goto badkey;
 216
 217                 memcpy(ctx->key, keys.authkey, keys.authkeylen);
 218                 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
 219                        keys.enckeylen);
 220                 dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
 221                                            ctx->adata.keylen_pad +
 222                                            keys.enckeylen, ctx->dir);
 223                 goto skip_split_key;
 224         }
 225
 226         ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey,
 227                             keys.authkeylen, CAAM_MAX_KEY_SIZE -
 228                             keys.enckeylen);
 229         if (ret)
 230                 goto badkey;
 231
 232         /* postpend encryption key to auth split key */
 233         memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
 234         dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
 235                                    ctx->adata.keylen_pad + keys.enckeylen,
 236                                    ctx->dir);
 237
 238         print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
 239                              DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 240                              ctx->adata.keylen_pad + keys.enckeylen, 1);
 241
 242 skip_split_key:
 243         ctx->cdata.keylen = keys.enckeylen;
 244
 245         ret = aead_set_sh_desc(aead);
 246         if (ret)
 247                 goto badkey;
 248
 249         /* Now update the driver contexts with the new shared descriptor */
 250         if (ctx->drv_ctx[ENCRYPT]) {
 251                 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
 252                                           ctx->sh_desc_enc);
 253                 if (ret) {
 254                         dev_err(jrdev, "driver enc context update failed\n");
 255                         goto badkey;
 256                 }
 257         }
 258
 259         if (ctx->drv_ctx[DECRYPT]) {
 260                 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
 261                                           ctx->sh_desc_dec);
 262                 if (ret) {
 263                         dev_err(jrdev, "driver dec context update failed\n");
 264                         goto badkey;
 265                 }
 266         }
 267
 268         memzero_explicit(&keys, sizeof(keys));
 269         return ret;
 270 badkey:
 271         memzero_explicit(&keys, sizeof(keys));
 272         return -EINVAL;
 273 }
 274
 275 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
 276                             unsigned int keylen)
 277 {
 278         struct crypto_authenc_keys keys;
 279         int err;
 280
 281         err = crypto_authenc_extractkeys(&keys, key, keylen);
 282         if (unlikely(err))
 283                 return err;
 284
 285         err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?:
 286               aead_setkey(aead, key, keylen);
 287
 288         memzero_explicit(&keys, sizeof(keys));
 289         return err;
 290 }
 291
 292 static int gcm_set_sh_desc(struct crypto_aead *aead)
 293 {
 294         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 295         unsigned int ivsize = crypto_aead_ivsize(aead);
 296         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 297                         ctx->cdata.keylen;
 298
 299         if (!ctx->cdata.keylen || !ctx->authsize)
 300                 return 0;
 301
 302         /*
 303          * Job Descriptor and Shared Descriptor
 304          * must fit into the 64-word Descriptor h/w Buffer
 305          */
 306         if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {
 307                 ctx->cdata.key_inline = true;
 308                 ctx->cdata.key_virt = ctx->key;
 309         } else {
 310                 ctx->cdata.key_inline = false;
 311                 ctx->cdata.key_dma = ctx->key_dma;
 312         }
 313
 314         cnstr_shdsc_gcm_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
 315                               ctx->authsize, true);
 316
 317         /*
 318          * Job Descriptor and Shared Descriptor
 319          * must fit into the 64-word Descriptor h/w Buffer
 320          */
 321         if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {
 322                 ctx->cdata.key_inline = true;
 323                 ctx->cdata.key_virt = ctx->key;
 324         } else {
 325                 ctx->cdata.key_inline = false;
 326                 ctx->cdata.key_dma = ctx->key_dma;
 327         }
 328
 329         cnstr_shdsc_gcm_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
 330                               ctx->authsize, true);
 331
 332         return 0;
 333 }
 334
 335 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 336 {
 337         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 338         int err;
 339
 340         err = crypto_gcm_check_authsize(authsize);
 341         if (err)
 342                 return err;
 343
 344         ctx->authsize = authsize;
 345         gcm_set_sh_desc(authenc);
 346
 347         return 0;
 348 }
 349
 350 static int gcm_setkey(struct crypto_aead *aead,
 351                       const u8 *key, unsigned int keylen)
 352 {
 353         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 354         struct device *jrdev = ctx->jrdev;
 355         int ret;
 356
 357         ret = aes_check_keylen(keylen);
 358         if (ret)
 359                 return ret;
 360
 361         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 362                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 363
 364         memcpy(ctx->key, key, keylen);
 365         dma_sync_single_for_device(jrdev->parent, ctx->key_dma, keylen,
 366                                    ctx->dir);
 367         ctx->cdata.keylen = keylen;
 368
 369         ret = gcm_set_sh_desc(aead);
 370         if (ret)
 371                 return ret;
 372
 373         /* Now update the driver contexts with the new shared descriptor */
 374         if (ctx->drv_ctx[ENCRYPT]) {
 375                 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
 376                                           ctx->sh_desc_enc);
 377                 if (ret) {
 378                         dev_err(jrdev, "driver enc context update failed\n");
 379                         return ret;
 380                 }
 381         }
 382
 383         if (ctx->drv_ctx[DECRYPT]) {
 384                 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
 385                                           ctx->sh_desc_dec);
 386                 if (ret) {
 387                         dev_err(jrdev, "driver dec context update failed\n");
 388                         return ret;
 389                 }
 390         }
 391
 392         return 0;
 393 }
 394
 395 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 396 {
 397         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 398         unsigned int ivsize = crypto_aead_ivsize(aead);
 399         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 400                         ctx->cdata.keylen;
 401
 402         if (!ctx->cdata.keylen || !ctx->authsize)
 403                 return 0;
 404
 405         ctx->cdata.key_virt = ctx->key;
 406
 407         /*
 408          * Job Descriptor and Shared Descriptor
 409          * must fit into the 64-word Descriptor h/w Buffer
 410          */
 411         if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {
 412                 ctx->cdata.key_inline = true;
 413         } else {
 414                 ctx->cdata.key_inline = false;
 415                 ctx->cdata.key_dma = ctx->key_dma;
 416         }
 417
 418         cnstr_shdsc_rfc4106_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
 419                                   ctx->authsize, true);
 420
 421         /*
 422          * Job Descriptor and Shared Descriptor
 423          * must fit into the 64-word Descriptor h/w Buffer
 424          */
 425         if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {
 426                 ctx->cdata.key_inline = true;
 427         } else {
 428                 ctx->cdata.key_inline = false;
 429                 ctx->cdata.key_dma = ctx->key_dma;
 430         }
 431
 432         cnstr_shdsc_rfc4106_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
 433                                   ctx->authsize, true);
 434
 435         return 0;
 436 }
 437
 438 static int rfc4106_setauthsize(struct crypto_aead *authenc,
 439                                unsigned int authsize)
 440 {
 441         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 442         int err;
 443
 444         err = crypto_rfc4106_check_authsize(authsize);
 445         if (err)
 446                 return err;
 447
 448         ctx->authsize = authsize;
 449         rfc4106_set_sh_desc(authenc);
 450
 451         return 0;
 452 }
 453
 454 static int rfc4106_setkey(struct crypto_aead *aead,
 455                           const u8 *key, unsigned int keylen)
 456 {
 457         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 458         struct device *jrdev = ctx->jrdev;
 459         int ret;
 460
 461         ret = aes_check_keylen(keylen - 4);
 462         if (ret)
 463                 return ret;
 464
 465         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 466                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 467
 468         memcpy(ctx->key, key, keylen);
 469         /*
 470          * The last four bytes of the key material are used as the salt value
 471          * in the nonce. Update the AES key length.
 472          */
 473         ctx->cdata.keylen = keylen - 4;
 474         dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
 475                                    ctx->cdata.keylen, ctx->dir);
 476
 477         ret = rfc4106_set_sh_desc(aead);
 478         if (ret)
 479                 return ret;
 480
 481         /* Now update the driver contexts with the new shared descriptor */
 482         if (ctx->drv_ctx[ENCRYPT]) {
 483                 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
 484                                           ctx->sh_desc_enc);
 485                 if (ret) {
 486                         dev_err(jrdev, "driver enc context update failed\n");
 487                         return ret;
 488                 }
 489         }
 490
 491         if (ctx->drv_ctx[DECRYPT]) {
 492                 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
 493                                           ctx->sh_desc_dec);
 494                 if (ret) {
 495                         dev_err(jrdev, "driver dec context update failed\n");
 496                         return ret;
 497                 }
 498         }
 499
 500         return 0;
 501 }
 502
 503 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
 504 {
 505         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 506         unsigned int ivsize = crypto_aead_ivsize(aead);
 507         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 508                         ctx->cdata.keylen;
 509
 510         if (!ctx->cdata.keylen || !ctx->authsize)
 511                 return 0;
 512
 513         ctx->cdata.key_virt = ctx->key;
 514
 515         /*
 516          * Job Descriptor and Shared Descriptor
 517          * must fit into the 64-word Descriptor h/w Buffer
 518          */
 519         if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {
 520                 ctx->cdata.key_inline = true;
 521         } else {
 522                 ctx->cdata.key_inline = false;
 523                 ctx->cdata.key_dma = ctx->key_dma;
 524         }
 525
 526         cnstr_shdsc_rfc4543_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
 527                                   ctx->authsize, true);
 528
 529         /*
 530          * Job Descriptor and Shared Descriptor
 531          * must fit into the 64-word Descriptor h/w Buffer
 532          */
 533         if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {
 534                 ctx->cdata.key_inline = true;
 535         } else {
 536                 ctx->cdata.key_inline = false;
 537                 ctx->cdata.key_dma = ctx->key_dma;
 538         }
 539
 540         cnstr_shdsc_rfc4543_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
 541                                   ctx->authsize, true);
 542
 543         return 0;
 544 }
 545
 546 static int rfc4543_setauthsize(struct crypto_aead *authenc,
 547                                unsigned int authsize)
 548 {
 549         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 550
 551         if (authsize != 16)
 552                 return -EINVAL;
 553
 554         ctx->authsize = authsize;
 555         rfc4543_set_sh_desc(authenc);
 556
 557         return 0;
 558 }
 559
 560 static int rfc4543_setkey(struct crypto_aead *aead,
 561                           const u8 *key, unsigned int keylen)
 562 {
 563         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 564         struct device *jrdev = ctx->jrdev;
 565         int ret;
 566
 567         ret = aes_check_keylen(keylen - 4);
 568         if (ret)
 569                 return ret;
 570
 571         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 572                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 573
 574         memcpy(ctx->key, key, keylen);
 575         /*
 576          * The last four bytes of the key material are used as the salt value
 577          * in the nonce. Update the AES key length.
 578          */
 579         ctx->cdata.keylen = keylen - 4;
 580         dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
 581                                    ctx->cdata.keylen, ctx->dir);
 582
 583         ret = rfc4543_set_sh_desc(aead);
 584         if (ret)
 585                 return ret;
 586
 587         /* Now update the driver contexts with the new shared descriptor */
 588         if (ctx->drv_ctx[ENCRYPT]) {
 589                 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
 590                                           ctx->sh_desc_enc);
 591                 if (ret) {
 592                         dev_err(jrdev, "driver enc context update failed\n");
 593                         return ret;
 594                 }
 595         }
 596
 597         if (ctx->drv_ctx[DECRYPT]) {
 598                 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
 599                                           ctx->sh_desc_dec);
 600                 if (ret) {
 601                         dev_err(jrdev, "driver dec context update failed\n");
 602                         return ret;
 603                 }
 604         }
 605
 606         return 0;
 607 }
 608
 609 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 610                            unsigned int keylen, const u32 ctx1_iv_off)
 611 {
 612         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
 613         struct caam_skcipher_alg *alg =
 614                 container_of(crypto_skcipher_alg(skcipher), typeof(*alg),
 615                              skcipher);
 616         struct device *jrdev = ctx->jrdev;
 617         unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
 618         const bool is_rfc3686 = alg->caam.rfc3686;
 619         int ret = 0;
 620
 621         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 622                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 623
 624         ctx->cdata.keylen = keylen;
 625         ctx->cdata.key_virt = key;
 626         ctx->cdata.key_inline = true;
 627
 628         /* skcipher encrypt, decrypt shared descriptors */
 629         cnstr_shdsc_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
 630                                    is_rfc3686, ctx1_iv_off);
 631         cnstr_shdsc_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
 632                                    is_rfc3686, ctx1_iv_off);
 633
 634         /* Now update the driver contexts with the new shared descriptor */
 635         if (ctx->drv_ctx[ENCRYPT]) {
 636                 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
 637                                           ctx->sh_desc_enc);
 638                 if (ret) {
 639                         dev_err(jrdev, "driver enc context update failed\n");
 640                         return -EINVAL;
 641                 }
 642         }
 643
 644         if (ctx->drv_ctx[DECRYPT]) {
 645                 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
 646                                           ctx->sh_desc_dec);
 647                 if (ret) {
 648                         dev_err(jrdev, "driver dec context update failed\n");
 649                         return -EINVAL;
 650                 }
 651         }
 652
 653         return ret;
 654 }
 655
 656 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
 657                                const u8 *key, unsigned int keylen)
 658 {
 659         int err;
 660
 661         err = aes_check_keylen(keylen);
 662         if (err)
 663                 return err;
 664
 665         return skcipher_setkey(skcipher, key, keylen, 0);
 666 }
 667
 668 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
 669                                    const u8 *key, unsigned int keylen)
 670 {
 671         u32 ctx1_iv_off;
 672         int err;
 673
 674         /*
 675          * RFC3686 specific:
 676          *      | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 677          *      | *key = {KEY, NONCE}
 678          */
 679         ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 680         keylen -= CTR_RFC3686_NONCE_SIZE;
 681
 682         err = aes_check_keylen(keylen);
 683         if (err)
 684                 return err;
 685
 686         return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
 687 }
 688
 689 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
 690                                const u8 *key, unsigned int keylen)
 691 {
 692         u32 ctx1_iv_off;
 693         int err;
 694
 695         /*
 696          * AES-CTR needs to load IV in CONTEXT1 reg
 697          * at an offset of 128bits (16bytes)
 698          * CONTEXT1[255:128] = IV
 699          */
 700         ctx1_iv_off = 16;
 701
 702         err = aes_check_keylen(keylen);
 703         if (err)
 704                 return err;
 705
 706         return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
 707 }
 708
 709 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
 710                                 const u8 *key, unsigned int keylen)
 711 {
 712         return verify_skcipher_des3_key(skcipher, key) ?:
 713                skcipher_setkey(skcipher, key, keylen, 0);
 714 }
 715
 716 static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
 717                                const u8 *key, unsigned int keylen)
 718 {
 719         return verify_skcipher_des_key(skcipher, key) ?:
 720                skcipher_setkey(skcipher, key, keylen, 0);
 721 }
 722
 723 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 724                                unsigned int keylen)
 725 {
 726         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
 727         struct device *jrdev = ctx->jrdev;
 728         int ret = 0;
 729
 730         if (keylen != 2 * AES_MIN_KEY_SIZE  && keylen != 2 * AES_MAX_KEY_SIZE) {
 731                 dev_err(jrdev, "key size mismatch\n");
 732                 return -EINVAL;
 733         }
 734
 735         ctx->cdata.keylen = keylen;
 736         ctx->cdata.key_virt = key;
 737         ctx->cdata.key_inline = true;
 738
 739         /* xts skcipher encrypt, decrypt shared descriptors */
 740         cnstr_shdsc_xts_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata);
 741         cnstr_shdsc_xts_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata);
 742
 743         /* Now update the driver contexts with the new shared descriptor */
 744         if (ctx->drv_ctx[ENCRYPT]) {
 745                 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
 746                                           ctx->sh_desc_enc);
 747                 if (ret) {
 748                         dev_err(jrdev, "driver enc context update failed\n");
 749                         return -EINVAL;
 750                 }
 751         }
 752
 753         if (ctx->drv_ctx[DECRYPT]) {
 754                 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
 755                                           ctx->sh_desc_dec);
 756                 if (ret) {
 757                         dev_err(jrdev, "driver dec context update failed\n");
 758                         return -EINVAL;
 759                 }
 760         }
 761
 762         return ret;
 763 }
 764
 765 /*
 766  * aead_edesc - s/w-extended aead descriptor
 767  * @src_nents: number of segments in input scatterlist
 768  * @dst_nents: number of segments in output scatterlist
 769  * @iv_dma: dma address of iv for checking continuity and link table
 770  * @qm_sg_bytes: length of dma mapped h/w link table
 771  * @qm_sg_dma: bus physical mapped address of h/w link table
 772  * @assoclen: associated data length, in CAAM endianness
 773  * @assoclen_dma: bus physical mapped address of req->assoclen
 774  * @drv_req: driver-specific request structure
 775  * @sgt: the h/w link table, followed by IV
 776  */
 777 struct aead_edesc {
 778         int src_nents;
 779         int dst_nents;
 780         dma_addr_t iv_dma;
 781         int qm_sg_bytes;
 782         dma_addr_t qm_sg_dma;
 783         unsigned int assoclen;
 784         dma_addr_t assoclen_dma;
 785         struct caam_drv_req drv_req;
 786         struct qm_sg_entry sgt[];
 787 };
 788
 789 /*
 790  * skcipher_edesc - s/w-extended skcipher descriptor
 791  * @src_nents: number of segments in input scatterlist
 792  * @dst_nents: number of segments in output scatterlist
 793  * @iv_dma: dma address of iv for checking continuity and link table
 794  * @qm_sg_bytes: length of dma mapped h/w link table
 795  * @qm_sg_dma: bus physical mapped address of h/w link table
 796  * @drv_req: driver-specific request structure
 797  * @sgt: the h/w link table, followed by IV
 798  */
 799 struct skcipher_edesc {
 800         int src_nents;
 801         int dst_nents;
 802         dma_addr_t iv_dma;
 803         int qm_sg_bytes;
 804         dma_addr_t qm_sg_dma;
 805         struct caam_drv_req drv_req;
 806         struct qm_sg_entry sgt[];
 807 };
 808
 809 static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx,
 810                                         enum optype type)
 811 {
 812         /*
 813          * This function is called on the fast path with values of 'type'
 814          * known at compile time. Invalid arguments are not expected and
 815          * thus no checks are made.
 816          */
 817         struct caam_drv_ctx *drv_ctx = ctx->drv_ctx[type];
 818         u32 *desc;
 819
 820         if (unlikely(!drv_ctx)) {
 821                 spin_lock(&ctx->lock);
 822
 823                 /* Read again to check if some other core init drv_ctx */
 824                 drv_ctx = ctx->drv_ctx[type];
 825                 if (!drv_ctx) {
 826                         int cpu;
 827
 828                         if (type == ENCRYPT)
 829                                 desc = ctx->sh_desc_enc;
 830                         else /* (type == DECRYPT) */
 831                                 desc = ctx->sh_desc_dec;
 832
 833                         cpu = smp_processor_id();
 834                         drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc);
 835                         if (!IS_ERR_OR_NULL(drv_ctx))
 836                                 drv_ctx->op_type = type;
 837
 838                         ctx->drv_ctx[type] = drv_ctx;
 839                 }
 840
 841                 spin_unlock(&ctx->lock);
 842         }
 843
 844         return drv_ctx;
 845 }
 846
 847 static void caam_unmap(struct device *dev, struct scatterlist *src,
 848                        struct scatterlist *dst, int src_nents,
 849                        int dst_nents, dma_addr_t iv_dma, int ivsize,
 850                        enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
 851                        int qm_sg_bytes)
 852 {
 853         if (dst != src) {
 854                 if (src_nents)
 855                         dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 856                 if (dst_nents)
 857                         dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
 858         } else {
 859                 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 860         }
 861
 862         if (iv_dma)
 863                 dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
 864         if (qm_sg_bytes)
 865                 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
 866 }
 867
 868 static void aead_unmap(struct device *dev,
 869                        struct aead_edesc *edesc,
 870                        struct aead_request *req)
 871 {
 872         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 873         int ivsize = crypto_aead_ivsize(aead);
 874
 875         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
 876                    edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
 877                    edesc->qm_sg_bytes);
 878         dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
 879 }
 880
 881 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
 882                            struct skcipher_request *req)
 883 {
 884         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
 885         int ivsize = crypto_skcipher_ivsize(skcipher);
 886
 887         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
 888                    edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
 889                    edesc->qm_sg_bytes);
 890 }
 891
 892 static void aead_done(struct caam_drv_req *drv_req, u32 status)
 893 {
 894         struct device *qidev;
 895         struct aead_edesc *edesc;
 896         struct aead_request *aead_req = drv_req->app_ctx;
 897         struct crypto_aead *aead = crypto_aead_reqtfm(aead_req);
 898         struct caam_ctx *caam_ctx = crypto_aead_ctx(aead);
 899         int ecode = 0;
 900
 901         qidev = caam_ctx->qidev;
 902
 903         if (unlikely(status))
 904                 ecode = caam_jr_strstatus(qidev, status);
 905
 906         edesc = container_of(drv_req, typeof(*edesc), drv_req);
 907         aead_unmap(qidev, edesc, aead_req);
 908
 909         aead_request_complete(aead_req, ecode);
 910         qi_cache_free(edesc);
 911 }
 912
 913 /*
 914  * allocate and map the aead extended descriptor
 915  */
 916 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
 917                                            bool encrypt)
 918 {
 919         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 920         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 921         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 922                                                  typeof(*alg), aead);
 923         struct device *qidev = ctx->qidev;
 924         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 925                        GFP_KERNEL : GFP_ATOMIC;
 926         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
 927         int src_len, dst_len = 0;
 928         struct aead_edesc *edesc;
 929         dma_addr_t qm_sg_dma, iv_dma = 0;
 930         int ivsize = 0;
 931         unsigned int authsize = ctx->authsize;
 932         int qm_sg_index = 0, qm_sg_ents = 0, qm_sg_bytes;
 933         int in_len, out_len;
 934         struct qm_sg_entry *sg_table, *fd_sgt;
 935         struct caam_drv_ctx *drv_ctx;
 936
 937         drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);
 938         if (IS_ERR_OR_NULL(drv_ctx))
 939                 return (struct aead_edesc *)drv_ctx;
 940
 941         /* allocate space for base edesc and hw desc commands, link tables */
 942         edesc = qi_cache_alloc(GFP_DMA | flags);
 943         if (unlikely(!edesc)) {
 944                 dev_err(qidev, "could not allocate extended descriptor\n");
 945                 return ERR_PTR(-ENOMEM);
 946         }
 947
 948         if (likely(req->src == req->dst)) {
 949                 src_len = req->assoclen + req->cryptlen +
 950                           (encrypt ? authsize : 0);
 951
 952                 src_nents = sg_nents_for_len(req->src, src_len);
 953                 if (unlikely(src_nents < 0)) {
 954                         dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
 955                                 src_len);
 956                         qi_cache_free(edesc);
 957                         return ERR_PTR(src_nents);
 958                 }
 959
 960                 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,
 961                                               DMA_BIDIRECTIONAL);
 962                 if (unlikely(!mapped_src_nents)) {
 963                         dev_err(qidev, "unable to map source\n");
 964                         qi_cache_free(edesc);
 965                         return ERR_PTR(-ENOMEM);
 966                 }
 967         } else {
 968                 src_len = req->assoclen + req->cryptlen;
 969                 dst_len = src_len + (encrypt ? authsize : (-authsize));
 970
 971                 src_nents = sg_nents_for_len(req->src, src_len);
 972                 if (unlikely(src_nents < 0)) {
 973                         dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
 974                                 src_len);
 975                         qi_cache_free(edesc);
 976                         return ERR_PTR(src_nents);
 977                 }
 978
 979                 dst_nents = sg_nents_for_len(req->dst, dst_len);
 980                 if (unlikely(dst_nents < 0)) {
 981                         dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n",
 982                                 dst_len);
 983                         qi_cache_free(edesc);
 984                         return ERR_PTR(dst_nents);
 985                 }
 986
 987                 if (src_nents) {
 988                         mapped_src_nents = dma_map_sg(qidev, req->src,
 989                                                       src_nents, DMA_TO_DEVICE);
 990                         if (unlikely(!mapped_src_nents)) {
 991                                 dev_err(qidev, "unable to map source\n");
 992                                 qi_cache_free(edesc);
 993                                 return ERR_PTR(-ENOMEM);
 994                         }
 995                 } else {
 996                         mapped_src_nents = 0;
 997                 }
 998
 999                 if (dst_nents) {
1000                         mapped_dst_nents = dma_map_sg(qidev, req->dst,
1001                                                       dst_nents,
1002                                                       DMA_FROM_DEVICE);
1003                         if (unlikely(!mapped_dst_nents)) {
1004                                 dev_err(qidev, "unable to map destination\n");
1005                                 dma_unmap_sg(qidev, req->src, src_nents,
1006                                              DMA_TO_DEVICE);
1007                                 qi_cache_free(edesc);
1008                                 return ERR_PTR(-ENOMEM);
1009                         }
1010                 } else {
1011                         mapped_dst_nents = 0;
1012                 }
1013         }
1014
1015         if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv)
1016                 ivsize = crypto_aead_ivsize(aead);
1017
1018         /*
1019          * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
1020          * Input is not contiguous.
1021          * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1022          * the end of the table by allocating more S/G entries. Logic:
1023          * if (src != dst && output S/G)
1024          *      pad output S/G, if needed
1025          * else if (src == dst && S/G)
1026          *      overlapping S/Gs; pad one of them
1027          * else if (input S/G) ...
1028          *      pad input S/G, if needed
1029          */
1030         qm_sg_ents = 1 + !!ivsize + mapped_src_nents;
1031         if (mapped_dst_nents > 1)
1032                 qm_sg_ents += pad_sg_nents(mapped_dst_nents);
1033         else if ((req->src == req->dst) && (mapped_src_nents > 1))
1034                 qm_sg_ents = max(pad_sg_nents(qm_sg_ents),
1035                                  1 + !!ivsize + pad_sg_nents(mapped_src_nents));
1036         else
1037                 qm_sg_ents = pad_sg_nents(qm_sg_ents);
1038
1039         sg_table = &edesc->sgt[0];
1040         qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
1041         if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
1042                      CAAM_QI_MEMCACHE_SIZE)) {
1043                 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
1044                         qm_sg_ents, ivsize);
1045                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1046                            0, DMA_NONE, 0, 0);
1047                 qi_cache_free(edesc);
1048                 return ERR_PTR(-ENOMEM);
1049         }
1050
1051         if (ivsize) {
1052                 u8 *iv = (u8 *)(sg_table + qm_sg_ents);
1053
1054                 /* Make sure IV is located in a DMAable area */
1055                 memcpy(iv, req->iv, ivsize);
1056
1057                 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE);
1058                 if (dma_mapping_error(qidev, iv_dma)) {
1059                         dev_err(qidev, "unable to map IV\n");
1060                         caam_unmap(qidev, req->src, req->dst, src_nents,
1061                                    dst_nents, 0, 0, DMA_NONE, 0, 0);
1062                         qi_cache_free(edesc);
1063                         return ERR_PTR(-ENOMEM);
1064                 }
1065         }
1066
1067         edesc->src_nents = src_nents;
1068         edesc->dst_nents = dst_nents;
1069         edesc->iv_dma = iv_dma;
1070         edesc->drv_req.app_ctx = req;
1071         edesc->drv_req.cbk = aead_done;
1072         edesc->drv_req.drv_ctx = drv_ctx;
1073
1074         edesc->assoclen = cpu_to_caam32(req->assoclen);
1075         edesc->assoclen_dma = dma_map_single(qidev, &edesc->assoclen, 4,
1076                                              DMA_TO_DEVICE);
1077         if (dma_mapping_error(qidev, edesc->assoclen_dma)) {
1078                 dev_err(qidev, "unable to map assoclen\n");
1079                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
1080                            iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
1081                 qi_cache_free(edesc);
1082                 return ERR_PTR(-ENOMEM);
1083         }
1084
1085         dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0);
1086         qm_sg_index++;
1087         if (ivsize) {
1088                 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
1089                 qm_sg_index++;
1090         }
1091         sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
1092         qm_sg_index += mapped_src_nents;
1093
1094         if (mapped_dst_nents > 1)
1095                 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
1096
1097         qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
1098         if (dma_mapping_error(qidev, qm_sg_dma)) {
1099                 dev_err(qidev, "unable to map S/G table\n");
1100                 dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
1101                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
1102                            iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
1103                 qi_cache_free(edesc);
1104                 return ERR_PTR(-ENOMEM);
1105         }
1106
1107         edesc->qm_sg_dma = qm_sg_dma;
1108         edesc->qm_sg_bytes = qm_sg_bytes;
1109
1110         out_len = req->assoclen + req->cryptlen +
1111                   (encrypt ? ctx->authsize : (-ctx->authsize));
1112         in_len = 4 + ivsize + req->assoclen + req->cryptlen;
1113
1114         fd_sgt = &edesc->drv_req.fd_sgt[0];
1115         dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0);
1116
1117         if (req->dst == req->src) {
1118                 if (mapped_src_nents == 1)
1119                         dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src),
1120                                          out_len, 0);
1121                 else
1122                         dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma +
1123                                              (1 + !!ivsize) * sizeof(*sg_table),
1124                                              out_len, 0);
1125         } else if (mapped_dst_nents <= 1) {
1126                 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len,
1127                                  0);
1128         } else {
1129                 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) *
1130                                      qm_sg_index, out_len, 0);
1131         }
1132
1133         return edesc;
1134 }
1135
1136 static inline int aead_crypt(struct aead_request *req, bool encrypt)
1137 {
1138         struct aead_edesc *edesc;
1139         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1140         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1141         int ret;
1142
1143         if (unlikely(caam_congested))
1144                 return -EAGAIN;
1145
1146         /* allocate extended descriptor */
1147         edesc = aead_edesc_alloc(req, encrypt);
1148         if (IS_ERR_OR_NULL(edesc))
1149                 return PTR_ERR(edesc);
1150
1151         /* Create and submit job descriptor */
1152         ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req);
1153         if (!ret) {
1154                 ret = -EINPROGRESS;
1155         } else {
1156                 aead_unmap(ctx->qidev, edesc, req);
1157                 qi_cache_free(edesc);
1158         }
1159
1160         return ret;
1161 }
1162
1163 static int aead_encrypt(struct aead_request *req)
1164 {
1165         return aead_crypt(req, true);
1166 }
1167
1168 static int aead_decrypt(struct aead_request *req)
1169 {
1170         return aead_crypt(req, false);
1171 }
1172
1173 static int ipsec_gcm_encrypt(struct aead_request *req)
1174 {
1175         return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req,
1176                                            true);
1177 }
1178
1179 static int ipsec_gcm_decrypt(struct aead_request *req)
1180 {
1181         return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req,
1182                                            false);
1183 }
1184
1185 static void skcipher_done(struct caam_drv_req *drv_req, u32 status)
1186 {
1187         struct skcipher_edesc *edesc;
1188         struct skcipher_request *req = drv_req->app_ctx;
1189         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1190         struct caam_ctx *caam_ctx = crypto_skcipher_ctx(skcipher);
1191         struct device *qidev = caam_ctx->qidev;
1192         int ivsize = crypto_skcipher_ivsize(skcipher);
1193         int ecode = 0;
1194
1195         dev_dbg(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status);
1196
1197         edesc = container_of(drv_req, typeof(*edesc), drv_req);
1198
1199         if (status)
1200                 ecode = caam_jr_strstatus(qidev, status);
1201
1202         print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1203                              DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1204                              edesc->src_nents > 1 ? 100 : ivsize, 1);
1205         caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1206                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1207                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1208
1209         skcipher_unmap(qidev, edesc, req);
1210
1211         /*
1212          * The crypto API expects us to set the IV (req->iv) to the last
1213          * ciphertext block (CBC mode) or last counter (CTR mode).
1214          * This is used e.g. by the CTS mode.
1215          */
1216         if (!ecode)
1217                 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1218                        ivsize);
1219
1220         qi_cache_free(edesc);
1221         skcipher_request_complete(req, ecode);
1222 }
1223
1224 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
1225                                                    bool encrypt)
1226 {
1227         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1228         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1229         struct device *qidev = ctx->qidev;
1230         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1231                        GFP_KERNEL : GFP_ATOMIC;
1232         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1233         struct skcipher_edesc *edesc;
1234         dma_addr_t iv_dma;
1235         u8 *iv;
1236         int ivsize = crypto_skcipher_ivsize(skcipher);
1237         int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
1238         struct qm_sg_entry *sg_table, *fd_sgt;
1239         struct caam_drv_ctx *drv_ctx;
1240
1241         drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);
1242         if (IS_ERR_OR_NULL(drv_ctx))
1243                 return (struct skcipher_edesc *)drv_ctx;
1244
1245         src_nents = sg_nents_for_len(req->src, req->cryptlen);
1246         if (unlikely(src_nents < 0)) {
1247                 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
1248                         req->cryptlen);
1249                 return ERR_PTR(src_nents);
1250         }
1251
1252         if (unlikely(req->src != req->dst)) {
1253                 dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
1254                 if (unlikely(dst_nents < 0)) {
1255                         dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n",
1256                                 req->cryptlen);
1257                         return ERR_PTR(dst_nents);
1258                 }
1259
1260                 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,
1261                                               DMA_TO_DEVICE);
1262                 if (unlikely(!mapped_src_nents)) {
1263                         dev_err(qidev, "unable to map source\n");
1264                         return ERR_PTR(-ENOMEM);
1265                 }
1266
1267                 mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents,
1268                                               DMA_FROM_DEVICE);
1269                 if (unlikely(!mapped_dst_nents)) {
1270                         dev_err(qidev, "unable to map destination\n");
1271                         dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE);
1272                         return ERR_PTR(-ENOMEM);
1273                 }
1274         } else {
1275                 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,
1276                                               DMA_BIDIRECTIONAL);
1277                 if (unlikely(!mapped_src_nents)) {
1278                         dev_err(qidev, "unable to map source\n");
1279                         return ERR_PTR(-ENOMEM);
1280                 }
1281         }
1282
1283         qm_sg_ents = 1 + mapped_src_nents;
1284         dst_sg_idx = qm_sg_ents;
1285
1286         /*
1287          * Input, output HW S/G tables: [IV, src][dst, IV]
1288          * IV entries point to the same buffer
1289          * If src == dst, S/G entries are reused (S/G tables overlap)
1290          *
1291          * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1292          * the end of the table by allocating more S/G entries.
1293          */
1294         if (req->src != req->dst)
1295                 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
1296         else
1297                 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
1298
1299         qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);
1300         if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
1301                      ivsize > CAAM_QI_MEMCACHE_SIZE)) {
1302                 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
1303                         qm_sg_ents, ivsize);
1304                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1305                            0, DMA_NONE, 0, 0);
1306                 return ERR_PTR(-ENOMEM);
1307         }
1308
1309         /* allocate space for base edesc, link tables and IV */
1310         edesc = qi_cache_alloc(GFP_DMA | flags);
1311         if (unlikely(!edesc)) {
1312                 dev_err(qidev, "could not allocate extended descriptor\n");
1313                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1314                            0, DMA_NONE, 0, 0);
1315                 return ERR_PTR(-ENOMEM);
1316         }
1317
1318         /* Make sure IV is located in a DMAable area */
1319         sg_table = &edesc->sgt[0];
1320         iv = (u8 *)(sg_table + qm_sg_ents);
1321         memcpy(iv, req->iv, ivsize);
1322
1323         iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL);
1324         if (dma_mapping_error(qidev, iv_dma)) {
1325                 dev_err(qidev, "unable to map IV\n");
1326                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1327                            0, DMA_NONE, 0, 0);
1328                 qi_cache_free(edesc);
1329                 return ERR_PTR(-ENOMEM);
1330         }
1331
1332         edesc->src_nents = src_nents;
1333         edesc->dst_nents = dst_nents;
1334         edesc->iv_dma = iv_dma;
1335         edesc->qm_sg_bytes = qm_sg_bytes;
1336         edesc->drv_req.app_ctx = req;
1337         edesc->drv_req.cbk = skcipher_done;
1338         edesc->drv_req.drv_ctx = drv_ctx;
1339
1340         dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
1341         sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
1342
1343         if (req->src != req->dst)
1344                 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
1345
1346         dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
1347                          ivsize, 0);
1348
1349         edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes,
1350                                           DMA_TO_DEVICE);
1351         if (dma_mapping_error(qidev, edesc->qm_sg_dma)) {
1352                 dev_err(qidev, "unable to map S/G table\n");
1353                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
1354                            iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
1355                 qi_cache_free(edesc);
1356                 return ERR_PTR(-ENOMEM);
1357         }
1358
1359         fd_sgt = &edesc->drv_req.fd_sgt[0];
1360
1361         dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma,
1362                                   ivsize + req->cryptlen, 0);
1363
1364         if (req->src == req->dst)
1365                 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma +
1366                                      sizeof(*sg_table), req->cryptlen + ivsize,
1367                                      0);
1368         else
1369                 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx *
1370                                      sizeof(*sg_table), req->cryptlen + ivsize,
1371                                      0);
1372
1373         return edesc;
1374 }
1375
1376 static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
1377 {
1378         struct skcipher_edesc *edesc;
1379         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1380         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1381         int ret;
1382
1383         if (!req->cryptlen)
1384                 return 0;
1385
1386         if (unlikely(caam_congested))
1387                 return -EAGAIN;
1388
1389         /* allocate extended descriptor */
1390         edesc = skcipher_edesc_alloc(req, encrypt);
1391         if (IS_ERR(edesc))
1392                 return PTR_ERR(edesc);
1393
1394         ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req);
1395         if (!ret) {
1396                 ret = -EINPROGRESS;
1397         } else {
1398                 skcipher_unmap(ctx->qidev, edesc, req);
1399                 qi_cache_free(edesc);
1400         }
1401
1402         return ret;
1403 }
1404
1405 static int skcipher_encrypt(struct skcipher_request *req)
1406 {
1407         return skcipher_crypt(req, true);
1408 }
1409
1410 static int skcipher_decrypt(struct skcipher_request *req)
1411 {
1412         return skcipher_crypt(req, false);
1413 }
1414
1415 static struct caam_skcipher_alg driver_algs[] = {
1416         {
1417                 .skcipher = {
1418                         .base = {
1419                                 .cra_name = "cbc(aes)",
1420                                 .cra_driver_name = "cbc-aes-caam-qi",
1421                                 .cra_blocksize = AES_BLOCK_SIZE,
1422                         },
1423                         .setkey = aes_skcipher_setkey,
1424                         .encrypt = skcipher_encrypt,
1425                         .decrypt = skcipher_decrypt,
1426                         .min_keysize = AES_MIN_KEY_SIZE,
1427                         .max_keysize = AES_MAX_KEY_SIZE,
1428                         .ivsize = AES_BLOCK_SIZE,
1429                 },
1430                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1431         },
1432         {
1433                 .skcipher = {
1434                         .base = {
1435                                 .cra_name = "cbc(des3_ede)",
1436                                 .cra_driver_name = "cbc-3des-caam-qi",
1437                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1438                         },
1439                         .setkey = des3_skcipher_setkey,
1440                         .encrypt = skcipher_encrypt,
1441                         .decrypt = skcipher_decrypt,
1442                         .min_keysize = DES3_EDE_KEY_SIZE,
1443                         .max_keysize = DES3_EDE_KEY_SIZE,
1444                         .ivsize = DES3_EDE_BLOCK_SIZE,
1445                 },
1446                 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1447         },
1448         {
1449                 .skcipher = {
1450                         .base = {
1451                                 .cra_name = "cbc(des)",
1452                                 .cra_driver_name = "cbc-des-caam-qi",
1453                                 .cra_blocksize = DES_BLOCK_SIZE,
1454                         },
1455                         .setkey = des_skcipher_setkey,
1456                         .encrypt = skcipher_encrypt,
1457                         .decrypt = skcipher_decrypt,
1458                         .min_keysize = DES_KEY_SIZE,
1459                         .max_keysize = DES_KEY_SIZE,
1460                         .ivsize = DES_BLOCK_SIZE,
1461                 },
1462                 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1463         },
1464         {
1465                 .skcipher = {
1466                         .base = {
1467                                 .cra_name = "ctr(aes)",
1468                                 .cra_driver_name = "ctr-aes-caam-qi",
1469                                 .cra_blocksize = 1,
1470                         },
1471                         .setkey = ctr_skcipher_setkey,
1472                         .encrypt = skcipher_encrypt,
1473                         .decrypt = skcipher_decrypt,
1474                         .min_keysize = AES_MIN_KEY_SIZE,
1475                         .max_keysize = AES_MAX_KEY_SIZE,
1476                         .ivsize = AES_BLOCK_SIZE,
1477                         .chunksize = AES_BLOCK_SIZE,
1478                 },
1479                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1480                                         OP_ALG_AAI_CTR_MOD128,
1481         },
1482         {
1483                 .skcipher = {
1484                         .base = {
1485                                 .cra_name = "rfc3686(ctr(aes))",
1486                                 .cra_driver_name = "rfc3686-ctr-aes-caam-qi",
1487                                 .cra_blocksize = 1,
1488                         },
1489                         .setkey = rfc3686_skcipher_setkey,
1490                         .encrypt = skcipher_encrypt,
1491                         .decrypt = skcipher_decrypt,
1492                         .min_keysize = AES_MIN_KEY_SIZE +
1493                                        CTR_RFC3686_NONCE_SIZE,
1494                         .max_keysize = AES_MAX_KEY_SIZE +
1495                                        CTR_RFC3686_NONCE_SIZE,
1496                         .ivsize = CTR_RFC3686_IV_SIZE,
1497                         .chunksize = AES_BLOCK_SIZE,
1498                 },
1499                 .caam = {
1500                         .class1_alg_type = OP_ALG_ALGSEL_AES |
1501                                            OP_ALG_AAI_CTR_MOD128,
1502                         .rfc3686 = true,
1503                 },
1504         },
1505         {
1506                 .skcipher = {
1507                         .base = {
1508                                 .cra_name = "xts(aes)",
1509                                 .cra_driver_name = "xts-aes-caam-qi",
1510                                 .cra_blocksize = AES_BLOCK_SIZE,
1511                         },
1512                         .setkey = xts_skcipher_setkey,
1513                         .encrypt = skcipher_encrypt,
1514                         .decrypt = skcipher_decrypt,
1515                         .min_keysize = 2 * AES_MIN_KEY_SIZE,
1516                         .max_keysize = 2 * AES_MAX_KEY_SIZE,
1517                         .ivsize = AES_BLOCK_SIZE,
1518                 },
1519                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
1520         },
1521 };
1522
1523 static struct caam_aead_alg driver_aeads[] = {
1524         {
1525                 .aead = {
1526                         .base = {
1527                                 .cra_name = "rfc4106(gcm(aes))",
1528                                 .cra_driver_name = "rfc4106-gcm-aes-caam-qi",
1529                                 .cra_blocksize = 1,
1530                         },
1531                         .setkey = rfc4106_setkey,
1532                         .setauthsize = rfc4106_setauthsize,
1533                         .encrypt = ipsec_gcm_encrypt,
1534                         .decrypt = ipsec_gcm_decrypt,
1535                         .ivsize = 8,
1536                         .maxauthsize = AES_BLOCK_SIZE,
1537                 },
1538                 .caam = {
1539                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1540                         .nodkp = true,
1541                 },
1542         },
1543         {
1544                 .aead = {
1545                         .base = {
1546                                 .cra_name = "rfc4543(gcm(aes))",
1547                                 .cra_driver_name = "rfc4543-gcm-aes-caam-qi",
1548                                 .cra_blocksize = 1,
1549                         },
1550                         .setkey = rfc4543_setkey,
1551                         .setauthsize = rfc4543_setauthsize,
1552                         .encrypt = ipsec_gcm_encrypt,
1553                         .decrypt = ipsec_gcm_decrypt,
1554                         .ivsize = 8,
1555                         .maxauthsize = AES_BLOCK_SIZE,
1556                 },
1557                 .caam = {
1558                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1559                         .nodkp = true,
1560                 },
1561         },
1562         /* Galois Counter Mode */
1563         {
1564                 .aead = {
1565                         .base = {
1566                                 .cra_name = "gcm(aes)",
1567                                 .cra_driver_name = "gcm-aes-caam-qi",
1568                                 .cra_blocksize = 1,
1569                         },
1570                         .setkey = gcm_setkey,
1571                         .setauthsize = gcm_setauthsize,
1572                         .encrypt = aead_encrypt,
1573                         .decrypt = aead_decrypt,
1574                         .ivsize = 12,
1575                         .maxauthsize = AES_BLOCK_SIZE,
1576                 },
1577                 .caam = {
1578                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1579                         .nodkp = true,
1580                 }
1581         },
1582         /* single-pass ipsec_esp descriptor */
1583         {
1584                 .aead = {
1585                         .base = {
1586                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
1587                                 .cra_driver_name = "authenc-hmac-md5-"
1588                                                    "cbc-aes-caam-qi",
1589                                 .cra_blocksize = AES_BLOCK_SIZE,
1590                         },
1591                         .setkey = aead_setkey,
1592                         .setauthsize = aead_setauthsize,
1593                         .encrypt = aead_encrypt,
1594                         .decrypt = aead_decrypt,
1595                         .ivsize = AES_BLOCK_SIZE,
1596                         .maxauthsize = MD5_DIGEST_SIZE,
1597                 },
1598                 .caam = {
1599                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1600                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1601                                            OP_ALG_AAI_HMAC_PRECOMP,
1602                 }
1603         },
1604         {
1605                 .aead = {
1606                         .base = {
1607                                 .cra_name = "echainiv(authenc(hmac(md5),"
1608                                             "cbc(aes)))",
1609                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
1610                                                    "cbc-aes-caam-qi",
1611                                 .cra_blocksize = AES_BLOCK_SIZE,
1612                         },
1613                         .setkey = aead_setkey,
1614                         .setauthsize = aead_setauthsize,
1615                         .encrypt = aead_encrypt,
1616                         .decrypt = aead_decrypt,
1617                         .ivsize = AES_BLOCK_SIZE,
1618                         .maxauthsize = MD5_DIGEST_SIZE,
1619                 },
1620                 .caam = {
1621                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1622                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1623                                            OP_ALG_AAI_HMAC_PRECOMP,
1624                         .geniv = true,
1625                 }
1626         },
1627         {
1628                 .aead = {
1629                         .base = {
1630                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1631                                 .cra_driver_name = "authenc-hmac-sha1-"
1632                                                    "cbc-aes-caam-qi",
1633                                 .cra_blocksize = AES_BLOCK_SIZE,
1634                         },
1635                         .setkey = aead_setkey,
1636                         .setauthsize = aead_setauthsize,
1637                         .encrypt = aead_encrypt,
1638                         .decrypt = aead_decrypt,
1639                         .ivsize = AES_BLOCK_SIZE,
1640                         .maxauthsize = SHA1_DIGEST_SIZE,
1641                 },
1642                 .caam = {
1643                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1644                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1645                                            OP_ALG_AAI_HMAC_PRECOMP,
1646                 }
1647         },
1648         {
1649                 .aead = {
1650                         .base = {
1651                                 .cra_name = "echainiv(authenc(hmac(sha1),"
1652                                             "cbc(aes)))",
1653                                 .cra_driver_name = "echainiv-authenc-"
1654                                                    "hmac-sha1-cbc-aes-caam-qi",
1655                                 .cra_blocksize = AES_BLOCK_SIZE,
1656                         },
1657                         .setkey = aead_setkey,
1658                         .setauthsize = aead_setauthsize,
1659                         .encrypt = aead_encrypt,
1660                         .decrypt = aead_decrypt,
1661                         .ivsize = AES_BLOCK_SIZE,
1662                         .maxauthsize = SHA1_DIGEST_SIZE,
1663                 },
1664                 .caam = {
1665                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1666                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1667                                            OP_ALG_AAI_HMAC_PRECOMP,
1668                         .geniv = true,
1669                 },
1670         },
1671         {
1672                 .aead = {
1673                         .base = {
1674                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
1675                                 .cra_driver_name = "authenc-hmac-sha224-"
1676                                                    "cbc-aes-caam-qi",
1677                                 .cra_blocksize = AES_BLOCK_SIZE,
1678                         },
1679                         .setkey = aead_setkey,
1680                         .setauthsize = aead_setauthsize,
1681                         .encrypt = aead_encrypt,
1682                         .decrypt = aead_decrypt,
1683                         .ivsize = AES_BLOCK_SIZE,
1684                         .maxauthsize = SHA224_DIGEST_SIZE,
1685                 },
1686                 .caam = {
1687                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1688                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1689                                            OP_ALG_AAI_HMAC_PRECOMP,
1690                 }
1691         },
1692         {
1693                 .aead = {
1694                         .base = {
1695                                 .cra_name = "echainiv(authenc(hmac(sha224),"
1696                                             "cbc(aes)))",
1697                                 .cra_driver_name = "echainiv-authenc-"
1698                                                    "hmac-sha224-cbc-aes-caam-qi",
1699                                 .cra_blocksize = AES_BLOCK_SIZE,
1700                         },
1701                         .setkey = aead_setkey,
1702                         .setauthsize = aead_setauthsize,
1703                         .encrypt = aead_encrypt,
1704                         .decrypt = aead_decrypt,
1705                         .ivsize = AES_BLOCK_SIZE,
1706                         .maxauthsize = SHA224_DIGEST_SIZE,
1707                 },
1708                 .caam = {
1709                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1710                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1711                                            OP_ALG_AAI_HMAC_PRECOMP,
1712                         .geniv = true,
1713                 }
1714         },
1715         {
1716                 .aead = {
1717                         .base = {
1718                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1719                                 .cra_driver_name = "authenc-hmac-sha256-"
1720                                                    "cbc-aes-caam-qi",
1721                                 .cra_blocksize = AES_BLOCK_SIZE,
1722                         },
1723                         .setkey = aead_setkey,
1724                         .setauthsize = aead_setauthsize,
1725                         .encrypt = aead_encrypt,
1726                         .decrypt = aead_decrypt,
1727                         .ivsize = AES_BLOCK_SIZE,
1728                         .maxauthsize = SHA256_DIGEST_SIZE,
1729                 },
1730                 .caam = {
1731                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1732                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1733                                            OP_ALG_AAI_HMAC_PRECOMP,
1734                 }
1735         },
1736         {
1737                 .aead = {
1738                         .base = {
1739                                 .cra_name = "echainiv(authenc(hmac(sha256),"
1740                                             "cbc(aes)))",
1741                                 .cra_driver_name = "echainiv-authenc-"
1742                                                    "hmac-sha256-cbc-aes-"
1743                                                    "caam-qi",
1744                                 .cra_blocksize = AES_BLOCK_SIZE,
1745                         },
1746                         .setkey = aead_setkey,
1747                         .setauthsize = aead_setauthsize,
1748                         .encrypt = aead_encrypt,
1749                         .decrypt = aead_decrypt,
1750                         .ivsize = AES_BLOCK_SIZE,
1751                         .maxauthsize = SHA256_DIGEST_SIZE,
1752                 },
1753                 .caam = {
1754                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1755                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1756                                            OP_ALG_AAI_HMAC_PRECOMP,
1757                         .geniv = true,
1758                 }
1759         },
1760         {
1761                 .aead = {
1762                         .base = {
1763                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
1764                                 .cra_driver_name = "authenc-hmac-sha384-"
1765                                                    "cbc-aes-caam-qi",
1766                                 .cra_blocksize = AES_BLOCK_SIZE,
1767                         },
1768                         .setkey = aead_setkey,
1769                         .setauthsize = aead_setauthsize,
1770                         .encrypt = aead_encrypt,
1771                         .decrypt = aead_decrypt,
1772                         .ivsize = AES_BLOCK_SIZE,
1773                         .maxauthsize = SHA384_DIGEST_SIZE,
1774                 },
1775                 .caam = {
1776                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1777                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1778                                            OP_ALG_AAI_HMAC_PRECOMP,
1779                 }
1780         },
1781         {
1782                 .aead = {
1783                         .base = {
1784                                 .cra_name = "echainiv(authenc(hmac(sha384),"
1785                                             "cbc(aes)))",
1786                                 .cra_driver_name = "echainiv-authenc-"
1787                                                    "hmac-sha384-cbc-aes-"
1788                                                    "caam-qi",
1789                                 .cra_blocksize = AES_BLOCK_SIZE,
1790                         },
1791                         .setkey = aead_setkey,
1792                         .setauthsize = aead_setauthsize,
1793                         .encrypt = aead_encrypt,
1794                         .decrypt = aead_decrypt,
1795                         .ivsize = AES_BLOCK_SIZE,
1796                         .maxauthsize = SHA384_DIGEST_SIZE,
1797                 },
1798                 .caam = {
1799                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1800                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1801                                            OP_ALG_AAI_HMAC_PRECOMP,
1802                         .geniv = true,
1803                 }
1804         },
1805         {
1806                 .aead = {
1807                         .base = {
1808                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1809                                 .cra_driver_name = "authenc-hmac-sha512-"
1810                                                    "cbc-aes-caam-qi",
1811                                 .cra_blocksize = AES_BLOCK_SIZE,
1812                         },
1813                         .setkey = aead_setkey,
1814                         .setauthsize = aead_setauthsize,
1815                         .encrypt = aead_encrypt,
1816                         .decrypt = aead_decrypt,
1817                         .ivsize = AES_BLOCK_SIZE,
1818                         .maxauthsize = SHA512_DIGEST_SIZE,
1819                 },
1820                 .caam = {
1821                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1822                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1823                                            OP_ALG_AAI_HMAC_PRECOMP,
1824                 }
1825         },
1826         {
1827                 .aead = {
1828                         .base = {
1829                                 .cra_name = "echainiv(authenc(hmac(sha512),"
1830                                             "cbc(aes)))",
1831                                 .cra_driver_name = "echainiv-authenc-"
1832                                                    "hmac-sha512-cbc-aes-"
1833                                                    "caam-qi",
1834                                 .cra_blocksize = AES_BLOCK_SIZE,
1835                         },
1836                         .setkey = aead_setkey,
1837                         .setauthsize = aead_setauthsize,
1838                         .encrypt = aead_encrypt,
1839                         .decrypt = aead_decrypt,
1840                         .ivsize = AES_BLOCK_SIZE,
1841                         .maxauthsize = SHA512_DIGEST_SIZE,
1842                 },
1843                 .caam = {
1844                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1845                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1846                                            OP_ALG_AAI_HMAC_PRECOMP,
1847                         .geniv = true,
1848                 }
1849         },
1850         {
1851                 .aead = {
1852                         .base = {
1853                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
1854                                 .cra_driver_name = "authenc-hmac-md5-"
1855                                                    "cbc-des3_ede-caam-qi",
1856                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1857                         },
1858                         .setkey = des3_aead_setkey,
1859                         .setauthsize = aead_setauthsize,
1860                         .encrypt = aead_encrypt,
1861                         .decrypt = aead_decrypt,
1862                         .ivsize = DES3_EDE_BLOCK_SIZE,
1863                         .maxauthsize = MD5_DIGEST_SIZE,
1864                 },
1865                 .caam = {
1866                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1867                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1868                                            OP_ALG_AAI_HMAC_PRECOMP,
1869                 }
1870         },
1871         {
1872                 .aead = {
1873                         .base = {
1874                                 .cra_name = "echainiv(authenc(hmac(md5),"
1875                                             "cbc(des3_ede)))",
1876                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
1877                                                    "cbc-des3_ede-caam-qi",
1878                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1879                         },
1880                         .setkey = des3_aead_setkey,
1881                         .setauthsize = aead_setauthsize,
1882                         .encrypt = aead_encrypt,
1883                         .decrypt = aead_decrypt,
1884                         .ivsize = DES3_EDE_BLOCK_SIZE,
1885                         .maxauthsize = MD5_DIGEST_SIZE,
1886                 },
1887                 .caam = {
1888                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1889                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1890                                            OP_ALG_AAI_HMAC_PRECOMP,
1891                         .geniv = true,
1892                 }
1893         },
1894         {
1895                 .aead = {
1896                         .base = {
1897                                 .cra_name = "authenc(hmac(sha1),"
1898                                             "cbc(des3_ede))",
1899                                 .cra_driver_name = "authenc-hmac-sha1-"
1900                                                    "cbc-des3_ede-caam-qi",
1901                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1902                         },
1903                         .setkey = des3_aead_setkey,
1904                         .setauthsize = aead_setauthsize,
1905                         .encrypt = aead_encrypt,
1906                         .decrypt = aead_decrypt,
1907                         .ivsize = DES3_EDE_BLOCK_SIZE,
1908                         .maxauthsize = SHA1_DIGEST_SIZE,
1909                 },
1910                 .caam = {
1911                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1912                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1913                                            OP_ALG_AAI_HMAC_PRECOMP,
1914                 },
1915         },
1916         {
1917                 .aead = {
1918                         .base = {
1919                                 .cra_name = "echainiv(authenc(hmac(sha1),"
1920                                             "cbc(des3_ede)))",
1921                                 .cra_driver_name = "echainiv-authenc-"
1922                                                    "hmac-sha1-"
1923                                                    "cbc-des3_ede-caam-qi",
1924                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1925                         },
1926                         .setkey = des3_aead_setkey,
1927                         .setauthsize = aead_setauthsize,
1928                         .encrypt = aead_encrypt,
1929                         .decrypt = aead_decrypt,
1930                         .ivsize = DES3_EDE_BLOCK_SIZE,
1931                         .maxauthsize = SHA1_DIGEST_SIZE,
1932                 },
1933                 .caam = {
1934                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1935                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1936                                            OP_ALG_AAI_HMAC_PRECOMP,
1937                         .geniv = true,
1938                 }
1939         },
1940         {
1941                 .aead = {
1942                         .base = {
1943                                 .cra_name = "authenc(hmac(sha224),"
1944                                             "cbc(des3_ede))",
1945                                 .cra_driver_name = "authenc-hmac-sha224-"
1946                                                    "cbc-des3_ede-caam-qi",
1947                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1948                         },
1949                         .setkey = des3_aead_setkey,
1950                         .setauthsize = aead_setauthsize,
1951                         .encrypt = aead_encrypt,
1952                         .decrypt = aead_decrypt,
1953                         .ivsize = DES3_EDE_BLOCK_SIZE,
1954                         .maxauthsize = SHA224_DIGEST_SIZE,
1955                 },
1956                 .caam = {
1957                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1958                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1959                                            OP_ALG_AAI_HMAC_PRECOMP,
1960                 },
1961         },
1962         {
1963                 .aead = {
1964                         .base = {
1965                                 .cra_name = "echainiv(authenc(hmac(sha224),"
1966                                             "cbc(des3_ede)))",
1967                                 .cra_driver_name = "echainiv-authenc-"
1968                                                    "hmac-sha224-"
1969                                                    "cbc-des3_ede-caam-qi",
1970                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1971                         },
1972                         .setkey = des3_aead_setkey,
1973                         .setauthsize = aead_setauthsize,
1974                         .encrypt = aead_encrypt,
1975                         .decrypt = aead_decrypt,
1976                         .ivsize = DES3_EDE_BLOCK_SIZE,
1977                         .maxauthsize = SHA224_DIGEST_SIZE,
1978                 },
1979                 .caam = {
1980                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1981                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1982                                            OP_ALG_AAI_HMAC_PRECOMP,
1983                         .geniv = true,
1984                 }
1985         },
1986         {
1987                 .aead = {
1988                         .base = {
1989                                 .cra_name = "authenc(hmac(sha256),"
1990                                             "cbc(des3_ede))",
1991                                 .cra_driver_name = "authenc-hmac-sha256-"
1992                                                    "cbc-des3_ede-caam-qi",
1993                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1994                         },
1995                         .setkey = des3_aead_setkey,
1996                         .setauthsize = aead_setauthsize,
1997                         .encrypt = aead_encrypt,
1998                         .decrypt = aead_decrypt,
1999                         .ivsize = DES3_EDE_BLOCK_SIZE,
2000                         .maxauthsize = SHA256_DIGEST_SIZE,
2001                 },
2002                 .caam = {
2003                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2004                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2005                                            OP_ALG_AAI_HMAC_PRECOMP,
2006                 },
2007         },
2008         {
2009                 .aead = {
2010                         .base = {
2011                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2012                                             "cbc(des3_ede)))",
2013                                 .cra_driver_name = "echainiv-authenc-"
2014                                                    "hmac-sha256-"
2015                                                    "cbc-des3_ede-caam-qi",
2016                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2017                         },
2018                         .setkey = des3_aead_setkey,
2019                         .setauthsize = aead_setauthsize,
2020                         .encrypt = aead_encrypt,
2021                         .decrypt = aead_decrypt,
2022                         .ivsize = DES3_EDE_BLOCK_SIZE,
2023                         .maxauthsize = SHA256_DIGEST_SIZE,
2024                 },
2025                 .caam = {
2026                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2027                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2028                                            OP_ALG_AAI_HMAC_PRECOMP,
2029                         .geniv = true,
2030                 }
2031         },
2032         {
2033                 .aead = {
2034                         .base = {
2035                                 .cra_name = "authenc(hmac(sha384),"
2036                                             "cbc(des3_ede))",
2037                                 .cra_driver_name = "authenc-hmac-sha384-"
2038                                                    "cbc-des3_ede-caam-qi",
2039                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2040                         },
2041                         .setkey = des3_aead_setkey,
2042                         .setauthsize = aead_setauthsize,
2043                         .encrypt = aead_encrypt,
2044                         .decrypt = aead_decrypt,
2045                         .ivsize = DES3_EDE_BLOCK_SIZE,
2046                         .maxauthsize = SHA384_DIGEST_SIZE,
2047                 },
2048                 .caam = {
2049                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2050                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2051                                            OP_ALG_AAI_HMAC_PRECOMP,
2052                 },
2053         },
2054         {
2055                 .aead = {
2056                         .base = {
2057                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2058                                             "cbc(des3_ede)))",
2059                                 .cra_driver_name = "echainiv-authenc-"
2060                                                    "hmac-sha384-"
2061                                                    "cbc-des3_ede-caam-qi",
2062                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2063                         },
2064                         .setkey = des3_aead_setkey,
2065                         .setauthsize = aead_setauthsize,
2066                         .encrypt = aead_encrypt,
2067                         .decrypt = aead_decrypt,
2068                         .ivsize = DES3_EDE_BLOCK_SIZE,
2069                         .maxauthsize = SHA384_DIGEST_SIZE,
2070                 },
2071                 .caam = {
2072                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2073                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2074                                            OP_ALG_AAI_HMAC_PRECOMP,
2075                         .geniv = true,
2076                 }
2077         },
2078         {
2079                 .aead = {
2080                         .base = {
2081                                 .cra_name = "authenc(hmac(sha512),"
2082                                             "cbc(des3_ede))",
2083                                 .cra_driver_name = "authenc-hmac-sha512-"
2084                                                    "cbc-des3_ede-caam-qi",
2085                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2086                         },
2087                         .setkey = des3_aead_setkey,
2088                         .setauthsize = aead_setauthsize,
2089                         .encrypt = aead_encrypt,
2090                         .decrypt = aead_decrypt,
2091                         .ivsize = DES3_EDE_BLOCK_SIZE,
2092                         .maxauthsize = SHA512_DIGEST_SIZE,
2093                 },
2094                 .caam = {
2095                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2096                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2097                                            OP_ALG_AAI_HMAC_PRECOMP,
2098                 },
2099         },
2100         {
2101                 .aead = {
2102                         .base = {
2103                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2104                                             "cbc(des3_ede)))",
2105                                 .cra_driver_name = "echainiv-authenc-"
2106                                                    "hmac-sha512-"
2107                                                    "cbc-des3_ede-caam-qi",
2108                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2109                         },
2110                         .setkey = des3_aead_setkey,
2111                         .setauthsize = aead_setauthsize,
2112                         .encrypt = aead_encrypt,
2113                         .decrypt = aead_decrypt,
2114                         .ivsize = DES3_EDE_BLOCK_SIZE,
2115                         .maxauthsize = SHA512_DIGEST_SIZE,
2116                 },
2117                 .caam = {
2118                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2119                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2120                                            OP_ALG_AAI_HMAC_PRECOMP,
2121                         .geniv = true,
2122                 }
2123         },
2124         {
2125                 .aead = {
2126                         .base = {
2127                                 .cra_name = "authenc(hmac(md5),cbc(des))",
2128                                 .cra_driver_name = "authenc-hmac-md5-"
2129                                                    "cbc-des-caam-qi",
2130                                 .cra_blocksize = DES_BLOCK_SIZE,
2131                         },
2132                         .setkey = aead_setkey,
2133                         .setauthsize = aead_setauthsize,
2134                         .encrypt = aead_encrypt,
2135                         .decrypt = aead_decrypt,
2136                         .ivsize = DES_BLOCK_SIZE,
2137                         .maxauthsize = MD5_DIGEST_SIZE,
2138                 },
2139                 .caam = {
2140                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2141                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2142                                            OP_ALG_AAI_HMAC_PRECOMP,
2143                 },
2144         },
2145         {
2146                 .aead = {
2147                         .base = {
2148                                 .cra_name = "echainiv(authenc(hmac(md5),"
2149                                             "cbc(des)))",
2150                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2151                                                    "cbc-des-caam-qi",
2152                                 .cra_blocksize = DES_BLOCK_SIZE,
2153                         },
2154                         .setkey = aead_setkey,
2155                         .setauthsize = aead_setauthsize,
2156                         .encrypt = aead_encrypt,
2157                         .decrypt = aead_decrypt,
2158                         .ivsize = DES_BLOCK_SIZE,
2159                         .maxauthsize = MD5_DIGEST_SIZE,
2160                 },
2161                 .caam = {
2162                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2163                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2164                                            OP_ALG_AAI_HMAC_PRECOMP,
2165                         .geniv = true,
2166                 }
2167         },
2168         {
2169                 .aead = {
2170                         .base = {
2171                                 .cra_name = "authenc(hmac(sha1),cbc(des))",
2172                                 .cra_driver_name = "authenc-hmac-sha1-"
2173                                                    "cbc-des-caam-qi",
2174                                 .cra_blocksize = DES_BLOCK_SIZE,
2175                         },
2176                         .setkey = aead_setkey,
2177                         .setauthsize = aead_setauthsize,
2178                         .encrypt = aead_encrypt,
2179                         .decrypt = aead_decrypt,
2180                         .ivsize = DES_BLOCK_SIZE,
2181                         .maxauthsize = SHA1_DIGEST_SIZE,
2182                 },
2183                 .caam = {
2184                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2185                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2186                                            OP_ALG_AAI_HMAC_PRECOMP,
2187                 },
2188         },
2189         {
2190                 .aead = {
2191                         .base = {
2192                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2193                                             "cbc(des)))",
2194                                 .cra_driver_name = "echainiv-authenc-"
2195                                                    "hmac-sha1-cbc-des-caam-qi",
2196                                 .cra_blocksize = DES_BLOCK_SIZE,
2197                         },
2198                         .setkey = aead_setkey,
2199                         .setauthsize = aead_setauthsize,
2200                         .encrypt = aead_encrypt,
2201                         .decrypt = aead_decrypt,
2202                         .ivsize = DES_BLOCK_SIZE,
2203                         .maxauthsize = SHA1_DIGEST_SIZE,
2204                 },
2205                 .caam = {
2206                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2207                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2208                                            OP_ALG_AAI_HMAC_PRECOMP,
2209                         .geniv = true,
2210                 }
2211         },
2212         {
2213                 .aead = {
2214                         .base = {
2215                                 .cra_name = "authenc(hmac(sha224),cbc(des))",
2216                                 .cra_driver_name = "authenc-hmac-sha224-"
2217                                                    "cbc-des-caam-qi",
2218                                 .cra_blocksize = DES_BLOCK_SIZE,
2219                         },
2220                         .setkey = aead_setkey,
2221                         .setauthsize = aead_setauthsize,
2222                         .encrypt = aead_encrypt,
2223                         .decrypt = aead_decrypt,
2224                         .ivsize = DES_BLOCK_SIZE,
2225                         .maxauthsize = SHA224_DIGEST_SIZE,
2226                 },
2227                 .caam = {
2228                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2229                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2230                                            OP_ALG_AAI_HMAC_PRECOMP,
2231                 },
2232         },
2233         {
2234                 .aead = {
2235                         .base = {
2236                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2237                                             "cbc(des)))",
2238                                 .cra_driver_name = "echainiv-authenc-"
2239                                                    "hmac-sha224-cbc-des-"
2240                                                    "caam-qi",
2241                                 .cra_blocksize = DES_BLOCK_SIZE,
2242                         },
2243                         .setkey = aead_setkey,
2244                         .setauthsize = aead_setauthsize,
2245                         .encrypt = aead_encrypt,
2246                         .decrypt = aead_decrypt,
2247                         .ivsize = DES_BLOCK_SIZE,
2248                         .maxauthsize = SHA224_DIGEST_SIZE,
2249                 },
2250                 .caam = {
2251                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2252                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2253                                            OP_ALG_AAI_HMAC_PRECOMP,
2254                         .geniv = true,
2255                 }
2256         },
2257         {
2258                 .aead = {
2259                         .base = {
2260                                 .cra_name = "authenc(hmac(sha256),cbc(des))",
2261                                 .cra_driver_name = "authenc-hmac-sha256-"
2262                                                    "cbc-des-caam-qi",
2263                                 .cra_blocksize = DES_BLOCK_SIZE,
2264                         },
2265                         .setkey = aead_setkey,
2266                         .setauthsize = aead_setauthsize,
2267                         .encrypt = aead_encrypt,
2268                         .decrypt = aead_decrypt,
2269                         .ivsize = DES_BLOCK_SIZE,
2270                         .maxauthsize = SHA256_DIGEST_SIZE,
2271                 },
2272                 .caam = {
2273                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2274                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2275                                            OP_ALG_AAI_HMAC_PRECOMP,
2276                 },
2277         },
2278         {
2279                 .aead = {
2280                         .base = {
2281                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2282                                             "cbc(des)))",
2283                                 .cra_driver_name = "echainiv-authenc-"
2284                                                    "hmac-sha256-cbc-des-"
2285                                                    "caam-qi",
2286                                 .cra_blocksize = DES_BLOCK_SIZE,
2287                         },
2288                         .setkey = aead_setkey,
2289                         .setauthsize = aead_setauthsize,
2290                         .encrypt = aead_encrypt,
2291                         .decrypt = aead_decrypt,
2292                         .ivsize = DES_BLOCK_SIZE,
2293                         .maxauthsize = SHA256_DIGEST_SIZE,
2294                 },
2295                 .caam = {
2296                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2297                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2298                                            OP_ALG_AAI_HMAC_PRECOMP,
2299                         .geniv = true,
2300                 },
2301         },
2302         {
2303                 .aead = {
2304                         .base = {
2305                                 .cra_name = "authenc(hmac(sha384),cbc(des))",
2306                                 .cra_driver_name = "authenc-hmac-sha384-"
2307                                                    "cbc-des-caam-qi",
2308                                 .cra_blocksize = DES_BLOCK_SIZE,
2309                         },
2310                         .setkey = aead_setkey,
2311                         .setauthsize = aead_setauthsize,
2312                         .encrypt = aead_encrypt,
2313                         .decrypt = aead_decrypt,
2314                         .ivsize = DES_BLOCK_SIZE,
2315                         .maxauthsize = SHA384_DIGEST_SIZE,
2316                 },
2317                 .caam = {
2318                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2319                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2320                                            OP_ALG_AAI_HMAC_PRECOMP,
2321                 },
2322         },
2323         {
2324                 .aead = {
2325                         .base = {
2326                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2327                                             "cbc(des)))",
2328                                 .cra_driver_name = "echainiv-authenc-"
2329                                                    "hmac-sha384-cbc-des-"
2330                                                    "caam-qi",
2331                                 .cra_blocksize = DES_BLOCK_SIZE,
2332                         },
2333                         .setkey = aead_setkey,
2334                         .setauthsize = aead_setauthsize,
2335                         .encrypt = aead_encrypt,
2336                         .decrypt = aead_decrypt,
2337                         .ivsize = DES_BLOCK_SIZE,
2338                         .maxauthsize = SHA384_DIGEST_SIZE,
2339                 },
2340                 .caam = {
2341                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2342                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2343                                            OP_ALG_AAI_HMAC_PRECOMP,
2344                         .geniv = true,
2345                 }
2346         },
2347         {
2348                 .aead = {
2349                         .base = {
2350                                 .cra_name = "authenc(hmac(sha512),cbc(des))",
2351                                 .cra_driver_name = "authenc-hmac-sha512-"
2352                                                    "cbc-des-caam-qi",
2353                                 .cra_blocksize = DES_BLOCK_SIZE,
2354                         },
2355                         .setkey = aead_setkey,
2356                         .setauthsize = aead_setauthsize,
2357                         .encrypt = aead_encrypt,
2358                         .decrypt = aead_decrypt,
2359                         .ivsize = DES_BLOCK_SIZE,
2360                         .maxauthsize = SHA512_DIGEST_SIZE,
2361                 },
2362                 .caam = {
2363                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2364                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2365                                            OP_ALG_AAI_HMAC_PRECOMP,
2366                 }
2367         },
2368         {
2369                 .aead = {
2370                         .base = {
2371                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2372                                             "cbc(des)))",
2373                                 .cra_driver_name = "echainiv-authenc-"
2374                                                    "hmac-sha512-cbc-des-"
2375                                                    "caam-qi",
2376                                 .cra_blocksize = DES_BLOCK_SIZE,
2377                         },
2378                         .setkey = aead_setkey,
2379                         .setauthsize = aead_setauthsize,
2380                         .encrypt = aead_encrypt,
2381                         .decrypt = aead_decrypt,
2382                         .ivsize = DES_BLOCK_SIZE,
2383                         .maxauthsize = SHA512_DIGEST_SIZE,
2384                 },
2385                 .caam = {
2386                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2387                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2388                                            OP_ALG_AAI_HMAC_PRECOMP,
2389                         .geniv = true,
2390                 }
2391         },
2392 };
2393
2394 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
2395                             bool uses_dkp)
2396 {
2397         struct caam_drv_private *priv;
2398         struct device *dev;
2399
2400         /*
2401          * distribute tfms across job rings to ensure in-order
2402          * crypto request processing per tfm
2403          */
2404         ctx->jrdev = caam_jr_alloc();
2405         if (IS_ERR(ctx->jrdev)) {
2406                 pr_err("Job Ring Device allocation for transform failed\n");
2407                 return PTR_ERR(ctx->jrdev);
2408         }
2409
2410         dev = ctx->jrdev->parent;
2411         priv = dev_get_drvdata(dev);
2412         if (priv->era >= 6 && uses_dkp)
2413                 ctx->dir = DMA_BIDIRECTIONAL;
2414         else
2415                 ctx->dir = DMA_TO_DEVICE;
2416
2417         ctx->key_dma = dma_map_single(dev, ctx->key, sizeof(ctx->key),
2418                                       ctx->dir);
2419         if (dma_mapping_error(dev, ctx->key_dma)) {
2420                 dev_err(dev, "unable to map key\n");
2421                 caam_jr_free(ctx->jrdev);
2422                 return -ENOMEM;
2423         }
2424
2425         /* copy descriptor header template value */
2426         ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
2427         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
2428
2429         ctx->qidev = dev;
2430
2431         spin_lock_init(&ctx->lock);
2432         ctx->drv_ctx[ENCRYPT] = NULL;
2433         ctx->drv_ctx[DECRYPT] = NULL;
2434
2435         return 0;
2436 }
2437
2438 static int caam_cra_init(struct crypto_skcipher *tfm)
2439 {
2440         struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
2441         struct caam_skcipher_alg *caam_alg =
2442                 container_of(alg, typeof(*caam_alg), skcipher);
2443
2444         return caam_init_common(crypto_skcipher_ctx(tfm), &caam_alg->caam,
2445                                 false);
2446 }
2447
2448 static int caam_aead_init(struct crypto_aead *tfm)
2449 {
2450         struct aead_alg *alg = crypto_aead_alg(tfm);
2451         struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg),
2452                                                       aead);
2453         struct caam_ctx *ctx = crypto_aead_ctx(tfm);
2454
2455         return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp);
2456 }
2457
2458 static void caam_exit_common(struct caam_ctx *ctx)
2459 {
2460         caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]);
2461         caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]);
2462
2463         dma_unmap_single(ctx->jrdev->parent, ctx->key_dma, sizeof(ctx->key),
2464                          ctx->dir);
2465
2466         caam_jr_free(ctx->jrdev);
2467 }
2468
2469 static void caam_cra_exit(struct crypto_skcipher *tfm)
2470 {
2471         caam_exit_common(crypto_skcipher_ctx(tfm));
2472 }
2473
2474 static void caam_aead_exit(struct crypto_aead *tfm)
2475 {
2476         caam_exit_common(crypto_aead_ctx(tfm));
2477 }
2478
2479 void caam_qi_algapi_exit(void)
2480 {
2481         int i;
2482
2483         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
2484                 struct caam_aead_alg *t_alg = driver_aeads + i;
2485
2486                 if (t_alg->registered)
2487                         crypto_unregister_aead(&t_alg->aead);
2488         }
2489
2490         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2491                 struct caam_skcipher_alg *t_alg = driver_algs + i;
2492
2493                 if (t_alg->registered)
2494                         crypto_unregister_skcipher(&t_alg->skcipher);
2495         }
2496 }
2497
2498 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
2499 {
2500         struct skcipher_alg *alg = &t_alg->skcipher;
2501
2502         alg->base.cra_module = THIS_MODULE;
2503         alg->base.cra_priority = CAAM_CRA_PRIORITY;
2504         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
2505         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
2506                               CRYPTO_ALG_KERN_DRIVER_ONLY;
2507
2508         alg->init = caam_cra_init;
2509         alg->exit = caam_cra_exit;
2510 }
2511
2512 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
2513 {
2514         struct aead_alg *alg = &t_alg->aead;
2515
2516         alg->base.cra_module = THIS_MODULE;
2517         alg->base.cra_priority = CAAM_CRA_PRIORITY;
2518         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
2519         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
2520                               CRYPTO_ALG_KERN_DRIVER_ONLY;
2521
2522         alg->init = caam_aead_init;
2523         alg->exit = caam_aead_exit;
2524 }
2525
2526 int caam_qi_algapi_init(struct device *ctrldev)
2527 {
2528         struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
2529         int i = 0, err = 0;
2530         u32 aes_vid, aes_inst, des_inst, md_vid, md_inst;
2531         unsigned int md_limit = SHA512_DIGEST_SIZE;
2532         bool registered = false;
2533
2534         /* Make sure this runs only on (DPAA 1.x) QI */
2535         if (!priv->qi_present || caam_dpaa2)
2536                 return 0;
2537
2538         /*
2539          * Register crypto algorithms the device supports.
2540          * First, detect presence and attributes of DES, AES, and MD blocks.
2541          */
2542         if (priv->era < 10) {
2543                 u32 cha_vid, cha_inst;
2544
2545                 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
2546                 aes_vid = cha_vid & CHA_ID_LS_AES_MASK;
2547                 md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
2548
2549                 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
2550                 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >>
2551                            CHA_ID_LS_DES_SHIFT;
2552                 aes_inst = cha_inst & CHA_ID_LS_AES_MASK;
2553                 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
2554         } else {
2555                 u32 aesa, mdha;
2556
2557                 aesa = rd_reg32(&priv->ctrl->vreg.aesa);
2558                 mdha = rd_reg32(&priv->ctrl->vreg.mdha);
2559
2560                 aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
2561                 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
2562
2563                 des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK;
2564                 aes_inst = aesa & CHA_VER_NUM_MASK;
2565                 md_inst = mdha & CHA_VER_NUM_MASK;
2566         }
2567
2568         /* If MD is present, limit digest size based on LP256 */
2569         if (md_inst && md_vid  == CHA_VER_VID_MD_LP256)
2570                 md_limit = SHA256_DIGEST_SIZE;
2571
2572         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2573                 struct caam_skcipher_alg *t_alg = driver_algs + i;
2574                 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
2575
2576                 /* Skip DES algorithms if not supported by device */
2577                 if (!des_inst &&
2578                     ((alg_sel == OP_ALG_ALGSEL_3DES) ||
2579                      (alg_sel == OP_ALG_ALGSEL_DES)))
2580                         continue;
2581
2582                 /* Skip AES algorithms if not supported by device */
2583                 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))
2584                         continue;
2585
2586                 caam_skcipher_alg_init(t_alg);
2587
2588                 err = crypto_register_skcipher(&t_alg->skcipher);
2589                 if (err) {
2590                         dev_warn(ctrldev, "%s alg registration failed\n",
2591                                  t_alg->skcipher.base.cra_driver_name);
2592                         continue;
2593                 }
2594
2595                 t_alg->registered = true;
2596                 registered = true;
2597         }
2598
2599         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
2600                 struct caam_aead_alg *t_alg = driver_aeads + i;
2601                 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
2602                                  OP_ALG_ALGSEL_MASK;
2603                 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
2604                                  OP_ALG_ALGSEL_MASK;
2605                 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
2606
2607                 /* Skip DES algorithms if not supported by device */
2608                 if (!des_inst &&
2609                     ((c1_alg_sel == OP_ALG_ALGSEL_3DES) ||
2610                      (c1_alg_sel == OP_ALG_ALGSEL_DES)))
2611                         continue;
2612
2613                 /* Skip AES algorithms if not supported by device */
2614                 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES))
2615                         continue;
2616
2617                 /*
2618                  * Check support for AES algorithms not available
2619                  * on LP devices.
2620                  */
2621                 if (aes_vid  == CHA_VER_VID_AES_LP && alg_aai == OP_ALG_AAI_GCM)
2622                         continue;
2623
2624                 /*
2625                  * Skip algorithms requiring message digests
2626                  * if MD or MD size is not supported by device.
2627                  */
2628                 if (c2_alg_sel &&
2629                     (!md_inst || (t_alg->aead.maxauthsize > md_limit)))
2630                         continue;
2631
2632                 caam_aead_alg_init(t_alg);
2633
2634                 err = crypto_register_aead(&t_alg->aead);
2635                 if (err) {
2636                         pr_warn("%s alg registration failed\n",
2637                                 t_alg->aead.base.cra_driver_name);
2638                         continue;
2639                 }
2640
2641                 t_alg->registered = true;
2642                 registered = true;
2643         }
2644
2645         if (registered)
2646                 dev_info(ctrldev, "algorithms registered in /proc/crypto\n");
2647
2648         return err;
2649 }