drivers/crypto/caam/caamalg_qi2.c

   1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
   2 /*
   3  * Copyright 2015-2016 Freescale Semiconductor Inc.
   4  * Copyright 2017-2019 NXP
   5  */
   6
   7 #include "compat.h"
   8 #include "regs.h"
   9 #include "caamalg_qi2.h"
  10 #include "dpseci_cmd.h"
  11 #include "desc_constr.h"
  12 #include "error.h"
  13 #include "sg_sw_sec4.h"
  14 #include "sg_sw_qm2.h"
  15 #include "key_gen.h"
  16 #include "caamalg_desc.h"
  17 #include "caamhash_desc.h"
  18 #include "dpseci-debugfs.h"
  19 #include <linux/fsl/mc.h>
  20 #include <soc/fsl/dpaa2-io.h>
  21 #include <soc/fsl/dpaa2-fd.h>
  22
  23 #define CAAM_CRA_PRIORITY       2000
  24
  25 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  26 #define CAAM_MAX_KEY_SIZE       (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \
  27                                  SHA512_DIGEST_SIZE * 2)
  28
  29 /*
  30  * This is a a cache of buffers, from which the users of CAAM QI driver
  31  * can allocate short buffers. It's speedier than doing kmalloc on the hotpath.
  32  * NOTE: A more elegant solution would be to have some headroom in the frames
  33  *       being processed. This can be added by the dpaa2-eth driver. This would
  34  *       pose a problem for userspace application processing which cannot
  35  *       know of this limitation. So for now, this will work.
  36  * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here
  37  */
  38 static struct kmem_cache *qi_cache;
  39
  40 struct caam_alg_entry {
  41         struct device *dev;
  42         int class1_alg_type;
  43         int class2_alg_type;
  44         bool rfc3686;
  45         bool geniv;
  46         bool nodkp;
  47 };
  48
  49 struct caam_aead_alg {
  50         struct aead_alg aead;
  51         struct caam_alg_entry caam;
  52         bool registered;
  53 };
  54
  55 struct caam_skcipher_alg {
  56         struct skcipher_alg skcipher;
  57         struct caam_alg_entry caam;
  58         bool registered;
  59 };
  60
  61 /**
  62  * caam_ctx - per-session context
  63  * @flc: Flow Contexts array
  64  * @key:  [authentication key], encryption key
  65  * @flc_dma: I/O virtual addresses of the Flow Contexts
  66  * @key_dma: I/O virtual address of the key
  67  * @dir: DMA direction for mapping key and Flow Contexts
  68  * @dev: dpseci device
  69  * @adata: authentication algorithm details
  70  * @cdata: encryption algorithm details
  71  * @authsize: authentication tag (a.k.a. ICV / MAC) size
  72  */
  73 struct caam_ctx {
  74         struct caam_flc flc[NUM_OP];
  75         u8 key[CAAM_MAX_KEY_SIZE];
  76         dma_addr_t flc_dma[NUM_OP];
  77         dma_addr_t key_dma;
  78         enum dma_data_direction dir;
  79         struct device *dev;
  80         struct alginfo adata;
  81         struct alginfo cdata;
  82         unsigned int authsize;
  83 };
  84
  85 static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv,
  86                                      dma_addr_t iova_addr)
  87 {
  88         phys_addr_t phys_addr;
  89
  90         phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) :
  91                                    iova_addr;
  92
  93         return phys_to_virt(phys_addr);
  94 }
  95
  96 /*
  97  * qi_cache_zalloc - Allocate buffers from CAAM-QI cache
  98  *
  99  * Allocate data on the hotpath. Instead of using kzalloc, one can use the
 100  * services of the CAAM QI memory cache (backed by kmem_cache). The buffers
 101  * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for
 102  * hosting 16 SG entries.
 103  *
 104  * @flags - flags that would be used for the equivalent kmalloc(..) call
 105  *
 106  * Returns a pointer to a retrieved buffer on success or NULL on failure.
 107  */
 108 static inline void *qi_cache_zalloc(gfp_t flags)
 109 {
 110         return kmem_cache_zalloc(qi_cache, flags);
 111 }
 112
 113 /*
 114  * qi_cache_free - Frees buffers allocated from CAAM-QI cache
 115  *
 116  * @obj - buffer previously allocated by qi_cache_zalloc
 117  *
 118  * No checking is being done, the call is a passthrough call to
 119  * kmem_cache_free(...)
 120  */
 121 static inline void qi_cache_free(void *obj)
 122 {
 123         kmem_cache_free(qi_cache, obj);
 124 }
 125
 126 static struct caam_request *to_caam_req(struct crypto_async_request *areq)
 127 {
 128         switch (crypto_tfm_alg_type(areq->tfm)) {
 129         case CRYPTO_ALG_TYPE_SKCIPHER:
 130                 return skcipher_request_ctx(skcipher_request_cast(areq));
 131         case CRYPTO_ALG_TYPE_AEAD:
 132                 return aead_request_ctx(container_of(areq, struct aead_request,
 133                                                      base));
 134         case CRYPTO_ALG_TYPE_AHASH:
 135                 return ahash_request_ctx(ahash_request_cast(areq));
 136         default:
 137                 return ERR_PTR(-EINVAL);
 138         }
 139 }
 140
 141 static void caam_unmap(struct device *dev, struct scatterlist *src,
 142                        struct scatterlist *dst, int src_nents,
 143                        int dst_nents, dma_addr_t iv_dma, int ivsize,
 144                        enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
 145                        int qm_sg_bytes)
 146 {
 147         if (dst != src) {
 148                 if (src_nents)
 149                         dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 150                 if (dst_nents)
 151                         dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
 152         } else {
 153                 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 154         }
 155
 156         if (iv_dma)
 157                 dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
 158
 159         if (qm_sg_bytes)
 160                 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
 161 }
 162
 163 static int aead_set_sh_desc(struct crypto_aead *aead)
 164 {
 165         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 166                                                  typeof(*alg), aead);
 167         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 168         unsigned int ivsize = crypto_aead_ivsize(aead);
 169         struct device *dev = ctx->dev;
 170         struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
 171         struct caam_flc *flc;
 172         u32 *desc;
 173         u32 ctx1_iv_off = 0;
 174         u32 *nonce = NULL;
 175         unsigned int data_len[2];
 176         u32 inl_mask;
 177         const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
 178                                OP_ALG_AAI_CTR_MOD128);
 179         const bool is_rfc3686 = alg->caam.rfc3686;
 180
 181         if (!ctx->cdata.keylen || !ctx->authsize)
 182                 return 0;
 183
 184         /*
 185          * AES-CTR needs to load IV in CONTEXT1 reg
 186          * at an offset of 128bits (16bytes)
 187          * CONTEXT1[255:128] = IV
 188          */
 189         if (ctr_mode)
 190                 ctx1_iv_off = 16;
 191
 192         /*
 193          * RFC3686 specific:
 194          *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 195          */
 196         if (is_rfc3686) {
 197                 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 198                 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
 199                                 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
 200         }
 201
 202         /*
 203          * In case |user key| > |derived key|, using DKP<imm,imm> would result
 204          * in invalid opcodes (last bytes of user key) in the resulting
 205          * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
 206          * addresses are needed.
 207          */
 208         ctx->adata.key_virt = ctx->key;
 209         ctx->adata.key_dma = ctx->key_dma;
 210
 211         ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 212         ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 213
 214         data_len[0] = ctx->adata.keylen_pad;
 215         data_len[1] = ctx->cdata.keylen;
 216
 217         /* aead_encrypt shared descriptor */
 218         if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN :
 219                                                  DESC_QI_AEAD_ENC_LEN) +
 220                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 221                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 222                               ARRAY_SIZE(data_len)) < 0)
 223                 return -EINVAL;
 224
 225         ctx->adata.key_inline = !!(inl_mask & 1);
 226         ctx->cdata.key_inline = !!(inl_mask & 2);
 227
 228         flc = &ctx->flc[ENCRYPT];
 229         desc = flc->sh_desc;
 230
 231         if (alg->caam.geniv)
 232                 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata,
 233                                           ivsize, ctx->authsize, is_rfc3686,
 234                                           nonce, ctx1_iv_off, true,
 235                                           priv->sec_attr.era);
 236         else
 237                 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata,
 238                                        ivsize, ctx->authsize, is_rfc3686, nonce,
 239                                        ctx1_iv_off, true, priv->sec_attr.era);
 240
 241         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 242         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 243                                    sizeof(flc->flc) + desc_bytes(desc),
 244                                    ctx->dir);
 245
 246         /* aead_decrypt shared descriptor */
 247         if (desc_inline_query(DESC_QI_AEAD_DEC_LEN +
 248                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 249                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 250                               ARRAY_SIZE(data_len)) < 0)
 251                 return -EINVAL;
 252
 253         ctx->adata.key_inline = !!(inl_mask & 1);
 254         ctx->cdata.key_inline = !!(inl_mask & 2);
 255
 256         flc = &ctx->flc[DECRYPT];
 257         desc = flc->sh_desc;
 258         cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata,
 259                                ivsize, ctx->authsize, alg->caam.geniv,
 260                                is_rfc3686, nonce, ctx1_iv_off, true,
 261                                priv->sec_attr.era);
 262         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 263         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 264                                    sizeof(flc->flc) + desc_bytes(desc),
 265                                    ctx->dir);
 266
 267         return 0;
 268 }
 269
 270 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 271 {
 272         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 273
 274         ctx->authsize = authsize;
 275         aead_set_sh_desc(authenc);
 276
 277         return 0;
 278 }
 279
 280 static int aead_setkey(struct crypto_aead *aead, const u8 *key,
 281                        unsigned int keylen)
 282 {
 283         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 284         struct device *dev = ctx->dev;
 285         struct crypto_authenc_keys keys;
 286
 287         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 288                 goto badkey;
 289
 290         dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n",
 291                 keys.authkeylen + keys.enckeylen, keys.enckeylen,
 292                 keys.authkeylen);
 293         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 294                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 295
 296         ctx->adata.keylen = keys.authkeylen;
 297         ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
 298                                               OP_ALG_ALGSEL_MASK);
 299
 300         if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
 301                 goto badkey;
 302
 303         memcpy(ctx->key, keys.authkey, keys.authkeylen);
 304         memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
 305         dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad +
 306                                    keys.enckeylen, ctx->dir);
 307         print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
 308                              DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 309                              ctx->adata.keylen_pad + keys.enckeylen, 1);
 310
 311         ctx->cdata.keylen = keys.enckeylen;
 312
 313         memzero_explicit(&keys, sizeof(keys));
 314         return aead_set_sh_desc(aead);
 315 badkey:
 316         memzero_explicit(&keys, sizeof(keys));
 317         return -EINVAL;
 318 }
 319
 320 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
 321                             unsigned int keylen)
 322 {
 323         struct crypto_authenc_keys keys;
 324         int err;
 325
 326         err = crypto_authenc_extractkeys(&keys, key, keylen);
 327         if (unlikely(err))
 328                 goto out;
 329
 330         err = -EINVAL;
 331         if (keys.enckeylen != DES3_EDE_KEY_SIZE)
 332                 goto out;
 333
 334         err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?:
 335               aead_setkey(aead, key, keylen);
 336
 337 out:
 338         memzero_explicit(&keys, sizeof(keys));
 339         return err;
 340 }
 341
 342 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
 343                                            bool encrypt)
 344 {
 345         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 346         struct caam_request *req_ctx = aead_request_ctx(req);
 347         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
 348         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
 349         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 350         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 351                                                  typeof(*alg), aead);
 352         struct device *dev = ctx->dev;
 353         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 354                       GFP_KERNEL : GFP_ATOMIC;
 355         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
 356         int src_len, dst_len = 0;
 357         struct aead_edesc *edesc;
 358         dma_addr_t qm_sg_dma, iv_dma = 0;
 359         int ivsize = 0;
 360         unsigned int authsize = ctx->authsize;
 361         int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes;
 362         int in_len, out_len;
 363         struct dpaa2_sg_entry *sg_table;
 364
 365         /* allocate space for base edesc, link tables and IV */
 366         edesc = qi_cache_zalloc(GFP_DMA | flags);
 367         if (unlikely(!edesc)) {
 368                 dev_err(dev, "could not allocate extended descriptor\n");
 369                 return ERR_PTR(-ENOMEM);
 370         }
 371
 372         if (unlikely(req->dst != req->src)) {
 373                 src_len = req->assoclen + req->cryptlen;
 374                 dst_len = src_len + (encrypt ? authsize : (-authsize));
 375
 376                 src_nents = sg_nents_for_len(req->src, src_len);
 377                 if (unlikely(src_nents < 0)) {
 378                         dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
 379                                 src_len);
 380                         qi_cache_free(edesc);
 381                         return ERR_PTR(src_nents);
 382                 }
 383
 384                 dst_nents = sg_nents_for_len(req->dst, dst_len);
 385                 if (unlikely(dst_nents < 0)) {
 386                         dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
 387                                 dst_len);
 388                         qi_cache_free(edesc);
 389                         return ERR_PTR(dst_nents);
 390                 }
 391
 392                 if (src_nents) {
 393                         mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
 394                                                       DMA_TO_DEVICE);
 395                         if (unlikely(!mapped_src_nents)) {
 396                                 dev_err(dev, "unable to map source\n");
 397                                 qi_cache_free(edesc);
 398                                 return ERR_PTR(-ENOMEM);
 399                         }
 400                 } else {
 401                         mapped_src_nents = 0;
 402                 }
 403
 404                 if (dst_nents) {
 405                         mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
 406                                                       DMA_FROM_DEVICE);
 407                         if (unlikely(!mapped_dst_nents)) {
 408                                 dev_err(dev, "unable to map destination\n");
 409                                 dma_unmap_sg(dev, req->src, src_nents,
 410                                              DMA_TO_DEVICE);
 411                                 qi_cache_free(edesc);
 412                                 return ERR_PTR(-ENOMEM);
 413                         }
 414                 } else {
 415                         mapped_dst_nents = 0;
 416                 }
 417         } else {
 418                 src_len = req->assoclen + req->cryptlen +
 419                           (encrypt ? authsize : 0);
 420
 421                 src_nents = sg_nents_for_len(req->src, src_len);
 422                 if (unlikely(src_nents < 0)) {
 423                         dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
 424                                 src_len);
 425                         qi_cache_free(edesc);
 426                         return ERR_PTR(src_nents);
 427                 }
 428
 429                 mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
 430                                               DMA_BIDIRECTIONAL);
 431                 if (unlikely(!mapped_src_nents)) {
 432                         dev_err(dev, "unable to map source\n");
 433                         qi_cache_free(edesc);
 434                         return ERR_PTR(-ENOMEM);
 435                 }
 436         }
 437
 438         if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv)
 439                 ivsize = crypto_aead_ivsize(aead);
 440
 441         /*
 442          * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
 443          * Input is not contiguous.
 444          * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
 445          * the end of the table by allocating more S/G entries. Logic:
 446          * if (src != dst && output S/G)
 447          *      pad output S/G, if needed
 448          * else if (src == dst && S/G)
 449          *      overlapping S/Gs; pad one of them
 450          * else if (input S/G) ...
 451          *      pad input S/G, if needed
 452          */
 453         qm_sg_nents = 1 + !!ivsize + mapped_src_nents;
 454         if (mapped_dst_nents > 1)
 455                 qm_sg_nents += pad_sg_nents(mapped_dst_nents);
 456         else if ((req->src == req->dst) && (mapped_src_nents > 1))
 457                 qm_sg_nents = max(pad_sg_nents(qm_sg_nents),
 458                                   1 + !!ivsize +
 459                                   pad_sg_nents(mapped_src_nents));
 460         else
 461                 qm_sg_nents = pad_sg_nents(qm_sg_nents);
 462
 463         sg_table = &edesc->sgt[0];
 464         qm_sg_bytes = qm_sg_nents * sizeof(*sg_table);
 465         if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
 466                      CAAM_QI_MEMCACHE_SIZE)) {
 467                 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
 468                         qm_sg_nents, ivsize);
 469                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
 470                            0, DMA_NONE, 0, 0);
 471                 qi_cache_free(edesc);
 472                 return ERR_PTR(-ENOMEM);
 473         }
 474
 475         if (ivsize) {
 476                 u8 *iv = (u8 *)(sg_table + qm_sg_nents);
 477
 478                 /* Make sure IV is located in a DMAable area */
 479                 memcpy(iv, req->iv, ivsize);
 480
 481                 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
 482                 if (dma_mapping_error(dev, iv_dma)) {
 483                         dev_err(dev, "unable to map IV\n");
 484                         caam_unmap(dev, req->src, req->dst, src_nents,
 485                                    dst_nents, 0, 0, DMA_NONE, 0, 0);
 486                         qi_cache_free(edesc);
 487                         return ERR_PTR(-ENOMEM);
 488                 }
 489         }
 490
 491         edesc->src_nents = src_nents;
 492         edesc->dst_nents = dst_nents;
 493         edesc->iv_dma = iv_dma;
 494
 495         if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) ==
 496             OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE)
 497                 /*
 498                  * The associated data comes already with the IV but we need
 499                  * to skip it when we authenticate or encrypt...
 500                  */
 501                 edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize);
 502         else
 503                 edesc->assoclen = cpu_to_caam32(req->assoclen);
 504         edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4,
 505                                              DMA_TO_DEVICE);
 506         if (dma_mapping_error(dev, edesc->assoclen_dma)) {
 507                 dev_err(dev, "unable to map assoclen\n");
 508                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
 509                            iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
 510                 qi_cache_free(edesc);
 511                 return ERR_PTR(-ENOMEM);
 512         }
 513
 514         dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0);
 515         qm_sg_index++;
 516         if (ivsize) {
 517                 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
 518                 qm_sg_index++;
 519         }
 520         sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
 521         qm_sg_index += mapped_src_nents;
 522
 523         if (mapped_dst_nents > 1)
 524                 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
 525
 526         qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
 527         if (dma_mapping_error(dev, qm_sg_dma)) {
 528                 dev_err(dev, "unable to map S/G table\n");
 529                 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
 530                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
 531                            iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
 532                 qi_cache_free(edesc);
 533                 return ERR_PTR(-ENOMEM);
 534         }
 535
 536         edesc->qm_sg_dma = qm_sg_dma;
 537         edesc->qm_sg_bytes = qm_sg_bytes;
 538
 539         out_len = req->assoclen + req->cryptlen +
 540                   (encrypt ? ctx->authsize : (-ctx->authsize));
 541         in_len = 4 + ivsize + req->assoclen + req->cryptlen;
 542
 543         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
 544         dpaa2_fl_set_final(in_fle, true);
 545         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
 546         dpaa2_fl_set_addr(in_fle, qm_sg_dma);
 547         dpaa2_fl_set_len(in_fle, in_len);
 548
 549         if (req->dst == req->src) {
 550                 if (mapped_src_nents == 1) {
 551                         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
 552                         dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src));
 553                 } else {
 554                         dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
 555                         dpaa2_fl_set_addr(out_fle, qm_sg_dma +
 556                                           (1 + !!ivsize) * sizeof(*sg_table));
 557                 }
 558         } else if (!mapped_dst_nents) {
 559                 /*
 560                  * crypto engine requires the output entry to be present when
 561                  * "frame list" FD is used.
 562                  * Since engine does not support FMT=2'b11 (unused entry type),
 563                  * leaving out_fle zeroized is the best option.
 564                  */
 565                 goto skip_out_fle;
 566         } else if (mapped_dst_nents == 1) {
 567                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
 568                 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
 569         } else {
 570                 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
 571                 dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index *
 572                                   sizeof(*sg_table));
 573         }
 574
 575         dpaa2_fl_set_len(out_fle, out_len);
 576
 577 skip_out_fle:
 578         return edesc;
 579 }
 580
 581 static int chachapoly_set_sh_desc(struct crypto_aead *aead)
 582 {
 583         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 584         unsigned int ivsize = crypto_aead_ivsize(aead);
 585         struct device *dev = ctx->dev;
 586         struct caam_flc *flc;
 587         u32 *desc;
 588
 589         if (!ctx->cdata.keylen || !ctx->authsize)
 590                 return 0;
 591
 592         flc = &ctx->flc[ENCRYPT];
 593         desc = flc->sh_desc;
 594         cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
 595                                ctx->authsize, true, true);
 596         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 597         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 598                                    sizeof(flc->flc) + desc_bytes(desc),
 599                                    ctx->dir);
 600
 601         flc = &ctx->flc[DECRYPT];
 602         desc = flc->sh_desc;
 603         cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
 604                                ctx->authsize, false, true);
 605         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 606         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 607                                    sizeof(flc->flc) + desc_bytes(desc),
 608                                    ctx->dir);
 609
 610         return 0;
 611 }
 612
 613 static int chachapoly_setauthsize(struct crypto_aead *aead,
 614                                   unsigned int authsize)
 615 {
 616         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 617
 618         if (authsize != POLY1305_DIGEST_SIZE)
 619                 return -EINVAL;
 620
 621         ctx->authsize = authsize;
 622         return chachapoly_set_sh_desc(aead);
 623 }
 624
 625 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
 626                              unsigned int keylen)
 627 {
 628         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 629         unsigned int ivsize = crypto_aead_ivsize(aead);
 630         unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize;
 631
 632         if (keylen != CHACHA_KEY_SIZE + saltlen)
 633                 return -EINVAL;
 634
 635         ctx->cdata.key_virt = key;
 636         ctx->cdata.keylen = keylen - saltlen;
 637
 638         return chachapoly_set_sh_desc(aead);
 639 }
 640
 641 static int gcm_set_sh_desc(struct crypto_aead *aead)
 642 {
 643         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 644         struct device *dev = ctx->dev;
 645         unsigned int ivsize = crypto_aead_ivsize(aead);
 646         struct caam_flc *flc;
 647         u32 *desc;
 648         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 649                         ctx->cdata.keylen;
 650
 651         if (!ctx->cdata.keylen || !ctx->authsize)
 652                 return 0;
 653
 654         /*
 655          * AES GCM encrypt shared descriptor
 656          * Job Descriptor and Shared Descriptor
 657          * must fit into the 64-word Descriptor h/w Buffer
 658          */
 659         if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {
 660                 ctx->cdata.key_inline = true;
 661                 ctx->cdata.key_virt = ctx->key;
 662         } else {
 663                 ctx->cdata.key_inline = false;
 664                 ctx->cdata.key_dma = ctx->key_dma;
 665         }
 666
 667         flc = &ctx->flc[ENCRYPT];
 668         desc = flc->sh_desc;
 669         cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
 670         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 671         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 672                                    sizeof(flc->flc) + desc_bytes(desc),
 673                                    ctx->dir);
 674
 675         /*
 676          * Job Descriptor and Shared Descriptors
 677          * must all fit into the 64-word Descriptor h/w Buffer
 678          */
 679         if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {
 680                 ctx->cdata.key_inline = true;
 681                 ctx->cdata.key_virt = ctx->key;
 682         } else {
 683                 ctx->cdata.key_inline = false;
 684                 ctx->cdata.key_dma = ctx->key_dma;
 685         }
 686
 687         flc = &ctx->flc[DECRYPT];
 688         desc = flc->sh_desc;
 689         cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
 690         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 691         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 692                                    sizeof(flc->flc) + desc_bytes(desc),
 693                                    ctx->dir);
 694
 695         return 0;
 696 }
 697
 698 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 699 {
 700         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 701         int err;
 702
 703         err = crypto_gcm_check_authsize(authsize);
 704         if (err)
 705                 return err;
 706
 707         ctx->authsize = authsize;
 708         gcm_set_sh_desc(authenc);
 709
 710         return 0;
 711 }
 712
 713 static int gcm_setkey(struct crypto_aead *aead,
 714                       const u8 *key, unsigned int keylen)
 715 {
 716         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 717         struct device *dev = ctx->dev;
 718         int ret;
 719
 720         ret = aes_check_keylen(keylen);
 721         if (ret)
 722                 return ret;
 723         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 724                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 725
 726         memcpy(ctx->key, key, keylen);
 727         dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir);
 728         ctx->cdata.keylen = keylen;
 729
 730         return gcm_set_sh_desc(aead);
 731 }
 732
 733 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 734 {
 735         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 736         struct device *dev = ctx->dev;
 737         unsigned int ivsize = crypto_aead_ivsize(aead);
 738         struct caam_flc *flc;
 739         u32 *desc;
 740         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 741                         ctx->cdata.keylen;
 742
 743         if (!ctx->cdata.keylen || !ctx->authsize)
 744                 return 0;
 745
 746         ctx->cdata.key_virt = ctx->key;
 747
 748         /*
 749          * RFC4106 encrypt shared descriptor
 750          * Job Descriptor and Shared Descriptor
 751          * must fit into the 64-word Descriptor h/w Buffer
 752          */
 753         if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {
 754                 ctx->cdata.key_inline = true;
 755         } else {
 756                 ctx->cdata.key_inline = false;
 757                 ctx->cdata.key_dma = ctx->key_dma;
 758         }
 759
 760         flc = &ctx->flc[ENCRYPT];
 761         desc = flc->sh_desc;
 762         cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
 763                                   true);
 764         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 765         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 766                                    sizeof(flc->flc) + desc_bytes(desc),
 767                                    ctx->dir);
 768
 769         /*
 770          * Job Descriptor and Shared Descriptors
 771          * must all fit into the 64-word Descriptor h/w Buffer
 772          */
 773         if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {
 774                 ctx->cdata.key_inline = true;
 775         } else {
 776                 ctx->cdata.key_inline = false;
 777                 ctx->cdata.key_dma = ctx->key_dma;
 778         }
 779
 780         flc = &ctx->flc[DECRYPT];
 781         desc = flc->sh_desc;
 782         cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
 783                                   true);
 784         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 785         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 786                                    sizeof(flc->flc) + desc_bytes(desc),
 787                                    ctx->dir);
 788
 789         return 0;
 790 }
 791
 792 static int rfc4106_setauthsize(struct crypto_aead *authenc,
 793                                unsigned int authsize)
 794 {
 795         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 796         int err;
 797
 798         err = crypto_rfc4106_check_authsize(authsize);
 799         if (err)
 800                 return err;
 801
 802         ctx->authsize = authsize;
 803         rfc4106_set_sh_desc(authenc);
 804
 805         return 0;
 806 }
 807
 808 static int rfc4106_setkey(struct crypto_aead *aead,
 809                           const u8 *key, unsigned int keylen)
 810 {
 811         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 812         struct device *dev = ctx->dev;
 813         int ret;
 814
 815         ret = aes_check_keylen(keylen - 4);
 816         if (ret)
 817                 return ret;
 818
 819         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 820                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 821
 822         memcpy(ctx->key, key, keylen);
 823         /*
 824          * The last four bytes of the key material are used as the salt value
 825          * in the nonce. Update the AES key length.
 826          */
 827         ctx->cdata.keylen = keylen - 4;
 828         dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
 829                                    ctx->dir);
 830
 831         return rfc4106_set_sh_desc(aead);
 832 }
 833
 834 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
 835 {
 836         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 837         struct device *dev = ctx->dev;
 838         unsigned int ivsize = crypto_aead_ivsize(aead);
 839         struct caam_flc *flc;
 840         u32 *desc;
 841         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 842                         ctx->cdata.keylen;
 843
 844         if (!ctx->cdata.keylen || !ctx->authsize)
 845                 return 0;
 846
 847         ctx->cdata.key_virt = ctx->key;
 848
 849         /*
 850          * RFC4543 encrypt shared descriptor
 851          * Job Descriptor and Shared Descriptor
 852          * must fit into the 64-word Descriptor h/w Buffer
 853          */
 854         if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {
 855                 ctx->cdata.key_inline = true;
 856         } else {
 857                 ctx->cdata.key_inline = false;
 858                 ctx->cdata.key_dma = ctx->key_dma;
 859         }
 860
 861         flc = &ctx->flc[ENCRYPT];
 862         desc = flc->sh_desc;
 863         cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
 864                                   true);
 865         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 866         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 867                                    sizeof(flc->flc) + desc_bytes(desc),
 868                                    ctx->dir);
 869
 870         /*
 871          * Job Descriptor and Shared Descriptors
 872          * must all fit into the 64-word Descriptor h/w Buffer
 873          */
 874         if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {
 875                 ctx->cdata.key_inline = true;
 876         } else {
 877                 ctx->cdata.key_inline = false;
 878                 ctx->cdata.key_dma = ctx->key_dma;
 879         }
 880
 881         flc = &ctx->flc[DECRYPT];
 882         desc = flc->sh_desc;
 883         cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
 884                                   true);
 885         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 886         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 887                                    sizeof(flc->flc) + desc_bytes(desc),
 888                                    ctx->dir);
 889
 890         return 0;
 891 }
 892
 893 static int rfc4543_setauthsize(struct crypto_aead *authenc,
 894                                unsigned int authsize)
 895 {
 896         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 897
 898         if (authsize != 16)
 899                 return -EINVAL;
 900
 901         ctx->authsize = authsize;
 902         rfc4543_set_sh_desc(authenc);
 903
 904         return 0;
 905 }
 906
 907 static int rfc4543_setkey(struct crypto_aead *aead,
 908                           const u8 *key, unsigned int keylen)
 909 {
 910         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 911         struct device *dev = ctx->dev;
 912         int ret;
 913
 914         ret = aes_check_keylen(keylen - 4);
 915         if (ret)
 916                 return ret;
 917
 918         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 919                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 920
 921         memcpy(ctx->key, key, keylen);
 922         /*
 923          * The last four bytes of the key material are used as the salt value
 924          * in the nonce. Update the AES key length.
 925          */
 926         ctx->cdata.keylen = keylen - 4;
 927         dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
 928                                    ctx->dir);
 929
 930         return rfc4543_set_sh_desc(aead);
 931 }
 932
 933 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 934                            unsigned int keylen, const u32 ctx1_iv_off)
 935 {
 936         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
 937         struct caam_skcipher_alg *alg =
 938                 container_of(crypto_skcipher_alg(skcipher),
 939                              struct caam_skcipher_alg, skcipher);
 940         struct device *dev = ctx->dev;
 941         struct caam_flc *flc;
 942         unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
 943         u32 *desc;
 944         const bool is_rfc3686 = alg->caam.rfc3686;
 945
 946         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 947                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 948
 949         ctx->cdata.keylen = keylen;
 950         ctx->cdata.key_virt = key;
 951         ctx->cdata.key_inline = true;
 952
 953         /* skcipher_encrypt shared descriptor */
 954         flc = &ctx->flc[ENCRYPT];
 955         desc = flc->sh_desc;
 956         cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
 957                                    ctx1_iv_off);
 958         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 959         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 960                                    sizeof(flc->flc) + desc_bytes(desc),
 961                                    ctx->dir);
 962
 963         /* skcipher_decrypt shared descriptor */
 964         flc = &ctx->flc[DECRYPT];
 965         desc = flc->sh_desc;
 966         cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
 967                                    ctx1_iv_off);
 968         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 969         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 970                                    sizeof(flc->flc) + desc_bytes(desc),
 971                                    ctx->dir);
 972
 973         return 0;
 974 }
 975
 976 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
 977                                const u8 *key, unsigned int keylen)
 978 {
 979         int err;
 980
 981         err = aes_check_keylen(keylen);
 982         if (err)
 983                 return err;
 984
 985         return skcipher_setkey(skcipher, key, keylen, 0);
 986 }
 987
 988 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
 989                                    const u8 *key, unsigned int keylen)
 990 {
 991         u32 ctx1_iv_off;
 992         int err;
 993
 994         /*
 995          * RFC3686 specific:
 996          *      | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 997          *      | *key = {KEY, NONCE}
 998          */
 999         ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
1000         keylen -= CTR_RFC3686_NONCE_SIZE;
1001
1002         err = aes_check_keylen(keylen);
1003         if (err)
1004                 return err;
1005
1006         return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1007 }
1008
1009 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
1010                                const u8 *key, unsigned int keylen)
1011 {
1012         u32 ctx1_iv_off;
1013         int err;
1014
1015         /*
1016          * AES-CTR needs to load IV in CONTEXT1 reg
1017          * at an offset of 128bits (16bytes)
1018          * CONTEXT1[255:128] = IV
1019          */
1020         ctx1_iv_off = 16;
1021
1022         err = aes_check_keylen(keylen);
1023         if (err)
1024                 return err;
1025
1026         return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1027 }
1028
1029 static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher,
1030                                     const u8 *key, unsigned int keylen)
1031 {
1032         if (keylen != CHACHA_KEY_SIZE)
1033                 return -EINVAL;
1034
1035         return skcipher_setkey(skcipher, key, keylen, 0);
1036 }
1037
1038 static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
1039                                const u8 *key, unsigned int keylen)
1040 {
1041         return verify_skcipher_des_key(skcipher, key) ?:
1042                skcipher_setkey(skcipher, key, keylen, 0);
1043 }
1044
1045 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
1046                                 const u8 *key, unsigned int keylen)
1047 {
1048         return verify_skcipher_des3_key(skcipher, key) ?:
1049                skcipher_setkey(skcipher, key, keylen, 0);
1050 }
1051
1052 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
1053                                unsigned int keylen)
1054 {
1055         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1056         struct device *dev = ctx->dev;
1057         struct caam_flc *flc;
1058         u32 *desc;
1059
1060         if (keylen != 2 * AES_MIN_KEY_SIZE  && keylen != 2 * AES_MAX_KEY_SIZE) {
1061                 dev_dbg(dev, "key size mismatch\n");
1062                 return -EINVAL;
1063         }
1064
1065         ctx->cdata.keylen = keylen;
1066         ctx->cdata.key_virt = key;
1067         ctx->cdata.key_inline = true;
1068
1069         /* xts_skcipher_encrypt shared descriptor */
1070         flc = &ctx->flc[ENCRYPT];
1071         desc = flc->sh_desc;
1072         cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata);
1073         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1074         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
1075                                    sizeof(flc->flc) + desc_bytes(desc),
1076                                    ctx->dir);
1077
1078         /* xts_skcipher_decrypt shared descriptor */
1079         flc = &ctx->flc[DECRYPT];
1080         desc = flc->sh_desc;
1081         cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata);
1082         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1083         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
1084                                    sizeof(flc->flc) + desc_bytes(desc),
1085                                    ctx->dir);
1086
1087         return 0;
1088 }
1089
1090 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
1091 {
1092         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1093         struct caam_request *req_ctx = skcipher_request_ctx(req);
1094         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
1095         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
1096         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1097         struct device *dev = ctx->dev;
1098         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1099                        GFP_KERNEL : GFP_ATOMIC;
1100         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1101         struct skcipher_edesc *edesc;
1102         dma_addr_t iv_dma;
1103         u8 *iv;
1104         int ivsize = crypto_skcipher_ivsize(skcipher);
1105         int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
1106         struct dpaa2_sg_entry *sg_table;
1107
1108         src_nents = sg_nents_for_len(req->src, req->cryptlen);
1109         if (unlikely(src_nents < 0)) {
1110                 dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
1111                         req->cryptlen);
1112                 return ERR_PTR(src_nents);
1113         }
1114
1115         if (unlikely(req->dst != req->src)) {
1116                 dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
1117                 if (unlikely(dst_nents < 0)) {
1118                         dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
1119                                 req->cryptlen);
1120                         return ERR_PTR(dst_nents);
1121                 }
1122
1123                 mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1124                                               DMA_TO_DEVICE);
1125                 if (unlikely(!mapped_src_nents)) {
1126                         dev_err(dev, "unable to map source\n");
1127                         return ERR_PTR(-ENOMEM);
1128                 }
1129
1130                 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
1131                                               DMA_FROM_DEVICE);
1132                 if (unlikely(!mapped_dst_nents)) {
1133                         dev_err(dev, "unable to map destination\n");
1134                         dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
1135                         return ERR_PTR(-ENOMEM);
1136                 }
1137         } else {
1138                 mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1139                                               DMA_BIDIRECTIONAL);
1140                 if (unlikely(!mapped_src_nents)) {
1141                         dev_err(dev, "unable to map source\n");
1142                         return ERR_PTR(-ENOMEM);
1143                 }
1144         }
1145
1146         qm_sg_ents = 1 + mapped_src_nents;
1147         dst_sg_idx = qm_sg_ents;
1148
1149         /*
1150          * Input, output HW S/G tables: [IV, src][dst, IV]
1151          * IV entries point to the same buffer
1152          * If src == dst, S/G entries are reused (S/G tables overlap)
1153          *
1154          * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1155          * the end of the table by allocating more S/G entries.
1156          */
1157         if (req->src != req->dst)
1158                 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
1159         else
1160                 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
1161
1162         qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry);
1163         if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
1164                      ivsize > CAAM_QI_MEMCACHE_SIZE)) {
1165                 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
1166                         qm_sg_ents, ivsize);
1167                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1168                            0, DMA_NONE, 0, 0);
1169                 return ERR_PTR(-ENOMEM);
1170         }
1171
1172         /* allocate space for base edesc, link tables and IV */
1173         edesc = qi_cache_zalloc(GFP_DMA | flags);
1174         if (unlikely(!edesc)) {
1175                 dev_err(dev, "could not allocate extended descriptor\n");
1176                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1177                            0, DMA_NONE, 0, 0);
1178                 return ERR_PTR(-ENOMEM);
1179         }
1180
1181         /* Make sure IV is located in a DMAable area */
1182         sg_table = &edesc->sgt[0];
1183         iv = (u8 *)(sg_table + qm_sg_ents);
1184         memcpy(iv, req->iv, ivsize);
1185
1186         iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL);
1187         if (dma_mapping_error(dev, iv_dma)) {
1188                 dev_err(dev, "unable to map IV\n");
1189                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1190                            0, DMA_NONE, 0, 0);
1191                 qi_cache_free(edesc);
1192                 return ERR_PTR(-ENOMEM);
1193         }
1194
1195         edesc->src_nents = src_nents;
1196         edesc->dst_nents = dst_nents;
1197         edesc->iv_dma = iv_dma;
1198         edesc->qm_sg_bytes = qm_sg_bytes;
1199
1200         dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
1201         sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
1202
1203         if (req->src != req->dst)
1204                 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
1205
1206         dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
1207                          ivsize, 0);
1208
1209         edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes,
1210                                           DMA_TO_DEVICE);
1211         if (dma_mapping_error(dev, edesc->qm_sg_dma)) {
1212                 dev_err(dev, "unable to map S/G table\n");
1213                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
1214                            iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
1215                 qi_cache_free(edesc);
1216                 return ERR_PTR(-ENOMEM);
1217         }
1218
1219         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
1220         dpaa2_fl_set_final(in_fle, true);
1221         dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize);
1222         dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize);
1223
1224         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
1225         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
1226
1227         dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
1228
1229         if (req->src == req->dst)
1230                 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma +
1231                                   sizeof(*sg_table));
1232         else
1233                 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx *
1234                                   sizeof(*sg_table));
1235
1236         return edesc;
1237 }
1238
1239 static void aead_unmap(struct device *dev, struct aead_edesc *edesc,
1240                        struct aead_request *req)
1241 {
1242         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1243         int ivsize = crypto_aead_ivsize(aead);
1244
1245         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1246                    edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
1247                    edesc->qm_sg_bytes);
1248         dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
1249 }
1250
1251 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
1252                            struct skcipher_request *req)
1253 {
1254         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1255         int ivsize = crypto_skcipher_ivsize(skcipher);
1256
1257         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1258                    edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
1259                    edesc->qm_sg_bytes);
1260 }
1261
1262 static void aead_encrypt_done(void *cbk_ctx, u32 status)
1263 {
1264         struct crypto_async_request *areq = cbk_ctx;
1265         struct aead_request *req = container_of(areq, struct aead_request,
1266                                                 base);
1267         struct caam_request *req_ctx = to_caam_req(areq);
1268         struct aead_edesc *edesc = req_ctx->edesc;
1269         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1270         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1271         int ecode = 0;
1272
1273         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1274
1275         if (unlikely(status))
1276                 ecode = caam_qi2_strstatus(ctx->dev, status);
1277
1278         aead_unmap(ctx->dev, edesc, req);
1279         qi_cache_free(edesc);
1280         aead_request_complete(req, ecode);
1281 }
1282
1283 static void aead_decrypt_done(void *cbk_ctx, u32 status)
1284 {
1285         struct crypto_async_request *areq = cbk_ctx;
1286         struct aead_request *req = container_of(areq, struct aead_request,
1287                                                 base);
1288         struct caam_request *req_ctx = to_caam_req(areq);
1289         struct aead_edesc *edesc = req_ctx->edesc;
1290         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1291         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1292         int ecode = 0;
1293
1294         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1295
1296         if (unlikely(status))
1297                 ecode = caam_qi2_strstatus(ctx->dev, status);
1298
1299         aead_unmap(ctx->dev, edesc, req);
1300         qi_cache_free(edesc);
1301         aead_request_complete(req, ecode);
1302 }
1303
1304 static int aead_encrypt(struct aead_request *req)
1305 {
1306         struct aead_edesc *edesc;
1307         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1308         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1309         struct caam_request *caam_req = aead_request_ctx(req);
1310         int ret;
1311
1312         /* allocate extended descriptor */
1313         edesc = aead_edesc_alloc(req, true);
1314         if (IS_ERR(edesc))
1315                 return PTR_ERR(edesc);
1316
1317         caam_req->flc = &ctx->flc[ENCRYPT];
1318         caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1319         caam_req->cbk = aead_encrypt_done;
1320         caam_req->ctx = &req->base;
1321         caam_req->edesc = edesc;
1322         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1323         if (ret != -EINPROGRESS &&
1324             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1325                 aead_unmap(ctx->dev, edesc, req);
1326                 qi_cache_free(edesc);
1327         }
1328
1329         return ret;
1330 }
1331
1332 static int aead_decrypt(struct aead_request *req)
1333 {
1334         struct aead_edesc *edesc;
1335         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1336         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1337         struct caam_request *caam_req = aead_request_ctx(req);
1338         int ret;
1339
1340         /* allocate extended descriptor */
1341         edesc = aead_edesc_alloc(req, false);
1342         if (IS_ERR(edesc))
1343                 return PTR_ERR(edesc);
1344
1345         caam_req->flc = &ctx->flc[DECRYPT];
1346         caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1347         caam_req->cbk = aead_decrypt_done;
1348         caam_req->ctx = &req->base;
1349         caam_req->edesc = edesc;
1350         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1351         if (ret != -EINPROGRESS &&
1352             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1353                 aead_unmap(ctx->dev, edesc, req);
1354                 qi_cache_free(edesc);
1355         }
1356
1357         return ret;
1358 }
1359
1360 static int ipsec_gcm_encrypt(struct aead_request *req)
1361 {
1362         return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req);
1363 }
1364
1365 static int ipsec_gcm_decrypt(struct aead_request *req)
1366 {
1367         return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req);
1368 }
1369
1370 static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
1371 {
1372         struct crypto_async_request *areq = cbk_ctx;
1373         struct skcipher_request *req = skcipher_request_cast(areq);
1374         struct caam_request *req_ctx = to_caam_req(areq);
1375         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1376         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1377         struct skcipher_edesc *edesc = req_ctx->edesc;
1378         int ecode = 0;
1379         int ivsize = crypto_skcipher_ivsize(skcipher);
1380
1381         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1382
1383         if (unlikely(status))
1384                 ecode = caam_qi2_strstatus(ctx->dev, status);
1385
1386         print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1387                              DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1388                              edesc->src_nents > 1 ? 100 : ivsize, 1);
1389         caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1390                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1391                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1392
1393         skcipher_unmap(ctx->dev, edesc, req);
1394
1395         /*
1396          * The crypto API expects us to set the IV (req->iv) to the last
1397          * ciphertext block (CBC mode) or last counter (CTR mode).
1398          * This is used e.g. by the CTS mode.
1399          */
1400         if (!ecode)
1401                 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1402                        ivsize);
1403
1404         qi_cache_free(edesc);
1405         skcipher_request_complete(req, ecode);
1406 }
1407
1408 static void skcipher_decrypt_done(void *cbk_ctx, u32 status)
1409 {
1410         struct crypto_async_request *areq = cbk_ctx;
1411         struct skcipher_request *req = skcipher_request_cast(areq);
1412         struct caam_request *req_ctx = to_caam_req(areq);
1413         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1414         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1415         struct skcipher_edesc *edesc = req_ctx->edesc;
1416         int ecode = 0;
1417         int ivsize = crypto_skcipher_ivsize(skcipher);
1418
1419         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1420
1421         if (unlikely(status))
1422                 ecode = caam_qi2_strstatus(ctx->dev, status);
1423
1424         print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1425                              DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1426                              edesc->src_nents > 1 ? 100 : ivsize, 1);
1427         caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1428                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1429                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1430
1431         skcipher_unmap(ctx->dev, edesc, req);
1432
1433         /*
1434          * The crypto API expects us to set the IV (req->iv) to the last
1435          * ciphertext block (CBC mode) or last counter (CTR mode).
1436          * This is used e.g. by the CTS mode.
1437          */
1438         if (!ecode)
1439                 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1440                        ivsize);
1441
1442         qi_cache_free(edesc);
1443         skcipher_request_complete(req, ecode);
1444 }
1445
1446 static int skcipher_encrypt(struct skcipher_request *req)
1447 {
1448         struct skcipher_edesc *edesc;
1449         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1450         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1451         struct caam_request *caam_req = skcipher_request_ctx(req);
1452         int ret;
1453
1454         if (!req->cryptlen)
1455                 return 0;
1456
1457         /* allocate extended descriptor */
1458         edesc = skcipher_edesc_alloc(req);
1459         if (IS_ERR(edesc))
1460                 return PTR_ERR(edesc);
1461
1462         caam_req->flc = &ctx->flc[ENCRYPT];
1463         caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1464         caam_req->cbk = skcipher_encrypt_done;
1465         caam_req->ctx = &req->base;
1466         caam_req->edesc = edesc;
1467         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1468         if (ret != -EINPROGRESS &&
1469             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1470                 skcipher_unmap(ctx->dev, edesc, req);
1471                 qi_cache_free(edesc);
1472         }
1473
1474         return ret;
1475 }
1476
1477 static int skcipher_decrypt(struct skcipher_request *req)
1478 {
1479         struct skcipher_edesc *edesc;
1480         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1481         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1482         struct caam_request *caam_req = skcipher_request_ctx(req);
1483         int ret;
1484
1485         if (!req->cryptlen)
1486                 return 0;
1487         /* allocate extended descriptor */
1488         edesc = skcipher_edesc_alloc(req);
1489         if (IS_ERR(edesc))
1490                 return PTR_ERR(edesc);
1491
1492         caam_req->flc = &ctx->flc[DECRYPT];
1493         caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1494         caam_req->cbk = skcipher_decrypt_done;
1495         caam_req->ctx = &req->base;
1496         caam_req->edesc = edesc;
1497         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1498         if (ret != -EINPROGRESS &&
1499             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1500                 skcipher_unmap(ctx->dev, edesc, req);
1501                 qi_cache_free(edesc);
1502         }
1503
1504         return ret;
1505 }
1506
1507 static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam,
1508                          bool uses_dkp)
1509 {
1510         dma_addr_t dma_addr;
1511         int i;
1512
1513         /* copy descriptor header template value */
1514         ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
1515         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
1516
1517         ctx->dev = caam->dev;
1518         ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
1519
1520         dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc,
1521                                         offsetof(struct caam_ctx, flc_dma),
1522                                         ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
1523         if (dma_mapping_error(ctx->dev, dma_addr)) {
1524                 dev_err(ctx->dev, "unable to map key, shared descriptors\n");
1525                 return -ENOMEM;
1526         }
1527
1528         for (i = 0; i < NUM_OP; i++)
1529                 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
1530         ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]);
1531
1532         return 0;
1533 }
1534
1535 static int caam_cra_init_skcipher(struct crypto_skcipher *tfm)
1536 {
1537         struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1538         struct caam_skcipher_alg *caam_alg =
1539                 container_of(alg, typeof(*caam_alg), skcipher);
1540
1541         crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request));
1542         return caam_cra_init(crypto_skcipher_ctx(tfm), &caam_alg->caam, false);
1543 }
1544
1545 static int caam_cra_init_aead(struct crypto_aead *tfm)
1546 {
1547         struct aead_alg *alg = crypto_aead_alg(tfm);
1548         struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg),
1549                                                       aead);
1550
1551         crypto_aead_set_reqsize(tfm, sizeof(struct caam_request));
1552         return caam_cra_init(crypto_aead_ctx(tfm), &caam_alg->caam,
1553                              !caam_alg->caam.nodkp);
1554 }
1555
1556 static void caam_exit_common(struct caam_ctx *ctx)
1557 {
1558         dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0],
1559                                offsetof(struct caam_ctx, flc_dma), ctx->dir,
1560                                DMA_ATTR_SKIP_CPU_SYNC);
1561 }
1562
1563 static void caam_cra_exit(struct crypto_skcipher *tfm)
1564 {
1565         caam_exit_common(crypto_skcipher_ctx(tfm));
1566 }
1567
1568 static void caam_cra_exit_aead(struct crypto_aead *tfm)
1569 {
1570         caam_exit_common(crypto_aead_ctx(tfm));
1571 }
1572
1573 static struct caam_skcipher_alg driver_algs[] = {
1574         {
1575                 .skcipher = {
1576                         .base = {
1577                                 .cra_name = "cbc(aes)",
1578                                 .cra_driver_name = "cbc-aes-caam-qi2",
1579                                 .cra_blocksize = AES_BLOCK_SIZE,
1580                         },
1581                         .setkey = aes_skcipher_setkey,
1582                         .encrypt = skcipher_encrypt,
1583                         .decrypt = skcipher_decrypt,
1584                         .min_keysize = AES_MIN_KEY_SIZE,
1585                         .max_keysize = AES_MAX_KEY_SIZE,
1586                         .ivsize = AES_BLOCK_SIZE,
1587                 },
1588                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1589         },
1590         {
1591                 .skcipher = {
1592                         .base = {
1593                                 .cra_name = "cbc(des3_ede)",
1594                                 .cra_driver_name = "cbc-3des-caam-qi2",
1595                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1596                         },
1597                         .setkey = des3_skcipher_setkey,
1598                         .encrypt = skcipher_encrypt,
1599                         .decrypt = skcipher_decrypt,
1600                         .min_keysize = DES3_EDE_KEY_SIZE,
1601                         .max_keysize = DES3_EDE_KEY_SIZE,
1602                         .ivsize = DES3_EDE_BLOCK_SIZE,
1603                 },
1604                 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1605         },
1606         {
1607                 .skcipher = {
1608                         .base = {
1609                                 .cra_name = "cbc(des)",
1610                                 .cra_driver_name = "cbc-des-caam-qi2",
1611                                 .cra_blocksize = DES_BLOCK_SIZE,
1612                         },
1613                         .setkey = des_skcipher_setkey,
1614                         .encrypt = skcipher_encrypt,
1615                         .decrypt = skcipher_decrypt,
1616                         .min_keysize = DES_KEY_SIZE,
1617                         .max_keysize = DES_KEY_SIZE,
1618                         .ivsize = DES_BLOCK_SIZE,
1619                 },
1620                 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1621         },
1622         {
1623                 .skcipher = {
1624                         .base = {
1625                                 .cra_name = "ctr(aes)",
1626                                 .cra_driver_name = "ctr-aes-caam-qi2",
1627                                 .cra_blocksize = 1,
1628                         },
1629                         .setkey = ctr_skcipher_setkey,
1630                         .encrypt = skcipher_encrypt,
1631                         .decrypt = skcipher_decrypt,
1632                         .min_keysize = AES_MIN_KEY_SIZE,
1633                         .max_keysize = AES_MAX_KEY_SIZE,
1634                         .ivsize = AES_BLOCK_SIZE,
1635                         .chunksize = AES_BLOCK_SIZE,
1636                 },
1637                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1638                                         OP_ALG_AAI_CTR_MOD128,
1639         },
1640         {
1641                 .skcipher = {
1642                         .base = {
1643                                 .cra_name = "rfc3686(ctr(aes))",
1644                                 .cra_driver_name = "rfc3686-ctr-aes-caam-qi2",
1645                                 .cra_blocksize = 1,
1646                         },
1647                         .setkey = rfc3686_skcipher_setkey,
1648                         .encrypt = skcipher_encrypt,
1649                         .decrypt = skcipher_decrypt,
1650                         .min_keysize = AES_MIN_KEY_SIZE +
1651                                        CTR_RFC3686_NONCE_SIZE,
1652                         .max_keysize = AES_MAX_KEY_SIZE +
1653                                        CTR_RFC3686_NONCE_SIZE,
1654                         .ivsize = CTR_RFC3686_IV_SIZE,
1655                         .chunksize = AES_BLOCK_SIZE,
1656                 },
1657                 .caam = {
1658                         .class1_alg_type = OP_ALG_ALGSEL_AES |
1659                                            OP_ALG_AAI_CTR_MOD128,
1660                         .rfc3686 = true,
1661                 },
1662         },
1663         {
1664                 .skcipher = {
1665                         .base = {
1666                                 .cra_name = "xts(aes)",
1667                                 .cra_driver_name = "xts-aes-caam-qi2",
1668                                 .cra_blocksize = AES_BLOCK_SIZE,
1669                         },
1670                         .setkey = xts_skcipher_setkey,
1671                         .encrypt = skcipher_encrypt,
1672                         .decrypt = skcipher_decrypt,
1673                         .min_keysize = 2 * AES_MIN_KEY_SIZE,
1674                         .max_keysize = 2 * AES_MAX_KEY_SIZE,
1675                         .ivsize = AES_BLOCK_SIZE,
1676                 },
1677                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
1678         },
1679         {
1680                 .skcipher = {
1681                         .base = {
1682                                 .cra_name = "chacha20",
1683                                 .cra_driver_name = "chacha20-caam-qi2",
1684                                 .cra_blocksize = 1,
1685                         },
1686                         .setkey = chacha20_skcipher_setkey,
1687                         .encrypt = skcipher_encrypt,
1688                         .decrypt = skcipher_decrypt,
1689                         .min_keysize = CHACHA_KEY_SIZE,
1690                         .max_keysize = CHACHA_KEY_SIZE,
1691                         .ivsize = CHACHA_IV_SIZE,
1692                 },
1693                 .caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20,
1694         },
1695 };
1696
1697 static struct caam_aead_alg driver_aeads[] = {
1698         {
1699                 .aead = {
1700                         .base = {
1701                                 .cra_name = "rfc4106(gcm(aes))",
1702                                 .cra_driver_name = "rfc4106-gcm-aes-caam-qi2",
1703                                 .cra_blocksize = 1,
1704                         },
1705                         .setkey = rfc4106_setkey,
1706                         .setauthsize = rfc4106_setauthsize,
1707                         .encrypt = ipsec_gcm_encrypt,
1708                         .decrypt = ipsec_gcm_decrypt,
1709                         .ivsize = 8,
1710                         .maxauthsize = AES_BLOCK_SIZE,
1711                 },
1712                 .caam = {
1713                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1714                         .nodkp = true,
1715                 },
1716         },
1717         {
1718                 .aead = {
1719                         .base = {
1720                                 .cra_name = "rfc4543(gcm(aes))",
1721                                 .cra_driver_name = "rfc4543-gcm-aes-caam-qi2",
1722                                 .cra_blocksize = 1,
1723                         },
1724                         .setkey = rfc4543_setkey,
1725                         .setauthsize = rfc4543_setauthsize,
1726                         .encrypt = ipsec_gcm_encrypt,
1727                         .decrypt = ipsec_gcm_decrypt,
1728                         .ivsize = 8,
1729                         .maxauthsize = AES_BLOCK_SIZE,
1730                 },
1731                 .caam = {
1732                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1733                         .nodkp = true,
1734                 },
1735         },
1736         /* Galois Counter Mode */
1737         {
1738                 .aead = {
1739                         .base = {
1740                                 .cra_name = "gcm(aes)",
1741                                 .cra_driver_name = "gcm-aes-caam-qi2",
1742                                 .cra_blocksize = 1,
1743                         },
1744                         .setkey = gcm_setkey,
1745                         .setauthsize = gcm_setauthsize,
1746                         .encrypt = aead_encrypt,
1747                         .decrypt = aead_decrypt,
1748                         .ivsize = 12,
1749                         .maxauthsize = AES_BLOCK_SIZE,
1750                 },
1751                 .caam = {
1752                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1753                         .nodkp = true,
1754                 }
1755         },
1756         /* single-pass ipsec_esp descriptor */
1757         {
1758                 .aead = {
1759                         .base = {
1760                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
1761                                 .cra_driver_name = "authenc-hmac-md5-"
1762                                                    "cbc-aes-caam-qi2",
1763                                 .cra_blocksize = AES_BLOCK_SIZE,
1764                         },
1765                         .setkey = aead_setkey,
1766                         .setauthsize = aead_setauthsize,
1767                         .encrypt = aead_encrypt,
1768                         .decrypt = aead_decrypt,
1769                         .ivsize = AES_BLOCK_SIZE,
1770                         .maxauthsize = MD5_DIGEST_SIZE,
1771                 },
1772                 .caam = {
1773                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1774                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1775                                            OP_ALG_AAI_HMAC_PRECOMP,
1776                 }
1777         },
1778         {
1779                 .aead = {
1780                         .base = {
1781                                 .cra_name = "echainiv(authenc(hmac(md5),"
1782                                             "cbc(aes)))",
1783                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
1784                                                    "cbc-aes-caam-qi2",
1785                                 .cra_blocksize = AES_BLOCK_SIZE,
1786                         },
1787                         .setkey = aead_setkey,
1788                         .setauthsize = aead_setauthsize,
1789                         .encrypt = aead_encrypt,
1790                         .decrypt = aead_decrypt,
1791                         .ivsize = AES_BLOCK_SIZE,
1792                         .maxauthsize = MD5_DIGEST_SIZE,
1793                 },
1794                 .caam = {
1795                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1796                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1797                                            OP_ALG_AAI_HMAC_PRECOMP,
1798                         .geniv = true,
1799                 }
1800         },
1801         {
1802                 .aead = {
1803                         .base = {
1804                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1805                                 .cra_driver_name = "authenc-hmac-sha1-"
1806                                                    "cbc-aes-caam-qi2",
1807                                 .cra_blocksize = AES_BLOCK_SIZE,
1808                         },
1809                         .setkey = aead_setkey,
1810                         .setauthsize = aead_setauthsize,
1811                         .encrypt = aead_encrypt,
1812                         .decrypt = aead_decrypt,
1813                         .ivsize = AES_BLOCK_SIZE,
1814                         .maxauthsize = SHA1_DIGEST_SIZE,
1815                 },
1816                 .caam = {
1817                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1818                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1819                                            OP_ALG_AAI_HMAC_PRECOMP,
1820                 }
1821         },
1822         {
1823                 .aead = {
1824                         .base = {
1825                                 .cra_name = "echainiv(authenc(hmac(sha1),"
1826                                             "cbc(aes)))",
1827                                 .cra_driver_name = "echainiv-authenc-"
1828                                                    "hmac-sha1-cbc-aes-caam-qi2",
1829                                 .cra_blocksize = AES_BLOCK_SIZE,
1830                         },
1831                         .setkey = aead_setkey,
1832                         .setauthsize = aead_setauthsize,
1833                         .encrypt = aead_encrypt,
1834                         .decrypt = aead_decrypt,
1835                         .ivsize = AES_BLOCK_SIZE,
1836                         .maxauthsize = SHA1_DIGEST_SIZE,
1837                 },
1838                 .caam = {
1839                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1840                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1841                                            OP_ALG_AAI_HMAC_PRECOMP,
1842                         .geniv = true,
1843                 },
1844         },
1845         {
1846                 .aead = {
1847                         .base = {
1848                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
1849                                 .cra_driver_name = "authenc-hmac-sha224-"
1850                                                    "cbc-aes-caam-qi2",
1851                                 .cra_blocksize = AES_BLOCK_SIZE,
1852                         },
1853                         .setkey = aead_setkey,
1854                         .setauthsize = aead_setauthsize,
1855                         .encrypt = aead_encrypt,
1856                         .decrypt = aead_decrypt,
1857                         .ivsize = AES_BLOCK_SIZE,
1858                         .maxauthsize = SHA224_DIGEST_SIZE,
1859                 },
1860                 .caam = {
1861                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1862                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1863                                            OP_ALG_AAI_HMAC_PRECOMP,
1864                 }
1865         },
1866         {
1867                 .aead = {
1868                         .base = {
1869                                 .cra_name = "echainiv(authenc(hmac(sha224),"
1870                                             "cbc(aes)))",
1871                                 .cra_driver_name = "echainiv-authenc-"
1872                                                    "hmac-sha224-cbc-aes-caam-qi2",
1873                                 .cra_blocksize = AES_BLOCK_SIZE,
1874                         },
1875                         .setkey = aead_setkey,
1876                         .setauthsize = aead_setauthsize,
1877                         .encrypt = aead_encrypt,
1878                         .decrypt = aead_decrypt,
1879                         .ivsize = AES_BLOCK_SIZE,
1880                         .maxauthsize = SHA224_DIGEST_SIZE,
1881                 },
1882                 .caam = {
1883                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1884                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1885                                            OP_ALG_AAI_HMAC_PRECOMP,
1886                         .geniv = true,
1887                 }
1888         },
1889         {
1890                 .aead = {
1891                         .base = {
1892                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1893                                 .cra_driver_name = "authenc-hmac-sha256-"
1894                                                    "cbc-aes-caam-qi2",
1895                                 .cra_blocksize = AES_BLOCK_SIZE,
1896                         },
1897                         .setkey = aead_setkey,
1898                         .setauthsize = aead_setauthsize,
1899                         .encrypt = aead_encrypt,
1900                         .decrypt = aead_decrypt,
1901                         .ivsize = AES_BLOCK_SIZE,
1902                         .maxauthsize = SHA256_DIGEST_SIZE,
1903                 },
1904                 .caam = {
1905                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1906                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1907                                            OP_ALG_AAI_HMAC_PRECOMP,
1908                 }
1909         },
1910         {
1911                 .aead = {
1912                         .base = {
1913                                 .cra_name = "echainiv(authenc(hmac(sha256),"
1914                                             "cbc(aes)))",
1915                                 .cra_driver_name = "echainiv-authenc-"
1916                                                    "hmac-sha256-cbc-aes-"
1917                                                    "caam-qi2",
1918                                 .cra_blocksize = AES_BLOCK_SIZE,
1919                         },
1920                         .setkey = aead_setkey,
1921                         .setauthsize = aead_setauthsize,
1922                         .encrypt = aead_encrypt,
1923                         .decrypt = aead_decrypt,
1924                         .ivsize = AES_BLOCK_SIZE,
1925                         .maxauthsize = SHA256_DIGEST_SIZE,
1926                 },
1927                 .caam = {
1928                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1929                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1930                                            OP_ALG_AAI_HMAC_PRECOMP,
1931                         .geniv = true,
1932                 }
1933         },
1934         {
1935                 .aead = {
1936                         .base = {
1937                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
1938                                 .cra_driver_name = "authenc-hmac-sha384-"
1939                                                    "cbc-aes-caam-qi2",
1940                                 .cra_blocksize = AES_BLOCK_SIZE,
1941                         },
1942                         .setkey = aead_setkey,
1943                         .setauthsize = aead_setauthsize,
1944                         .encrypt = aead_encrypt,
1945                         .decrypt = aead_decrypt,
1946                         .ivsize = AES_BLOCK_SIZE,
1947                         .maxauthsize = SHA384_DIGEST_SIZE,
1948                 },
1949                 .caam = {
1950                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1951                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1952                                            OP_ALG_AAI_HMAC_PRECOMP,
1953                 }
1954         },
1955         {
1956                 .aead = {
1957                         .base = {
1958                                 .cra_name = "echainiv(authenc(hmac(sha384),"
1959                                             "cbc(aes)))",
1960                                 .cra_driver_name = "echainiv-authenc-"
1961                                                    "hmac-sha384-cbc-aes-"
1962                                                    "caam-qi2",
1963                                 .cra_blocksize = AES_BLOCK_SIZE,
1964                         },
1965                         .setkey = aead_setkey,
1966                         .setauthsize = aead_setauthsize,
1967                         .encrypt = aead_encrypt,
1968                         .decrypt = aead_decrypt,
1969                         .ivsize = AES_BLOCK_SIZE,
1970                         .maxauthsize = SHA384_DIGEST_SIZE,
1971                 },
1972                 .caam = {
1973                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1974                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1975                                            OP_ALG_AAI_HMAC_PRECOMP,
1976                         .geniv = true,
1977                 }
1978         },
1979         {
1980                 .aead = {
1981                         .base = {
1982                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1983                                 .cra_driver_name = "authenc-hmac-sha512-"
1984                                                    "cbc-aes-caam-qi2",
1985                                 .cra_blocksize = AES_BLOCK_SIZE,
1986                         },
1987                         .setkey = aead_setkey,
1988                         .setauthsize = aead_setauthsize,
1989                         .encrypt = aead_encrypt,
1990                         .decrypt = aead_decrypt,
1991                         .ivsize = AES_BLOCK_SIZE,
1992                         .maxauthsize = SHA512_DIGEST_SIZE,
1993                 },
1994                 .caam = {
1995                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1996                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1997                                            OP_ALG_AAI_HMAC_PRECOMP,
1998                 }
1999         },
2000         {
2001                 .aead = {
2002                         .base = {
2003                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2004                                             "cbc(aes)))",
2005                                 .cra_driver_name = "echainiv-authenc-"
2006                                                    "hmac-sha512-cbc-aes-"
2007                                                    "caam-qi2",
2008                                 .cra_blocksize = AES_BLOCK_SIZE,
2009                         },
2010                         .setkey = aead_setkey,
2011                         .setauthsize = aead_setauthsize,
2012                         .encrypt = aead_encrypt,
2013                         .decrypt = aead_decrypt,
2014                         .ivsize = AES_BLOCK_SIZE,
2015                         .maxauthsize = SHA512_DIGEST_SIZE,
2016                 },
2017                 .caam = {
2018                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2019                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2020                                            OP_ALG_AAI_HMAC_PRECOMP,
2021                         .geniv = true,
2022                 }
2023         },
2024         {
2025                 .aead = {
2026                         .base = {
2027                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2028                                 .cra_driver_name = "authenc-hmac-md5-"
2029                                                    "cbc-des3_ede-caam-qi2",
2030                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2031                         },
2032                         .setkey = des3_aead_setkey,
2033                         .setauthsize = aead_setauthsize,
2034                         .encrypt = aead_encrypt,
2035                         .decrypt = aead_decrypt,
2036                         .ivsize = DES3_EDE_BLOCK_SIZE,
2037                         .maxauthsize = MD5_DIGEST_SIZE,
2038                 },
2039                 .caam = {
2040                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2041                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2042                                            OP_ALG_AAI_HMAC_PRECOMP,
2043                 }
2044         },
2045         {
2046                 .aead = {
2047                         .base = {
2048                                 .cra_name = "echainiv(authenc(hmac(md5),"
2049                                             "cbc(des3_ede)))",
2050                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2051                                                    "cbc-des3_ede-caam-qi2",
2052                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2053                         },
2054                         .setkey = des3_aead_setkey,
2055                         .setauthsize = aead_setauthsize,
2056                         .encrypt = aead_encrypt,
2057                         .decrypt = aead_decrypt,
2058                         .ivsize = DES3_EDE_BLOCK_SIZE,
2059                         .maxauthsize = MD5_DIGEST_SIZE,
2060                 },
2061                 .caam = {
2062                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2063                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2064                                            OP_ALG_AAI_HMAC_PRECOMP,
2065                         .geniv = true,
2066                 }
2067         },
2068         {
2069                 .aead = {
2070                         .base = {
2071                                 .cra_name = "authenc(hmac(sha1),"
2072                                             "cbc(des3_ede))",
2073                                 .cra_driver_name = "authenc-hmac-sha1-"
2074                                                    "cbc-des3_ede-caam-qi2",
2075                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2076                         },
2077                         .setkey = des3_aead_setkey,
2078                         .setauthsize = aead_setauthsize,
2079                         .encrypt = aead_encrypt,
2080                         .decrypt = aead_decrypt,
2081                         .ivsize = DES3_EDE_BLOCK_SIZE,
2082                         .maxauthsize = SHA1_DIGEST_SIZE,
2083                 },
2084                 .caam = {
2085                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2086                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2087                                            OP_ALG_AAI_HMAC_PRECOMP,
2088                 },
2089         },
2090         {
2091                 .aead = {
2092                         .base = {
2093                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2094                                             "cbc(des3_ede)))",
2095                                 .cra_driver_name = "echainiv-authenc-"
2096                                                    "hmac-sha1-"
2097                                                    "cbc-des3_ede-caam-qi2",
2098                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2099                         },
2100                         .setkey = des3_aead_setkey,
2101                         .setauthsize = aead_setauthsize,
2102                         .encrypt = aead_encrypt,
2103                         .decrypt = aead_decrypt,
2104                         .ivsize = DES3_EDE_BLOCK_SIZE,
2105                         .maxauthsize = SHA1_DIGEST_SIZE,
2106                 },
2107                 .caam = {
2108                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2109                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2110                                            OP_ALG_AAI_HMAC_PRECOMP,
2111                         .geniv = true,
2112                 }
2113         },
2114         {
2115                 .aead = {
2116                         .base = {
2117                                 .cra_name = "authenc(hmac(sha224),"
2118                                             "cbc(des3_ede))",
2119                                 .cra_driver_name = "authenc-hmac-sha224-"
2120                                                    "cbc-des3_ede-caam-qi2",
2121                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2122                         },
2123                         .setkey = des3_aead_setkey,
2124                         .setauthsize = aead_setauthsize,
2125                         .encrypt = aead_encrypt,
2126                         .decrypt = aead_decrypt,
2127                         .ivsize = DES3_EDE_BLOCK_SIZE,
2128                         .maxauthsize = SHA224_DIGEST_SIZE,
2129                 },
2130                 .caam = {
2131                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2132                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2133                                            OP_ALG_AAI_HMAC_PRECOMP,
2134                 },
2135         },
2136         {
2137                 .aead = {
2138                         .base = {
2139                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2140                                             "cbc(des3_ede)))",
2141                                 .cra_driver_name = "echainiv-authenc-"
2142                                                    "hmac-sha224-"
2143                                                    "cbc-des3_ede-caam-qi2",
2144                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2145                         },
2146                         .setkey = des3_aead_setkey,
2147                         .setauthsize = aead_setauthsize,
2148                         .encrypt = aead_encrypt,
2149                         .decrypt = aead_decrypt,
2150                         .ivsize = DES3_EDE_BLOCK_SIZE,
2151                         .maxauthsize = SHA224_DIGEST_SIZE,
2152                 },
2153                 .caam = {
2154                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2155                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2156                                            OP_ALG_AAI_HMAC_PRECOMP,
2157                         .geniv = true,
2158                 }
2159         },
2160         {
2161                 .aead = {
2162                         .base = {
2163                                 .cra_name = "authenc(hmac(sha256),"
2164                                             "cbc(des3_ede))",
2165                                 .cra_driver_name = "authenc-hmac-sha256-"
2166                                                    "cbc-des3_ede-caam-qi2",
2167                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2168                         },
2169                         .setkey = des3_aead_setkey,
2170                         .setauthsize = aead_setauthsize,
2171                         .encrypt = aead_encrypt,
2172                         .decrypt = aead_decrypt,
2173                         .ivsize = DES3_EDE_BLOCK_SIZE,
2174                         .maxauthsize = SHA256_DIGEST_SIZE,
2175                 },
2176                 .caam = {
2177                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2178                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2179                                            OP_ALG_AAI_HMAC_PRECOMP,
2180                 },
2181         },
2182         {
2183                 .aead = {
2184                         .base = {
2185                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2186                                             "cbc(des3_ede)))",
2187                                 .cra_driver_name = "echainiv-authenc-"
2188                                                    "hmac-sha256-"
2189                                                    "cbc-des3_ede-caam-qi2",
2190                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2191                         },
2192                         .setkey = des3_aead_setkey,
2193                         .setauthsize = aead_setauthsize,
2194                         .encrypt = aead_encrypt,
2195                         .decrypt = aead_decrypt,
2196                         .ivsize = DES3_EDE_BLOCK_SIZE,
2197                         .maxauthsize = SHA256_DIGEST_SIZE,
2198                 },
2199                 .caam = {
2200                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2201                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2202                                            OP_ALG_AAI_HMAC_PRECOMP,
2203                         .geniv = true,
2204                 }
2205         },
2206         {
2207                 .aead = {
2208                         .base = {
2209                                 .cra_name = "authenc(hmac(sha384),"
2210                                             "cbc(des3_ede))",
2211                                 .cra_driver_name = "authenc-hmac-sha384-"
2212                                                    "cbc-des3_ede-caam-qi2",
2213                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2214                         },
2215                         .setkey = des3_aead_setkey,
2216                         .setauthsize = aead_setauthsize,
2217                         .encrypt = aead_encrypt,
2218                         .decrypt = aead_decrypt,
2219                         .ivsize = DES3_EDE_BLOCK_SIZE,
2220                         .maxauthsize = SHA384_DIGEST_SIZE,
2221                 },
2222                 .caam = {
2223                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2224                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2225                                            OP_ALG_AAI_HMAC_PRECOMP,
2226                 },
2227         },
2228         {
2229                 .aead = {
2230                         .base = {
2231                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2232                                             "cbc(des3_ede)))",
2233                                 .cra_driver_name = "echainiv-authenc-"
2234                                                    "hmac-sha384-"
2235                                                    "cbc-des3_ede-caam-qi2",
2236                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2237                         },
2238                         .setkey = des3_aead_setkey,
2239                         .setauthsize = aead_setauthsize,
2240                         .encrypt = aead_encrypt,
2241                         .decrypt = aead_decrypt,
2242                         .ivsize = DES3_EDE_BLOCK_SIZE,
2243                         .maxauthsize = SHA384_DIGEST_SIZE,
2244                 },
2245                 .caam = {
2246                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2247                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2248                                            OP_ALG_AAI_HMAC_PRECOMP,
2249                         .geniv = true,
2250                 }
2251         },
2252         {
2253                 .aead = {
2254                         .base = {
2255                                 .cra_name = "authenc(hmac(sha512),"
2256                                             "cbc(des3_ede))",
2257                                 .cra_driver_name = "authenc-hmac-sha512-"
2258                                                    "cbc-des3_ede-caam-qi2",
2259                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2260                         },
2261                         .setkey = des3_aead_setkey,
2262                         .setauthsize = aead_setauthsize,
2263                         .encrypt = aead_encrypt,
2264                         .decrypt = aead_decrypt,
2265                         .ivsize = DES3_EDE_BLOCK_SIZE,
2266                         .maxauthsize = SHA512_DIGEST_SIZE,
2267                 },
2268                 .caam = {
2269                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2270                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2271                                            OP_ALG_AAI_HMAC_PRECOMP,
2272                 },
2273         },
2274         {
2275                 .aead = {
2276                         .base = {
2277                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2278                                             "cbc(des3_ede)))",
2279                                 .cra_driver_name = "echainiv-authenc-"
2280                                                    "hmac-sha512-"
2281                                                    "cbc-des3_ede-caam-qi2",
2282                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2283                         },
2284                         .setkey = des3_aead_setkey,
2285                         .setauthsize = aead_setauthsize,
2286                         .encrypt = aead_encrypt,
2287                         .decrypt = aead_decrypt,
2288                         .ivsize = DES3_EDE_BLOCK_SIZE,
2289                         .maxauthsize = SHA512_DIGEST_SIZE,
2290                 },
2291                 .caam = {
2292                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2293                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2294                                            OP_ALG_AAI_HMAC_PRECOMP,
2295                         .geniv = true,
2296                 }
2297         },
2298         {
2299                 .aead = {
2300                         .base = {
2301                                 .cra_name = "authenc(hmac(md5),cbc(des))",
2302                                 .cra_driver_name = "authenc-hmac-md5-"
2303                                                    "cbc-des-caam-qi2",
2304                                 .cra_blocksize = DES_BLOCK_SIZE,
2305                         },
2306                         .setkey = aead_setkey,
2307                         .setauthsize = aead_setauthsize,
2308                         .encrypt = aead_encrypt,
2309                         .decrypt = aead_decrypt,
2310                         .ivsize = DES_BLOCK_SIZE,
2311                         .maxauthsize = MD5_DIGEST_SIZE,
2312                 },
2313                 .caam = {
2314                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2315                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2316                                            OP_ALG_AAI_HMAC_PRECOMP,
2317                 },
2318         },
2319         {
2320                 .aead = {
2321                         .base = {
2322                                 .cra_name = "echainiv(authenc(hmac(md5),"
2323                                             "cbc(des)))",
2324                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2325                                                    "cbc-des-caam-qi2",
2326                                 .cra_blocksize = DES_BLOCK_SIZE,
2327                         },
2328                         .setkey = aead_setkey,
2329                         .setauthsize = aead_setauthsize,
2330                         .encrypt = aead_encrypt,
2331                         .decrypt = aead_decrypt,
2332                         .ivsize = DES_BLOCK_SIZE,
2333                         .maxauthsize = MD5_DIGEST_SIZE,
2334                 },
2335                 .caam = {
2336                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2337                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2338                                            OP_ALG_AAI_HMAC_PRECOMP,
2339                         .geniv = true,
2340                 }
2341         },
2342         {
2343                 .aead = {
2344                         .base = {
2345                                 .cra_name = "authenc(hmac(sha1),cbc(des))",
2346                                 .cra_driver_name = "authenc-hmac-sha1-"
2347                                                    "cbc-des-caam-qi2",
2348                                 .cra_blocksize = DES_BLOCK_SIZE,
2349                         },
2350                         .setkey = aead_setkey,
2351                         .setauthsize = aead_setauthsize,
2352                         .encrypt = aead_encrypt,
2353                         .decrypt = aead_decrypt,
2354                         .ivsize = DES_BLOCK_SIZE,
2355                         .maxauthsize = SHA1_DIGEST_SIZE,
2356                 },
2357                 .caam = {
2358                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2359                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2360                                            OP_ALG_AAI_HMAC_PRECOMP,
2361                 },
2362         },
2363         {
2364                 .aead = {
2365                         .base = {
2366                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2367                                             "cbc(des)))",
2368                                 .cra_driver_name = "echainiv-authenc-"
2369                                                    "hmac-sha1-cbc-des-caam-qi2",
2370                                 .cra_blocksize = DES_BLOCK_SIZE,
2371                         },
2372                         .setkey = aead_setkey,
2373                         .setauthsize = aead_setauthsize,
2374                         .encrypt = aead_encrypt,
2375                         .decrypt = aead_decrypt,
2376                         .ivsize = DES_BLOCK_SIZE,
2377                         .maxauthsize = SHA1_DIGEST_SIZE,
2378                 },
2379                 .caam = {
2380                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2381                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2382                                            OP_ALG_AAI_HMAC_PRECOMP,
2383                         .geniv = true,
2384                 }
2385         },
2386         {
2387                 .aead = {
2388                         .base = {
2389                                 .cra_name = "authenc(hmac(sha224),cbc(des))",
2390                                 .cra_driver_name = "authenc-hmac-sha224-"
2391                                                    "cbc-des-caam-qi2",
2392                                 .cra_blocksize = DES_BLOCK_SIZE,
2393                         },
2394                         .setkey = aead_setkey,
2395                         .setauthsize = aead_setauthsize,
2396                         .encrypt = aead_encrypt,
2397                         .decrypt = aead_decrypt,
2398                         .ivsize = DES_BLOCK_SIZE,
2399                         .maxauthsize = SHA224_DIGEST_SIZE,
2400                 },
2401                 .caam = {
2402                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2403                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2404                                            OP_ALG_AAI_HMAC_PRECOMP,
2405                 },
2406         },
2407         {
2408                 .aead = {
2409                         .base = {
2410                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2411                                             "cbc(des)))",
2412                                 .cra_driver_name = "echainiv-authenc-"
2413                                                    "hmac-sha224-cbc-des-"
2414                                                    "caam-qi2",
2415                                 .cra_blocksize = DES_BLOCK_SIZE,
2416                         },
2417                         .setkey = aead_setkey,
2418                         .setauthsize = aead_setauthsize,
2419                         .encrypt = aead_encrypt,
2420                         .decrypt = aead_decrypt,
2421                         .ivsize = DES_BLOCK_SIZE,
2422                         .maxauthsize = SHA224_DIGEST_SIZE,
2423                 },
2424                 .caam = {
2425                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2426                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2427                                            OP_ALG_AAI_HMAC_PRECOMP,
2428                         .geniv = true,
2429                 }
2430         },
2431         {
2432                 .aead = {
2433                         .base = {
2434                                 .cra_name = "authenc(hmac(sha256),cbc(des))",
2435                                 .cra_driver_name = "authenc-hmac-sha256-"
2436                                                    "cbc-des-caam-qi2",
2437                                 .cra_blocksize = DES_BLOCK_SIZE,
2438                         },
2439                         .setkey = aead_setkey,
2440                         .setauthsize = aead_setauthsize,
2441                         .encrypt = aead_encrypt,
2442                         .decrypt = aead_decrypt,
2443                         .ivsize = DES_BLOCK_SIZE,
2444                         .maxauthsize = SHA256_DIGEST_SIZE,
2445                 },
2446                 .caam = {
2447                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2448                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2449                                            OP_ALG_AAI_HMAC_PRECOMP,
2450                 },
2451         },
2452         {
2453                 .aead = {
2454                         .base = {
2455                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2456                                             "cbc(des)))",
2457                                 .cra_driver_name = "echainiv-authenc-"
2458                                                    "hmac-sha256-cbc-des-"
2459                                                    "caam-qi2",
2460                                 .cra_blocksize = DES_BLOCK_SIZE,
2461                         },
2462                         .setkey = aead_setkey,
2463                         .setauthsize = aead_setauthsize,
2464                         .encrypt = aead_encrypt,
2465                         .decrypt = aead_decrypt,
2466                         .ivsize = DES_BLOCK_SIZE,
2467                         .maxauthsize = SHA256_DIGEST_SIZE,
2468                 },
2469                 .caam = {
2470                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2471                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2472                                            OP_ALG_AAI_HMAC_PRECOMP,
2473                         .geniv = true,
2474                 },
2475         },
2476         {
2477                 .aead = {
2478                         .base = {
2479                                 .cra_name = "authenc(hmac(sha384),cbc(des))",
2480                                 .cra_driver_name = "authenc-hmac-sha384-"
2481                                                    "cbc-des-caam-qi2",
2482                                 .cra_blocksize = DES_BLOCK_SIZE,
2483                         },
2484                         .setkey = aead_setkey,
2485                         .setauthsize = aead_setauthsize,
2486                         .encrypt = aead_encrypt,
2487                         .decrypt = aead_decrypt,
2488                         .ivsize = DES_BLOCK_SIZE,
2489                         .maxauthsize = SHA384_DIGEST_SIZE,
2490                 },
2491                 .caam = {
2492                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2493                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2494                                            OP_ALG_AAI_HMAC_PRECOMP,
2495                 },
2496         },
2497         {
2498                 .aead = {
2499                         .base = {
2500                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2501                                             "cbc(des)))",
2502                                 .cra_driver_name = "echainiv-authenc-"
2503                                                    "hmac-sha384-cbc-des-"
2504                                                    "caam-qi2",
2505                                 .cra_blocksize = DES_BLOCK_SIZE,
2506                         },
2507                         .setkey = aead_setkey,
2508                         .setauthsize = aead_setauthsize,
2509                         .encrypt = aead_encrypt,
2510                         .decrypt = aead_decrypt,
2511                         .ivsize = DES_BLOCK_SIZE,
2512                         .maxauthsize = SHA384_DIGEST_SIZE,
2513                 },
2514                 .caam = {
2515                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2516                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2517                                            OP_ALG_AAI_HMAC_PRECOMP,
2518                         .geniv = true,
2519                 }
2520         },
2521         {
2522                 .aead = {
2523                         .base = {
2524                                 .cra_name = "authenc(hmac(sha512),cbc(des))",
2525                                 .cra_driver_name = "authenc-hmac-sha512-"
2526                                                    "cbc-des-caam-qi2",
2527                                 .cra_blocksize = DES_BLOCK_SIZE,
2528                         },
2529                         .setkey = aead_setkey,
2530                         .setauthsize = aead_setauthsize,
2531                         .encrypt = aead_encrypt,
2532                         .decrypt = aead_decrypt,
2533                         .ivsize = DES_BLOCK_SIZE,
2534                         .maxauthsize = SHA512_DIGEST_SIZE,
2535                 },
2536                 .caam = {
2537                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2538                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2539                                            OP_ALG_AAI_HMAC_PRECOMP,
2540                 }
2541         },
2542         {
2543                 .aead = {
2544                         .base = {
2545                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2546                                             "cbc(des)))",
2547                                 .cra_driver_name = "echainiv-authenc-"
2548                                                    "hmac-sha512-cbc-des-"
2549                                                    "caam-qi2",
2550                                 .cra_blocksize = DES_BLOCK_SIZE,
2551                         },
2552                         .setkey = aead_setkey,
2553                         .setauthsize = aead_setauthsize,
2554                         .encrypt = aead_encrypt,
2555                         .decrypt = aead_decrypt,
2556                         .ivsize = DES_BLOCK_SIZE,
2557                         .maxauthsize = SHA512_DIGEST_SIZE,
2558                 },
2559                 .caam = {
2560                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2561                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2562                                            OP_ALG_AAI_HMAC_PRECOMP,
2563                         .geniv = true,
2564                 }
2565         },
2566         {
2567                 .aead = {
2568                         .base = {
2569                                 .cra_name = "authenc(hmac(md5),"
2570                                             "rfc3686(ctr(aes)))",
2571                                 .cra_driver_name = "authenc-hmac-md5-"
2572                                                    "rfc3686-ctr-aes-caam-qi2",
2573                                 .cra_blocksize = 1,
2574                         },
2575                         .setkey = aead_setkey,
2576                         .setauthsize = aead_setauthsize,
2577                         .encrypt = aead_encrypt,
2578                         .decrypt = aead_decrypt,
2579                         .ivsize = CTR_RFC3686_IV_SIZE,
2580                         .maxauthsize = MD5_DIGEST_SIZE,
2581                 },
2582                 .caam = {
2583                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2584                                            OP_ALG_AAI_CTR_MOD128,
2585                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2586                                            OP_ALG_AAI_HMAC_PRECOMP,
2587                         .rfc3686 = true,
2588                 },
2589         },
2590         {
2591                 .aead = {
2592                         .base = {
2593                                 .cra_name = "seqiv(authenc("
2594                                             "hmac(md5),rfc3686(ctr(aes))))",
2595                                 .cra_driver_name = "seqiv-authenc-hmac-md5-"
2596                                                    "rfc3686-ctr-aes-caam-qi2",
2597                                 .cra_blocksize = 1,
2598                         },
2599                         .setkey = aead_setkey,
2600                         .setauthsize = aead_setauthsize,
2601                         .encrypt = aead_encrypt,
2602                         .decrypt = aead_decrypt,
2603                         .ivsize = CTR_RFC3686_IV_SIZE,
2604                         .maxauthsize = MD5_DIGEST_SIZE,
2605                 },
2606                 .caam = {
2607                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2608                                            OP_ALG_AAI_CTR_MOD128,
2609                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2610                                            OP_ALG_AAI_HMAC_PRECOMP,
2611                         .rfc3686 = true,
2612                         .geniv = true,
2613                 },
2614         },
2615         {
2616                 .aead = {
2617                         .base = {
2618                                 .cra_name = "authenc(hmac(sha1),"
2619                                             "rfc3686(ctr(aes)))",
2620                                 .cra_driver_name = "authenc-hmac-sha1-"
2621                                                    "rfc3686-ctr-aes-caam-qi2",
2622                                 .cra_blocksize = 1,
2623                         },
2624                         .setkey = aead_setkey,
2625                         .setauthsize = aead_setauthsize,
2626                         .encrypt = aead_encrypt,
2627                         .decrypt = aead_decrypt,
2628                         .ivsize = CTR_RFC3686_IV_SIZE,
2629                         .maxauthsize = SHA1_DIGEST_SIZE,
2630                 },
2631                 .caam = {
2632                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2633                                            OP_ALG_AAI_CTR_MOD128,
2634                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2635                                            OP_ALG_AAI_HMAC_PRECOMP,
2636                         .rfc3686 = true,
2637                 },
2638         },
2639         {
2640                 .aead = {
2641                         .base = {
2642                                 .cra_name = "seqiv(authenc("
2643                                             "hmac(sha1),rfc3686(ctr(aes))))",
2644                                 .cra_driver_name = "seqiv-authenc-hmac-sha1-"
2645                                                    "rfc3686-ctr-aes-caam-qi2",
2646                                 .cra_blocksize = 1,
2647                         },
2648                         .setkey = aead_setkey,
2649                         .setauthsize = aead_setauthsize,
2650                         .encrypt = aead_encrypt,
2651                         .decrypt = aead_decrypt,
2652                         .ivsize = CTR_RFC3686_IV_SIZE,
2653                         .maxauthsize = SHA1_DIGEST_SIZE,
2654                 },
2655                 .caam = {
2656                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2657                                            OP_ALG_AAI_CTR_MOD128,
2658                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2659                                            OP_ALG_AAI_HMAC_PRECOMP,
2660                         .rfc3686 = true,
2661                         .geniv = true,
2662                 },
2663         },
2664         {
2665                 .aead = {
2666                         .base = {
2667                                 .cra_name = "authenc(hmac(sha224),"
2668                                             "rfc3686(ctr(aes)))",
2669                                 .cra_driver_name = "authenc-hmac-sha224-"
2670                                                    "rfc3686-ctr-aes-caam-qi2",
2671                                 .cra_blocksize = 1,
2672                         },
2673                         .setkey = aead_setkey,
2674                         .setauthsize = aead_setauthsize,
2675                         .encrypt = aead_encrypt,
2676                         .decrypt = aead_decrypt,
2677                         .ivsize = CTR_RFC3686_IV_SIZE,
2678                         .maxauthsize = SHA224_DIGEST_SIZE,
2679                 },
2680                 .caam = {
2681                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2682                                            OP_ALG_AAI_CTR_MOD128,
2683                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2684                                            OP_ALG_AAI_HMAC_PRECOMP,
2685                         .rfc3686 = true,
2686                 },
2687         },
2688         {
2689                 .aead = {
2690                         .base = {
2691                                 .cra_name = "seqiv(authenc("
2692                                             "hmac(sha224),rfc3686(ctr(aes))))",
2693                                 .cra_driver_name = "seqiv-authenc-hmac-sha224-"
2694                                                    "rfc3686-ctr-aes-caam-qi2",
2695                                 .cra_blocksize = 1,
2696                         },
2697                         .setkey = aead_setkey,
2698                         .setauthsize = aead_setauthsize,
2699                         .encrypt = aead_encrypt,
2700                         .decrypt = aead_decrypt,
2701                         .ivsize = CTR_RFC3686_IV_SIZE,
2702                         .maxauthsize = SHA224_DIGEST_SIZE,
2703                 },
2704                 .caam = {
2705                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2706                                            OP_ALG_AAI_CTR_MOD128,
2707                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2708                                            OP_ALG_AAI_HMAC_PRECOMP,
2709                         .rfc3686 = true,
2710                         .geniv = true,
2711                 },
2712         },
2713         {
2714                 .aead = {
2715                         .base = {
2716                                 .cra_name = "authenc(hmac(sha256),"
2717                                             "rfc3686(ctr(aes)))",
2718                                 .cra_driver_name = "authenc-hmac-sha256-"
2719                                                    "rfc3686-ctr-aes-caam-qi2",
2720                                 .cra_blocksize = 1,
2721                         },
2722                         .setkey = aead_setkey,
2723                         .setauthsize = aead_setauthsize,
2724                         .encrypt = aead_encrypt,
2725                         .decrypt = aead_decrypt,
2726                         .ivsize = CTR_RFC3686_IV_SIZE,
2727                         .maxauthsize = SHA256_DIGEST_SIZE,
2728                 },
2729                 .caam = {
2730                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2731                                            OP_ALG_AAI_CTR_MOD128,
2732                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2733                                            OP_ALG_AAI_HMAC_PRECOMP,
2734                         .rfc3686 = true,
2735                 },
2736         },
2737         {
2738                 .aead = {
2739                         .base = {
2740                                 .cra_name = "seqiv(authenc(hmac(sha256),"
2741                                             "rfc3686(ctr(aes))))",
2742                                 .cra_driver_name = "seqiv-authenc-hmac-sha256-"
2743                                                    "rfc3686-ctr-aes-caam-qi2",
2744                                 .cra_blocksize = 1,
2745                         },
2746                         .setkey = aead_setkey,
2747                         .setauthsize = aead_setauthsize,
2748                         .encrypt = aead_encrypt,
2749                         .decrypt = aead_decrypt,
2750                         .ivsize = CTR_RFC3686_IV_SIZE,
2751                         .maxauthsize = SHA256_DIGEST_SIZE,
2752                 },
2753                 .caam = {
2754                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2755                                            OP_ALG_AAI_CTR_MOD128,
2756                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2757                                            OP_ALG_AAI_HMAC_PRECOMP,
2758                         .rfc3686 = true,
2759                         .geniv = true,
2760                 },
2761         },
2762         {
2763                 .aead = {
2764                         .base = {
2765                                 .cra_name = "authenc(hmac(sha384),"
2766                                             "rfc3686(ctr(aes)))",
2767                                 .cra_driver_name = "authenc-hmac-sha384-"
2768                                                    "rfc3686-ctr-aes-caam-qi2",
2769                                 .cra_blocksize = 1,
2770                         },
2771                         .setkey = aead_setkey,
2772                         .setauthsize = aead_setauthsize,
2773                         .encrypt = aead_encrypt,
2774                         .decrypt = aead_decrypt,
2775                         .ivsize = CTR_RFC3686_IV_SIZE,
2776                         .maxauthsize = SHA384_DIGEST_SIZE,
2777                 },
2778                 .caam = {
2779                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2780                                            OP_ALG_AAI_CTR_MOD128,
2781                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2782                                            OP_ALG_AAI_HMAC_PRECOMP,
2783                         .rfc3686 = true,
2784                 },
2785         },
2786         {
2787                 .aead = {
2788                         .base = {
2789                                 .cra_name = "seqiv(authenc(hmac(sha384),"
2790                                             "rfc3686(ctr(aes))))",
2791                                 .cra_driver_name = "seqiv-authenc-hmac-sha384-"
2792                                                    "rfc3686-ctr-aes-caam-qi2",
2793                                 .cra_blocksize = 1,
2794                         },
2795                         .setkey = aead_setkey,
2796                         .setauthsize = aead_setauthsize,
2797                         .encrypt = aead_encrypt,
2798                         .decrypt = aead_decrypt,
2799                         .ivsize = CTR_RFC3686_IV_SIZE,
2800                         .maxauthsize = SHA384_DIGEST_SIZE,
2801                 },
2802                 .caam = {
2803                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2804                                            OP_ALG_AAI_CTR_MOD128,
2805                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2806                                            OP_ALG_AAI_HMAC_PRECOMP,
2807                         .rfc3686 = true,
2808                         .geniv = true,
2809                 },
2810         },
2811         {
2812                 .aead = {
2813                         .base = {
2814                                 .cra_name = "rfc7539(chacha20,poly1305)",
2815                                 .cra_driver_name = "rfc7539-chacha20-poly1305-"
2816                                                    "caam-qi2",
2817                                 .cra_blocksize = 1,
2818                         },
2819                         .setkey = chachapoly_setkey,
2820                         .setauthsize = chachapoly_setauthsize,
2821                         .encrypt = aead_encrypt,
2822                         .decrypt = aead_decrypt,
2823                         .ivsize = CHACHAPOLY_IV_SIZE,
2824                         .maxauthsize = POLY1305_DIGEST_SIZE,
2825                 },
2826                 .caam = {
2827                         .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2828                                            OP_ALG_AAI_AEAD,
2829                         .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2830                                            OP_ALG_AAI_AEAD,
2831                         .nodkp = true,
2832                 },
2833         },
2834         {
2835                 .aead = {
2836                         .base = {
2837                                 .cra_name = "rfc7539esp(chacha20,poly1305)",
2838                                 .cra_driver_name = "rfc7539esp-chacha20-"
2839                                                    "poly1305-caam-qi2",
2840                                 .cra_blocksize = 1,
2841                         },
2842                         .setkey = chachapoly_setkey,
2843                         .setauthsize = chachapoly_setauthsize,
2844                         .encrypt = aead_encrypt,
2845                         .decrypt = aead_decrypt,
2846                         .ivsize = 8,
2847                         .maxauthsize = POLY1305_DIGEST_SIZE,
2848                 },
2849                 .caam = {
2850                         .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2851                                            OP_ALG_AAI_AEAD,
2852                         .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2853                                            OP_ALG_AAI_AEAD,
2854                         .nodkp = true,
2855                 },
2856         },
2857         {
2858                 .aead = {
2859                         .base = {
2860                                 .cra_name = "authenc(hmac(sha512),"
2861                                             "rfc3686(ctr(aes)))",
2862                                 .cra_driver_name = "authenc-hmac-sha512-"
2863                                                    "rfc3686-ctr-aes-caam-qi2",
2864                                 .cra_blocksize = 1,
2865                         },
2866                         .setkey = aead_setkey,
2867                         .setauthsize = aead_setauthsize,
2868                         .encrypt = aead_encrypt,
2869                         .decrypt = aead_decrypt,
2870                         .ivsize = CTR_RFC3686_IV_SIZE,
2871                         .maxauthsize = SHA512_DIGEST_SIZE,
2872                 },
2873                 .caam = {
2874                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2875                                            OP_ALG_AAI_CTR_MOD128,
2876                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2877                                            OP_ALG_AAI_HMAC_PRECOMP,
2878                         .rfc3686 = true,
2879                 },
2880         },
2881         {
2882                 .aead = {
2883                         .base = {
2884                                 .cra_name = "seqiv(authenc(hmac(sha512),"
2885                                             "rfc3686(ctr(aes))))",
2886                                 .cra_driver_name = "seqiv-authenc-hmac-sha512-"
2887                                                    "rfc3686-ctr-aes-caam-qi2",
2888                                 .cra_blocksize = 1,
2889                         },
2890                         .setkey = aead_setkey,
2891                         .setauthsize = aead_setauthsize,
2892                         .encrypt = aead_encrypt,
2893                         .decrypt = aead_decrypt,
2894                         .ivsize = CTR_RFC3686_IV_SIZE,
2895                         .maxauthsize = SHA512_DIGEST_SIZE,
2896                 },
2897                 .caam = {
2898                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2899                                            OP_ALG_AAI_CTR_MOD128,
2900                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2901                                            OP_ALG_AAI_HMAC_PRECOMP,
2902                         .rfc3686 = true,
2903                         .geniv = true,
2904                 },
2905         },
2906 };
2907
2908 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
2909 {
2910         struct skcipher_alg *alg = &t_alg->skcipher;
2911
2912         alg->base.cra_module = THIS_MODULE;
2913         alg->base.cra_priority = CAAM_CRA_PRIORITY;
2914         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
2915         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
2916                               CRYPTO_ALG_KERN_DRIVER_ONLY;
2917
2918         alg->init = caam_cra_init_skcipher;
2919         alg->exit = caam_cra_exit;
2920 }
2921
2922 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
2923 {
2924         struct aead_alg *alg = &t_alg->aead;
2925
2926         alg->base.cra_module = THIS_MODULE;
2927         alg->base.cra_priority = CAAM_CRA_PRIORITY;
2928         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
2929         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
2930                               CRYPTO_ALG_KERN_DRIVER_ONLY;
2931
2932         alg->init = caam_cra_init_aead;
2933         alg->exit = caam_cra_exit_aead;
2934 }
2935
2936 /* max hash key is max split key size */
2937 #define CAAM_MAX_HASH_KEY_SIZE          (SHA512_DIGEST_SIZE * 2)
2938
2939 #define CAAM_MAX_HASH_BLOCK_SIZE        SHA512_BLOCK_SIZE
2940
2941 /* caam context sizes for hashes: running digest + 8 */
2942 #define HASH_MSG_LEN                    8
2943 #define MAX_CTX_LEN                     (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
2944
2945 enum hash_optype {
2946         UPDATE = 0,
2947         UPDATE_FIRST,
2948         FINALIZE,
2949         DIGEST,
2950         HASH_NUM_OP
2951 };
2952
2953 /**
2954  * caam_hash_ctx - ahash per-session context
2955  * @flc: Flow Contexts array
2956  * @key: authentication key
2957  * @flc_dma: I/O virtual addresses of the Flow Contexts
2958  * @dev: dpseci device
2959  * @ctx_len: size of Context Register
2960  * @adata: hashing algorithm details
2961  */
2962 struct caam_hash_ctx {
2963         struct caam_flc flc[HASH_NUM_OP];
2964         u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
2965         dma_addr_t flc_dma[HASH_NUM_OP];
2966         struct device *dev;
2967         int ctx_len;
2968         struct alginfo adata;
2969 };
2970
2971 /* ahash state */
2972 struct caam_hash_state {
2973         struct caam_request caam_req;
2974         dma_addr_t buf_dma;
2975         dma_addr_t ctx_dma;
2976         int ctx_dma_len;
2977         u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
2978         int buflen;
2979         int next_buflen;
2980         u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
2981         int (*update)(struct ahash_request *req);
2982         int (*final)(struct ahash_request *req);
2983         int (*finup)(struct ahash_request *req);
2984 };
2985
2986 struct caam_export_state {
2987         u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
2988         u8 caam_ctx[MAX_CTX_LEN];
2989         int buflen;
2990         int (*update)(struct ahash_request *req);
2991         int (*final)(struct ahash_request *req);
2992         int (*finup)(struct ahash_request *req);
2993 };
2994
2995 /* Map current buffer in state (if length > 0) and put it in link table */
2996 static inline int buf_map_to_qm_sg(struct device *dev,
2997                                    struct dpaa2_sg_entry *qm_sg,
2998                                    struct caam_hash_state *state)
2999 {
3000         int buflen = state->buflen;
3001
3002         if (!buflen)
3003                 return 0;
3004
3005         state->buf_dma = dma_map_single(dev, state->buf, buflen,
3006                                         DMA_TO_DEVICE);
3007         if (dma_mapping_error(dev, state->buf_dma)) {
3008                 dev_err(dev, "unable to map buf\n");
3009                 state->buf_dma = 0;
3010                 return -ENOMEM;
3011         }
3012
3013         dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0);
3014
3015         return 0;
3016 }
3017
3018 /* Map state->caam_ctx, and add it to link table */
3019 static inline int ctx_map_to_qm_sg(struct device *dev,
3020                                    struct caam_hash_state *state, int ctx_len,
3021                                    struct dpaa2_sg_entry *qm_sg, u32 flag)
3022 {
3023         state->ctx_dma_len = ctx_len;
3024         state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag);
3025         if (dma_mapping_error(dev, state->ctx_dma)) {
3026                 dev_err(dev, "unable to map ctx\n");
3027                 state->ctx_dma = 0;
3028                 return -ENOMEM;
3029         }
3030
3031         dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0);
3032
3033         return 0;
3034 }
3035
3036 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
3037 {
3038         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3039         int digestsize = crypto_ahash_digestsize(ahash);
3040         struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
3041         struct caam_flc *flc;
3042         u32 *desc;
3043
3044         /* ahash_update shared descriptor */
3045         flc = &ctx->flc[UPDATE];
3046         desc = flc->sh_desc;
3047         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len,
3048                           ctx->ctx_len, true, priv->sec_attr.era);
3049         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3050         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE],
3051                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3052         print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ",
3053                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3054                              1);
3055
3056         /* ahash_update_first shared descriptor */
3057         flc = &ctx->flc[UPDATE_FIRST];
3058         desc = flc->sh_desc;
3059         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
3060                           ctx->ctx_len, false, priv->sec_attr.era);
3061         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3062         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST],
3063                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3064         print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ",
3065                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3066                              1);
3067
3068         /* ahash_final shared descriptor */
3069         flc = &ctx->flc[FINALIZE];
3070         desc = flc->sh_desc;
3071         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize,
3072                           ctx->ctx_len, true, priv->sec_attr.era);
3073         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3074         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE],
3075                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3076         print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ",
3077                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3078                              1);
3079
3080         /* ahash_digest shared descriptor */
3081         flc = &ctx->flc[DIGEST];
3082         desc = flc->sh_desc;
3083         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize,
3084                           ctx->ctx_len, false, priv->sec_attr.era);
3085         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3086         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST],
3087                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3088         print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ",
3089                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3090                              1);
3091
3092         return 0;
3093 }
3094
3095 struct split_key_sh_result {
3096         struct completion completion;
3097         int err;
3098         struct device *dev;
3099 };
3100
3101 static void split_key_sh_done(void *cbk_ctx, u32 err)
3102 {
3103         struct split_key_sh_result *res = cbk_ctx;
3104
3105         dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
3106
3107         res->err = err ? caam_qi2_strstatus(res->dev, err) : 0;
3108         complete(&res->completion);
3109 }
3110
3111 /* Digest hash size if it is too large */
3112 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
3113                            u32 digestsize)
3114 {
3115         struct caam_request *req_ctx;
3116         u32 *desc;
3117         struct split_key_sh_result result;
3118         dma_addr_t key_dma;
3119         struct caam_flc *flc;
3120         dma_addr_t flc_dma;
3121         int ret = -ENOMEM;
3122         struct dpaa2_fl_entry *in_fle, *out_fle;
3123
3124         req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA);
3125         if (!req_ctx)
3126                 return -ENOMEM;
3127
3128         in_fle = &req_ctx->fd_flt[1];
3129         out_fle = &req_ctx->fd_flt[0];
3130
3131         flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA);
3132         if (!flc)
3133                 goto err_flc;
3134
3135         key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL);
3136         if (dma_mapping_error(ctx->dev, key_dma)) {
3137                 dev_err(ctx->dev, "unable to map key memory\n");
3138                 goto err_key_dma;
3139         }
3140
3141         desc = flc->sh_desc;
3142
3143         init_sh_desc(desc, 0);
3144
3145         /* descriptor to perform unkeyed hash on key_in */
3146         append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
3147                          OP_ALG_AS_INITFINAL);
3148         append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
3149                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
3150         append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
3151                          LDST_SRCDST_BYTE_CONTEXT);
3152
3153         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3154         flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) +
3155                                  desc_bytes(desc), DMA_TO_DEVICE);
3156         if (dma_mapping_error(ctx->dev, flc_dma)) {
3157                 dev_err(ctx->dev, "unable to map shared descriptor\n");
3158                 goto err_flc_dma;
3159         }
3160
3161         dpaa2_fl_set_final(in_fle, true);
3162         dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3163         dpaa2_fl_set_addr(in_fle, key_dma);
3164         dpaa2_fl_set_len(in_fle, *keylen);
3165         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3166         dpaa2_fl_set_addr(out_fle, key_dma);
3167         dpaa2_fl_set_len(out_fle, digestsize);
3168
3169         print_hex_dump_debug("key_in@" __stringify(__LINE__)": ",
3170                              DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1);
3171         print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ",
3172                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3173                              1);
3174
3175         result.err = 0;
3176         init_completion(&result.completion);
3177         result.dev = ctx->dev;
3178
3179         req_ctx->flc = flc;
3180         req_ctx->flc_dma = flc_dma;
3181         req_ctx->cbk = split_key_sh_done;
3182         req_ctx->ctx = &result;
3183
3184         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3185         if (ret == -EINPROGRESS) {
3186                 /* in progress */
3187                 wait_for_completion(&result.completion);
3188                 ret = result.err;
3189                 print_hex_dump_debug("digested key@" __stringify(__LINE__)": ",
3190                                      DUMP_PREFIX_ADDRESS, 16, 4, key,
3191                                      digestsize, 1);
3192         }
3193
3194         dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc),
3195                          DMA_TO_DEVICE);
3196 err_flc_dma:
3197         dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL);
3198 err_key_dma:
3199         kfree(flc);
3200 err_flc:
3201         kfree(req_ctx);
3202
3203         *keylen = digestsize;
3204
3205         return ret;
3206 }
3207
3208 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
3209                         unsigned int keylen)
3210 {
3211         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3212         unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
3213         unsigned int digestsize = crypto_ahash_digestsize(ahash);
3214         int ret;
3215         u8 *hashed_key = NULL;
3216
3217         dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize);
3218
3219         if (keylen > blocksize) {
3220                 hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
3221                 if (!hashed_key)
3222                         return -ENOMEM;
3223                 ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize);
3224                 if (ret)
3225                         goto bad_free_key;
3226                 key = hashed_key;
3227         }
3228
3229         ctx->adata.keylen = keylen;
3230         ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
3231                                               OP_ALG_ALGSEL_MASK);
3232         if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE)
3233                 goto bad_free_key;
3234
3235         ctx->adata.key_virt = key;
3236         ctx->adata.key_inline = true;
3237
3238         /*
3239          * In case |user key| > |derived key|, using DKP<imm,imm> would result
3240          * in invalid opcodes (last bytes of user key) in the resulting
3241          * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
3242          * addresses are needed.
3243          */
3244         if (keylen > ctx->adata.keylen_pad) {
3245                 memcpy(ctx->key, key, keylen);
3246                 dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma,
3247                                            ctx->adata.keylen_pad,
3248                                            DMA_TO_DEVICE);
3249         }
3250
3251         ret = ahash_set_sh_desc(ahash);
3252         kfree(hashed_key);
3253         return ret;
3254 bad_free_key:
3255         kfree(hashed_key);
3256         return -EINVAL;
3257 }
3258
3259 static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc,
3260                                struct ahash_request *req)
3261 {
3262         struct caam_hash_state *state = ahash_request_ctx(req);
3263
3264         if (edesc->src_nents)
3265                 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
3266
3267         if (edesc->qm_sg_bytes)
3268                 dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes,
3269                                  DMA_TO_DEVICE);
3270
3271         if (state->buf_dma) {
3272                 dma_unmap_single(dev, state->buf_dma, state->buflen,
3273                                  DMA_TO_DEVICE);
3274                 state->buf_dma = 0;
3275         }
3276 }
3277
3278 static inline void ahash_unmap_ctx(struct device *dev,
3279                                    struct ahash_edesc *edesc,
3280                                    struct ahash_request *req, u32 flag)
3281 {
3282         struct caam_hash_state *state = ahash_request_ctx(req);
3283
3284         if (state->ctx_dma) {
3285                 dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag);
3286                 state->ctx_dma = 0;
3287         }
3288         ahash_unmap(dev, edesc, req);
3289 }
3290
3291 static void ahash_done(void *cbk_ctx, u32 status)
3292 {
3293         struct crypto_async_request *areq = cbk_ctx;
3294         struct ahash_request *req = ahash_request_cast(areq);
3295         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3296         struct caam_hash_state *state = ahash_request_ctx(req);
3297         struct ahash_edesc *edesc = state->caam_req.edesc;
3298         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3299         int digestsize = crypto_ahash_digestsize(ahash);
3300         int ecode = 0;
3301
3302         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3303
3304         if (unlikely(status))
3305                 ecode = caam_qi2_strstatus(ctx->dev, status);
3306
3307         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3308         memcpy(req->result, state->caam_ctx, digestsize);
3309         qi_cache_free(edesc);
3310
3311         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3312                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3313                              ctx->ctx_len, 1);
3314
3315         req->base.complete(&req->base, ecode);
3316 }
3317
3318 static void ahash_done_bi(void *cbk_ctx, u32 status)
3319 {
3320         struct crypto_async_request *areq = cbk_ctx;
3321         struct ahash_request *req = ahash_request_cast(areq);
3322         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3323         struct caam_hash_state *state = ahash_request_ctx(req);
3324         struct ahash_edesc *edesc = state->caam_req.edesc;
3325         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3326         int ecode = 0;
3327
3328         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3329
3330         if (unlikely(status))
3331                 ecode = caam_qi2_strstatus(ctx->dev, status);
3332
3333         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3334         qi_cache_free(edesc);
3335
3336         scatterwalk_map_and_copy(state->buf, req->src,
3337                                  req->nbytes - state->next_buflen,
3338                                  state->next_buflen, 0);
3339         state->buflen = state->next_buflen;
3340
3341         print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3342                              DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
3343                              state->buflen, 1);
3344
3345         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3346                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3347                              ctx->ctx_len, 1);
3348         if (req->result)
3349                 print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3350                                      DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3351                                      crypto_ahash_digestsize(ahash), 1);
3352
3353         req->base.complete(&req->base, ecode);
3354 }
3355
3356 static void ahash_done_ctx_src(void *cbk_ctx, u32 status)
3357 {
3358         struct crypto_async_request *areq = cbk_ctx;
3359         struct ahash_request *req = ahash_request_cast(areq);
3360         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3361         struct caam_hash_state *state = ahash_request_ctx(req);
3362         struct ahash_edesc *edesc = state->caam_req.edesc;
3363         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3364         int digestsize = crypto_ahash_digestsize(ahash);
3365         int ecode = 0;
3366
3367         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3368
3369         if (unlikely(status))
3370                 ecode = caam_qi2_strstatus(ctx->dev, status);
3371
3372         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3373         memcpy(req->result, state->caam_ctx, digestsize);
3374         qi_cache_free(edesc);
3375
3376         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3377                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3378                              ctx->ctx_len, 1);
3379
3380         req->base.complete(&req->base, ecode);
3381 }
3382
3383 static void ahash_done_ctx_dst(void *cbk_ctx, u32 status)
3384 {
3385         struct crypto_async_request *areq = cbk_ctx;
3386         struct ahash_request *req = ahash_request_cast(areq);
3387         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3388         struct caam_hash_state *state = ahash_request_ctx(req);
3389         struct ahash_edesc *edesc = state->caam_req.edesc;
3390         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3391         int ecode = 0;
3392
3393         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3394
3395         if (unlikely(status))
3396                 ecode = caam_qi2_strstatus(ctx->dev, status);
3397
3398         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3399         qi_cache_free(edesc);
3400
3401         scatterwalk_map_and_copy(state->buf, req->src,
3402                                  req->nbytes - state->next_buflen,
3403                                  state->next_buflen, 0);
3404         state->buflen = state->next_buflen;
3405
3406         print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3407                              DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
3408                              state->buflen, 1);
3409
3410         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3411                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3412                              ctx->ctx_len, 1);
3413         if (req->result)
3414                 print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3415                                      DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3416                                      crypto_ahash_digestsize(ahash), 1);
3417
3418         req->base.complete(&req->base, ecode);
3419 }
3420
3421 static int ahash_update_ctx(struct ahash_request *req)
3422 {
3423         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3424         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3425         struct caam_hash_state *state = ahash_request_ctx(req);
3426         struct caam_request *req_ctx = &state->caam_req;
3427         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3428         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3429         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3430                       GFP_KERNEL : GFP_ATOMIC;
3431         u8 *buf = state->buf;
3432         int *buflen = &state->buflen;
3433         int *next_buflen = &state->next_buflen;
3434         int in_len = *buflen + req->nbytes, to_hash;
3435         int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index;
3436         struct ahash_edesc *edesc;
3437         int ret = 0;
3438
3439         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
3440         to_hash = in_len - *next_buflen;
3441
3442         if (to_hash) {
3443                 struct dpaa2_sg_entry *sg_table;
3444                 int src_len = req->nbytes - *next_buflen;
3445
3446                 src_nents = sg_nents_for_len(req->src, src_len);
3447                 if (src_nents < 0) {
3448                         dev_err(ctx->dev, "Invalid number of src SG.\n");
3449                         return src_nents;
3450                 }
3451
3452                 if (src_nents) {
3453                         mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3454                                                   DMA_TO_DEVICE);
3455                         if (!mapped_nents) {
3456                                 dev_err(ctx->dev, "unable to DMA map source\n");
3457                                 return -ENOMEM;
3458                         }
3459                 } else {
3460                         mapped_nents = 0;
3461                 }
3462
3463                 /* allocate space for base edesc and link tables */
3464                 edesc = qi_cache_zalloc(GFP_DMA | flags);
3465                 if (!edesc) {
3466                         dma_unmap_sg(ctx->dev, req->src, src_nents,
3467                                      DMA_TO_DEVICE);
3468                         return -ENOMEM;
3469                 }
3470
3471                 edesc->src_nents = src_nents;
3472                 qm_sg_src_index = 1 + (*buflen ? 1 : 0);
3473                 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3474                               sizeof(*sg_table);
3475                 sg_table = &edesc->sgt[0];
3476
3477                 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3478                                        DMA_BIDIRECTIONAL);
3479                 if (ret)
3480                         goto unmap_ctx;
3481
3482                 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3483                 if (ret)
3484                         goto unmap_ctx;
3485
3486                 if (mapped_nents) {
3487                         sg_to_qm_sg_last(req->src, src_len,
3488                                          sg_table + qm_sg_src_index, 0);
3489                 } else {
3490                         dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1,
3491                                            true);
3492                 }
3493
3494                 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3495                                                   qm_sg_bytes, DMA_TO_DEVICE);
3496                 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3497                         dev_err(ctx->dev, "unable to map S/G table\n");
3498                         ret = -ENOMEM;
3499                         goto unmap_ctx;
3500                 }
3501                 edesc->qm_sg_bytes = qm_sg_bytes;
3502
3503                 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3504                 dpaa2_fl_set_final(in_fle, true);
3505                 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3506                 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3507                 dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash);
3508                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3509                 dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3510                 dpaa2_fl_set_len(out_fle, ctx->ctx_len);
3511
3512                 req_ctx->flc = &ctx->flc[UPDATE];
3513                 req_ctx->flc_dma = ctx->flc_dma[UPDATE];
3514                 req_ctx->cbk = ahash_done_bi;
3515                 req_ctx->ctx = &req->base;
3516                 req_ctx->edesc = edesc;
3517
3518                 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3519                 if (ret != -EINPROGRESS &&
3520                     !(ret == -EBUSY &&
3521                       req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3522                         goto unmap_ctx;
3523         } else if (*next_buflen) {
3524                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
3525                                          req->nbytes, 0);
3526                 *buflen = *next_buflen;
3527
3528                 print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3529                                      DUMP_PREFIX_ADDRESS, 16, 4, buf,
3530                                      *buflen, 1);
3531         }
3532
3533         return ret;
3534 unmap_ctx:
3535         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3536         qi_cache_free(edesc);
3537         return ret;
3538 }
3539
3540 static int ahash_final_ctx(struct ahash_request *req)
3541 {
3542         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3543         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3544         struct caam_hash_state *state = ahash_request_ctx(req);
3545         struct caam_request *req_ctx = &state->caam_req;
3546         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3547         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3548         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3549                       GFP_KERNEL : GFP_ATOMIC;
3550         int buflen = state->buflen;
3551         int qm_sg_bytes;
3552         int digestsize = crypto_ahash_digestsize(ahash);
3553         struct ahash_edesc *edesc;
3554         struct dpaa2_sg_entry *sg_table;
3555         int ret;
3556
3557         /* allocate space for base edesc and link tables */
3558         edesc = qi_cache_zalloc(GFP_DMA | flags);
3559         if (!edesc)
3560                 return -ENOMEM;
3561
3562         qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table);
3563         sg_table = &edesc->sgt[0];
3564
3565         ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3566                                DMA_BIDIRECTIONAL);
3567         if (ret)
3568                 goto unmap_ctx;
3569
3570         ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3571         if (ret)
3572                 goto unmap_ctx;
3573
3574         dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true);
3575
3576         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3577                                           DMA_TO_DEVICE);
3578         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3579                 dev_err(ctx->dev, "unable to map S/G table\n");
3580                 ret = -ENOMEM;
3581                 goto unmap_ctx;
3582         }
3583         edesc->qm_sg_bytes = qm_sg_bytes;
3584
3585         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3586         dpaa2_fl_set_final(in_fle, true);
3587         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3588         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3589         dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen);
3590         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3591         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3592         dpaa2_fl_set_len(out_fle, digestsize);
3593
3594         req_ctx->flc = &ctx->flc[FINALIZE];
3595         req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3596         req_ctx->cbk = ahash_done_ctx_src;
3597         req_ctx->ctx = &req->base;
3598         req_ctx->edesc = edesc;
3599
3600         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3601         if (ret == -EINPROGRESS ||
3602             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3603                 return ret;
3604
3605 unmap_ctx:
3606         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3607         qi_cache_free(edesc);
3608         return ret;
3609 }
3610
3611 static int ahash_finup_ctx(struct ahash_request *req)
3612 {
3613         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3614         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3615         struct caam_hash_state *state = ahash_request_ctx(req);
3616         struct caam_request *req_ctx = &state->caam_req;
3617         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3618         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3619         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3620                       GFP_KERNEL : GFP_ATOMIC;
3621         int buflen = state->buflen;
3622         int qm_sg_bytes, qm_sg_src_index;
3623         int src_nents, mapped_nents;
3624         int digestsize = crypto_ahash_digestsize(ahash);
3625         struct ahash_edesc *edesc;
3626         struct dpaa2_sg_entry *sg_table;
3627         int ret;
3628
3629         src_nents = sg_nents_for_len(req->src, req->nbytes);
3630         if (src_nents < 0) {
3631                 dev_err(ctx->dev, "Invalid number of src SG.\n");
3632                 return src_nents;
3633         }
3634
3635         if (src_nents) {
3636                 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3637                                           DMA_TO_DEVICE);
3638                 if (!mapped_nents) {
3639                         dev_err(ctx->dev, "unable to DMA map source\n");
3640                         return -ENOMEM;
3641                 }
3642         } else {
3643                 mapped_nents = 0;
3644         }
3645
3646         /* allocate space for base edesc and link tables */
3647         edesc = qi_cache_zalloc(GFP_DMA | flags);
3648         if (!edesc) {
3649                 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3650                 return -ENOMEM;
3651         }
3652
3653         edesc->src_nents = src_nents;
3654         qm_sg_src_index = 1 + (buflen ? 1 : 0);
3655         qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3656                       sizeof(*sg_table);
3657         sg_table = &edesc->sgt[0];
3658
3659         ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3660                                DMA_BIDIRECTIONAL);
3661         if (ret)
3662                 goto unmap_ctx;
3663
3664         ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3665         if (ret)
3666                 goto unmap_ctx;
3667
3668         sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0);
3669
3670         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3671                                           DMA_TO_DEVICE);
3672         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3673                 dev_err(ctx->dev, "unable to map S/G table\n");
3674                 ret = -ENOMEM;
3675                 goto unmap_ctx;
3676         }
3677         edesc->qm_sg_bytes = qm_sg_bytes;
3678
3679         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3680         dpaa2_fl_set_final(in_fle, true);
3681         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3682         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3683         dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes);
3684         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3685         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3686         dpaa2_fl_set_len(out_fle, digestsize);
3687
3688         req_ctx->flc = &ctx->flc[FINALIZE];
3689         req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3690         req_ctx->cbk = ahash_done_ctx_src;
3691         req_ctx->ctx = &req->base;
3692         req_ctx->edesc = edesc;
3693
3694         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3695         if (ret == -EINPROGRESS ||
3696             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3697                 return ret;
3698
3699 unmap_ctx:
3700         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3701         qi_cache_free(edesc);
3702         return ret;
3703 }
3704
3705 static int ahash_digest(struct ahash_request *req)
3706 {
3707         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3708         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3709         struct caam_hash_state *state = ahash_request_ctx(req);
3710         struct caam_request *req_ctx = &state->caam_req;
3711         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3712         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3713         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3714                       GFP_KERNEL : GFP_ATOMIC;
3715         int digestsize = crypto_ahash_digestsize(ahash);
3716         int src_nents, mapped_nents;
3717         struct ahash_edesc *edesc;
3718         int ret = -ENOMEM;
3719
3720         state->buf_dma = 0;
3721
3722         src_nents = sg_nents_for_len(req->src, req->nbytes);
3723         if (src_nents < 0) {
3724                 dev_err(ctx->dev, "Invalid number of src SG.\n");
3725                 return src_nents;
3726         }
3727
3728         if (src_nents) {
3729                 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3730                                           DMA_TO_DEVICE);
3731                 if (!mapped_nents) {
3732                         dev_err(ctx->dev, "unable to map source for DMA\n");
3733                         return ret;
3734                 }
3735         } else {
3736                 mapped_nents = 0;
3737         }
3738
3739         /* allocate space for base edesc and link tables */
3740         edesc = qi_cache_zalloc(GFP_DMA | flags);
3741         if (!edesc) {
3742                 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3743                 return ret;
3744         }
3745
3746         edesc->src_nents = src_nents;
3747         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3748
3749         if (mapped_nents > 1) {
3750                 int qm_sg_bytes;
3751                 struct dpaa2_sg_entry *sg_table = &edesc->sgt[0];
3752
3753                 qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table);
3754                 sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0);
3755                 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3756                                                   qm_sg_bytes, DMA_TO_DEVICE);
3757                 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3758                         dev_err(ctx->dev, "unable to map S/G table\n");
3759                         goto unmap;
3760                 }
3761                 edesc->qm_sg_bytes = qm_sg_bytes;
3762                 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3763                 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3764         } else {
3765                 dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3766                 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
3767         }
3768
3769         state->ctx_dma_len = digestsize;
3770         state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3771                                         DMA_FROM_DEVICE);
3772         if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3773                 dev_err(ctx->dev, "unable to map ctx\n");
3774                 state->ctx_dma = 0;
3775                 goto unmap;
3776         }
3777
3778         dpaa2_fl_set_final(in_fle, true);
3779         dpaa2_fl_set_len(in_fle, req->nbytes);
3780         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3781         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3782         dpaa2_fl_set_len(out_fle, digestsize);
3783
3784         req_ctx->flc = &ctx->flc[DIGEST];
3785         req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3786         req_ctx->cbk = ahash_done;
3787         req_ctx->ctx = &req->base;
3788         req_ctx->edesc = edesc;
3789         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3790         if (ret == -EINPROGRESS ||
3791             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3792                 return ret;
3793
3794 unmap:
3795         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3796         qi_cache_free(edesc);
3797         return ret;
3798 }
3799
3800 static int ahash_final_no_ctx(struct ahash_request *req)
3801 {
3802         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3803         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3804         struct caam_hash_state *state = ahash_request_ctx(req);
3805         struct caam_request *req_ctx = &state->caam_req;
3806         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3807         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3808         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3809                       GFP_KERNEL : GFP_ATOMIC;
3810         u8 *buf = state->buf;
3811         int buflen = state->buflen;
3812         int digestsize = crypto_ahash_digestsize(ahash);
3813         struct ahash_edesc *edesc;
3814         int ret = -ENOMEM;
3815
3816         /* allocate space for base edesc and link tables */
3817         edesc = qi_cache_zalloc(GFP_DMA | flags);
3818         if (!edesc)
3819                 return ret;
3820
3821         if (buflen) {
3822                 state->buf_dma = dma_map_single(ctx->dev, buf, buflen,
3823                                                 DMA_TO_DEVICE);
3824                 if (dma_mapping_error(ctx->dev, state->buf_dma)) {
3825                         dev_err(ctx->dev, "unable to map src\n");
3826                         goto unmap;
3827                 }
3828         }
3829
3830         state->ctx_dma_len = digestsize;
3831         state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3832                                         DMA_FROM_DEVICE);
3833         if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3834                 dev_err(ctx->dev, "unable to map ctx\n");
3835                 state->ctx_dma = 0;
3836                 goto unmap;
3837         }
3838
3839         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3840         dpaa2_fl_set_final(in_fle, true);
3841         /*
3842          * crypto engine requires the input entry to be present when
3843          * "frame list" FD is used.
3844          * Since engine does not support FMT=2'b11 (unused entry type), leaving
3845          * in_fle zeroized (except for "Final" flag) is the best option.
3846          */
3847         if (buflen) {
3848                 dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3849                 dpaa2_fl_set_addr(in_fle, state->buf_dma);
3850                 dpaa2_fl_set_len(in_fle, buflen);
3851         }
3852         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3853         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3854         dpaa2_fl_set_len(out_fle, digestsize);
3855
3856         req_ctx->flc = &ctx->flc[DIGEST];
3857         req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3858         req_ctx->cbk = ahash_done;
3859         req_ctx->ctx = &req->base;
3860         req_ctx->edesc = edesc;
3861
3862         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3863         if (ret == -EINPROGRESS ||
3864             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3865                 return ret;
3866
3867 unmap:
3868         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3869         qi_cache_free(edesc);
3870         return ret;
3871 }
3872
3873 static int ahash_update_no_ctx(struct ahash_request *req)
3874 {
3875         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3876         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3877         struct caam_hash_state *state = ahash_request_ctx(req);
3878         struct caam_request *req_ctx = &state->caam_req;
3879         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3880         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3881         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3882                       GFP_KERNEL : GFP_ATOMIC;
3883         u8 *buf = state->buf;
3884         int *buflen = &state->buflen;
3885         int *next_buflen = &state->next_buflen;
3886         int in_len = *buflen + req->nbytes, to_hash;
3887         int qm_sg_bytes, src_nents, mapped_nents;
3888         struct ahash_edesc *edesc;
3889         int ret = 0;
3890
3891         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
3892         to_hash = in_len - *next_buflen;
3893
3894         if (to_hash) {
3895                 struct dpaa2_sg_entry *sg_table;
3896                 int src_len = req->nbytes - *next_buflen;
3897
3898                 src_nents = sg_nents_for_len(req->src, src_len);
3899                 if (src_nents < 0) {
3900                         dev_err(ctx->dev, "Invalid number of src SG.\n");
3901                         return src_nents;
3902                 }
3903
3904                 if (src_nents) {
3905                         mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3906                                                   DMA_TO_DEVICE);
3907                         if (!mapped_nents) {
3908                                 dev_err(ctx->dev, "unable to DMA map source\n");
3909                                 return -ENOMEM;
3910                         }
3911                 } else {
3912                         mapped_nents = 0;
3913                 }
3914
3915                 /* allocate space for base edesc and link tables */
3916                 edesc = qi_cache_zalloc(GFP_DMA | flags);
3917                 if (!edesc) {
3918                         dma_unmap_sg(ctx->dev, req->src, src_nents,
3919                                      DMA_TO_DEVICE);
3920                         return -ENOMEM;
3921                 }
3922
3923                 edesc->src_nents = src_nents;
3924                 qm_sg_bytes = pad_sg_nents(1 + mapped_nents) *
3925                               sizeof(*sg_table);
3926                 sg_table = &edesc->sgt[0];
3927
3928                 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
3929                 if (ret)
3930                         goto unmap_ctx;
3931
3932                 sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0);
3933
3934                 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3935                                                   qm_sg_bytes, DMA_TO_DEVICE);
3936                 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3937                         dev_err(ctx->dev, "unable to map S/G table\n");
3938                         ret = -ENOMEM;
3939                         goto unmap_ctx;
3940                 }
3941                 edesc->qm_sg_bytes = qm_sg_bytes;
3942
3943                 state->ctx_dma_len = ctx->ctx_len;
3944                 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
3945                                                 ctx->ctx_len, DMA_FROM_DEVICE);
3946                 if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3947                         dev_err(ctx->dev, "unable to map ctx\n");
3948                         state->ctx_dma = 0;
3949                         ret = -ENOMEM;
3950                         goto unmap_ctx;
3951                 }
3952
3953                 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3954                 dpaa2_fl_set_final(in_fle, true);
3955                 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3956                 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3957                 dpaa2_fl_set_len(in_fle, to_hash);
3958                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3959                 dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3960                 dpaa2_fl_set_len(out_fle, ctx->ctx_len);
3961
3962                 req_ctx->flc = &ctx->flc[UPDATE_FIRST];
3963                 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
3964                 req_ctx->cbk = ahash_done_ctx_dst;
3965                 req_ctx->ctx = &req->base;
3966                 req_ctx->edesc = edesc;
3967
3968                 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3969                 if (ret != -EINPROGRESS &&
3970                     !(ret == -EBUSY &&
3971                       req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3972                         goto unmap_ctx;
3973
3974                 state->update = ahash_update_ctx;
3975                 state->finup = ahash_finup_ctx;
3976                 state->final = ahash_final_ctx;
3977         } else if (*next_buflen) {
3978                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
3979                                          req->nbytes, 0);
3980                 *buflen = *next_buflen;
3981
3982                 print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3983                                      DUMP_PREFIX_ADDRESS, 16, 4, buf,
3984                                      *buflen, 1);
3985         }
3986
3987         return ret;
3988 unmap_ctx:
3989         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
3990         qi_cache_free(edesc);
3991         return ret;
3992 }
3993
3994 static int ahash_finup_no_ctx(struct ahash_request *req)
3995 {
3996         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3997         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3998         struct caam_hash_state *state = ahash_request_ctx(req);
3999         struct caam_request *req_ctx = &state->caam_req;
4000         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
4001         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4002         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4003                       GFP_KERNEL : GFP_ATOMIC;
4004         int buflen = state->buflen;
4005         int qm_sg_bytes, src_nents, mapped_nents;
4006         int digestsize = crypto_ahash_digestsize(ahash);
4007         struct ahash_edesc *edesc;
4008         struct dpaa2_sg_entry *sg_table;
4009         int ret = -ENOMEM;
4010
4011         src_nents = sg_nents_for_len(req->src, req->nbytes);
4012         if (src_nents < 0) {
4013                 dev_err(ctx->dev, "Invalid number of src SG.\n");
4014                 return src_nents;
4015         }
4016
4017         if (src_nents) {
4018                 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4019                                           DMA_TO_DEVICE);
4020                 if (!mapped_nents) {
4021                         dev_err(ctx->dev, "unable to DMA map source\n");
4022                         return ret;
4023                 }
4024         } else {
4025                 mapped_nents = 0;
4026         }
4027
4028         /* allocate space for base edesc and link tables */
4029         edesc = qi_cache_zalloc(GFP_DMA | flags);
4030         if (!edesc) {
4031                 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
4032                 return ret;
4033         }
4034
4035         edesc->src_nents = src_nents;
4036         qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table);
4037         sg_table = &edesc->sgt[0];
4038
4039         ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
4040         if (ret)
4041                 goto unmap;
4042
4043         sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0);
4044
4045         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
4046                                           DMA_TO_DEVICE);
4047         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4048                 dev_err(ctx->dev, "unable to map S/G table\n");
4049                 ret = -ENOMEM;
4050                 goto unmap;
4051         }
4052         edesc->qm_sg_bytes = qm_sg_bytes;
4053
4054         state->ctx_dma_len = digestsize;
4055         state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
4056                                         DMA_FROM_DEVICE);
4057         if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4058                 dev_err(ctx->dev, "unable to map ctx\n");
4059                 state->ctx_dma = 0;
4060                 ret = -ENOMEM;
4061                 goto unmap;
4062         }
4063
4064         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4065         dpaa2_fl_set_final(in_fle, true);
4066         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4067         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4068         dpaa2_fl_set_len(in_fle, buflen + req->nbytes);
4069         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4070         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4071         dpaa2_fl_set_len(out_fle, digestsize);
4072
4073         req_ctx->flc = &ctx->flc[DIGEST];
4074         req_ctx->flc_dma = ctx->flc_dma[DIGEST];
4075         req_ctx->cbk = ahash_done;
4076         req_ctx->ctx = &req->base;
4077         req_ctx->edesc = edesc;
4078         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4079         if (ret != -EINPROGRESS &&
4080             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
4081                 goto unmap;
4082
4083         return ret;
4084 unmap:
4085         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
4086         qi_cache_free(edesc);
4087         return ret;
4088 }
4089
4090 static int ahash_update_first(struct ahash_request *req)
4091 {
4092         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
4093         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
4094         struct caam_hash_state *state = ahash_request_ctx(req);
4095         struct caam_request *req_ctx = &state->caam_req;
4096         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
4097         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4098         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4099                       GFP_KERNEL : GFP_ATOMIC;
4100         u8 *buf = state->buf;
4101         int *buflen = &state->buflen;
4102         int *next_buflen = &state->next_buflen;
4103         int to_hash;
4104         int src_nents, mapped_nents;
4105         struct ahash_edesc *edesc;
4106         int ret = 0;
4107
4108         *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
4109                                       1);
4110         to_hash = req->nbytes - *next_buflen;
4111
4112         if (to_hash) {
4113                 struct dpaa2_sg_entry *sg_table;
4114                 int src_len = req->nbytes - *next_buflen;
4115
4116                 src_nents = sg_nents_for_len(req->src, src_len);
4117                 if (src_nents < 0) {
4118                         dev_err(ctx->dev, "Invalid number of src SG.\n");
4119                         return src_nents;
4120                 }
4121
4122                 if (src_nents) {
4123                         mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4124                                                   DMA_TO_DEVICE);
4125                         if (!mapped_nents) {
4126                                 dev_err(ctx->dev, "unable to map source for DMA\n");
4127                                 return -ENOMEM;
4128                         }
4129                 } else {
4130                         mapped_nents = 0;
4131                 }
4132
4133                 /* allocate space for base edesc and link tables */
4134                 edesc = qi_cache_zalloc(GFP_DMA | flags);
4135                 if (!edesc) {
4136                         dma_unmap_sg(ctx->dev, req->src, src_nents,
4137                                      DMA_TO_DEVICE);
4138                         return -ENOMEM;
4139                 }
4140
4141                 edesc->src_nents = src_nents;
4142                 sg_table = &edesc->sgt[0];
4143
4144                 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4145                 dpaa2_fl_set_final(in_fle, true);
4146                 dpaa2_fl_set_len(in_fle, to_hash);
4147
4148                 if (mapped_nents > 1) {
4149                         int qm_sg_bytes;
4150
4151                         sg_to_qm_sg_last(req->src, src_len, sg_table, 0);
4152                         qm_sg_bytes = pad_sg_nents(mapped_nents) *
4153                                       sizeof(*sg_table);
4154                         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
4155                                                           qm_sg_bytes,
4156                                                           DMA_TO_DEVICE);
4157                         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4158                                 dev_err(ctx->dev, "unable to map S/G table\n");
4159                                 ret = -ENOMEM;
4160                                 goto unmap_ctx;
4161                         }
4162                         edesc->qm_sg_bytes = qm_sg_bytes;
4163                         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4164                         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4165                 } else {
4166                         dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
4167                         dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
4168                 }
4169
4170                 state->ctx_dma_len = ctx->ctx_len;
4171                 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
4172                                                 ctx->ctx_len, DMA_FROM_DEVICE);
4173                 if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4174                         dev_err(ctx->dev, "unable to map ctx\n");
4175                         state->ctx_dma = 0;
4176                         ret = -ENOMEM;
4177                         goto unmap_ctx;
4178                 }
4179
4180                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4181                 dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4182                 dpaa2_fl_set_len(out_fle, ctx->ctx_len);
4183
4184                 req_ctx->flc = &ctx->flc[UPDATE_FIRST];
4185                 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
4186                 req_ctx->cbk = ahash_done_ctx_dst;
4187                 req_ctx->ctx = &req->base;
4188                 req_ctx->edesc = edesc;
4189
4190                 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4191                 if (ret != -EINPROGRESS &&
4192                     !(ret == -EBUSY && req->base.flags &
4193                       CRYPTO_TFM_REQ_MAY_BACKLOG))
4194                         goto unmap_ctx;
4195
4196                 state->update = ahash_update_ctx;
4197                 state->finup = ahash_finup_ctx;
4198                 state->final = ahash_final_ctx;
4199         } else if (*next_buflen) {
4200                 state->update = ahash_update_no_ctx;
4201                 state->finup = ahash_finup_no_ctx;
4202                 state->final = ahash_final_no_ctx;
4203                 scatterwalk_map_and_copy(buf, req->src, 0,
4204                                          req->nbytes, 0);
4205                 *buflen = *next_buflen;
4206
4207                 print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
4208                                      DUMP_PREFIX_ADDRESS, 16, 4, buf,
4209                                      *buflen, 1);
4210         }
4211
4212         return ret;
4213 unmap_ctx:
4214         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
4215         qi_cache_free(edesc);
4216         return ret;
4217 }
4218
4219 static int ahash_finup_first(struct ahash_request *req)
4220 {
4221         return ahash_digest(req);
4222 }
4223
4224 static int ahash_init(struct ahash_request *req)
4225 {
4226         struct caam_hash_state *state = ahash_request_ctx(req);
4227
4228         state->update = ahash_update_first;
4229         state->finup = ahash_finup_first;
4230         state->final = ahash_final_no_ctx;
4231
4232         state->ctx_dma = 0;
4233         state->ctx_dma_len = 0;
4234         state->buf_dma = 0;
4235         state->buflen = 0;
4236         state->next_buflen = 0;
4237
4238         return 0;
4239 }
4240
4241 static int ahash_update(struct ahash_request *req)
4242 {
4243         struct caam_hash_state *state = ahash_request_ctx(req);
4244
4245         return state->update(req);
4246 }
4247
4248 static int ahash_finup(struct ahash_request *req)
4249 {
4250         struct caam_hash_state *state = ahash_request_ctx(req);
4251
4252         return state->finup(req);
4253 }
4254
4255 static int ahash_final(struct ahash_request *req)
4256 {
4257         struct caam_hash_state *state = ahash_request_ctx(req);
4258
4259         return state->final(req);
4260 }
4261
4262 static int ahash_export(struct ahash_request *req, void *out)
4263 {
4264         struct caam_hash_state *state = ahash_request_ctx(req);
4265         struct caam_export_state *export = out;
4266         u8 *buf = state->buf;
4267         int len = state->buflen;
4268
4269         memcpy(export->buf, buf, len);
4270         memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
4271         export->buflen = len;
4272         export->update = state->update;
4273         export->final = state->final;
4274         export->finup = state->finup;
4275
4276         return 0;
4277 }
4278
4279 static int ahash_import(struct ahash_request *req, const void *in)
4280 {
4281         struct caam_hash_state *state = ahash_request_ctx(req);
4282         const struct caam_export_state *export = in;
4283
4284         memset(state, 0, sizeof(*state));
4285         memcpy(state->buf, export->buf, export->buflen);
4286         memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
4287         state->buflen = export->buflen;
4288         state->update = export->update;
4289         state->final = export->final;
4290         state->finup = export->finup;
4291
4292         return 0;
4293 }
4294
4295 struct caam_hash_template {
4296         char name[CRYPTO_MAX_ALG_NAME];
4297         char driver_name[CRYPTO_MAX_ALG_NAME];
4298         char hmac_name[CRYPTO_MAX_ALG_NAME];
4299         char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
4300         unsigned int blocksize;
4301         struct ahash_alg template_ahash;
4302         u32 alg_type;
4303 };
4304
4305 /* ahash descriptors */
4306 static struct caam_hash_template driver_hash[] = {
4307         {
4308                 .name = "sha1",
4309                 .driver_name = "sha1-caam-qi2",
4310                 .hmac_name = "hmac(sha1)",
4311                 .hmac_driver_name = "hmac-sha1-caam-qi2",
4312                 .blocksize = SHA1_BLOCK_SIZE,
4313                 .template_ahash = {
4314                         .init = ahash_init,
4315                         .update = ahash_update,
4316                         .final = ahash_final,
4317                         .finup = ahash_finup,
4318                         .digest = ahash_digest,
4319                         .export = ahash_export,
4320                         .import = ahash_import,
4321                         .setkey = ahash_setkey,
4322                         .halg = {
4323                                 .digestsize = SHA1_DIGEST_SIZE,
4324                                 .statesize = sizeof(struct caam_export_state),
4325                         },
4326                 },
4327                 .alg_type = OP_ALG_ALGSEL_SHA1,
4328         }, {
4329                 .name = "sha224",
4330                 .driver_name = "sha224-caam-qi2",
4331                 .hmac_name = "hmac(sha224)",
4332                 .hmac_driver_name = "hmac-sha224-caam-qi2",
4333                 .blocksize = SHA224_BLOCK_SIZE,
4334                 .template_ahash = {
4335                         .init = ahash_init,
4336                         .update = ahash_update,
4337                         .final = ahash_final,
4338                         .finup = ahash_finup,
4339                         .digest = ahash_digest,
4340                         .export = ahash_export,
4341                         .import = ahash_import,
4342                         .setkey = ahash_setkey,
4343                         .halg = {
4344                                 .digestsize = SHA224_DIGEST_SIZE,
4345                                 .statesize = sizeof(struct caam_export_state),
4346                         },
4347                 },
4348                 .alg_type = OP_ALG_ALGSEL_SHA224,
4349         }, {
4350                 .name = "sha256",
4351                 .driver_name = "sha256-caam-qi2",
4352                 .hmac_name = "hmac(sha256)",
4353                 .hmac_driver_name = "hmac-sha256-caam-qi2",
4354                 .blocksize = SHA256_BLOCK_SIZE,
4355                 .template_ahash = {
4356                         .init = ahash_init,
4357                         .update = ahash_update,
4358                         .final = ahash_final,
4359                         .finup = ahash_finup,
4360                         .digest = ahash_digest,
4361                         .export = ahash_export,
4362                         .import = ahash_import,
4363                         .setkey = ahash_setkey,
4364                         .halg = {
4365                                 .digestsize = SHA256_DIGEST_SIZE,
4366                                 .statesize = sizeof(struct caam_export_state),
4367                         },
4368                 },
4369                 .alg_type = OP_ALG_ALGSEL_SHA256,
4370         }, {
4371                 .name = "sha384",
4372                 .driver_name = "sha384-caam-qi2",
4373                 .hmac_name = "hmac(sha384)",
4374                 .hmac_driver_name = "hmac-sha384-caam-qi2",
4375                 .blocksize = SHA384_BLOCK_SIZE,
4376                 .template_ahash = {
4377                         .init = ahash_init,
4378                         .update = ahash_update,
4379                         .final = ahash_final,
4380                         .finup = ahash_finup,
4381                         .digest = ahash_digest,
4382                         .export = ahash_export,
4383                         .import = ahash_import,
4384                         .setkey = ahash_setkey,
4385                         .halg = {
4386                                 .digestsize = SHA384_DIGEST_SIZE,
4387                                 .statesize = sizeof(struct caam_export_state),
4388                         },
4389                 },
4390                 .alg_type = OP_ALG_ALGSEL_SHA384,
4391         }, {
4392                 .name = "sha512",
4393                 .driver_name = "sha512-caam-qi2",
4394                 .hmac_name = "hmac(sha512)",
4395                 .hmac_driver_name = "hmac-sha512-caam-qi2",
4396                 .blocksize = SHA512_BLOCK_SIZE,
4397                 .template_ahash = {
4398                         .init = ahash_init,
4399                         .update = ahash_update,
4400                         .final = ahash_final,
4401                         .finup = ahash_finup,
4402                         .digest = ahash_digest,
4403                         .export = ahash_export,
4404                         .import = ahash_import,
4405                         .setkey = ahash_setkey,
4406                         .halg = {
4407                                 .digestsize = SHA512_DIGEST_SIZE,
4408                                 .statesize = sizeof(struct caam_export_state),
4409                         },
4410                 },
4411                 .alg_type = OP_ALG_ALGSEL_SHA512,
4412         }, {
4413                 .name = "md5",
4414                 .driver_name = "md5-caam-qi2",
4415                 .hmac_name = "hmac(md5)",
4416                 .hmac_driver_name = "hmac-md5-caam-qi2",
4417                 .blocksize = MD5_BLOCK_WORDS * 4,
4418                 .template_ahash = {
4419                         .init = ahash_init,
4420                         .update = ahash_update,
4421                         .final = ahash_final,
4422                         .finup = ahash_finup,
4423                         .digest = ahash_digest,
4424                         .export = ahash_export,
4425                         .import = ahash_import,
4426                         .setkey = ahash_setkey,
4427                         .halg = {
4428                                 .digestsize = MD5_DIGEST_SIZE,
4429                                 .statesize = sizeof(struct caam_export_state),
4430                         },
4431                 },
4432                 .alg_type = OP_ALG_ALGSEL_MD5,
4433         }
4434 };
4435
4436 struct caam_hash_alg {
4437         struct list_head entry;
4438         struct device *dev;
4439         int alg_type;
4440         struct ahash_alg ahash_alg;
4441 };
4442
4443 static int caam_hash_cra_init(struct crypto_tfm *tfm)
4444 {
4445         struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
4446         struct crypto_alg *base = tfm->__crt_alg;
4447         struct hash_alg_common *halg =
4448                  container_of(base, struct hash_alg_common, base);
4449         struct ahash_alg *alg =
4450                  container_of(halg, struct ahash_alg, halg);
4451         struct caam_hash_alg *caam_hash =
4452                  container_of(alg, struct caam_hash_alg, ahash_alg);
4453         struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
4454         /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
4455         static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
4456                                          HASH_MSG_LEN + SHA1_DIGEST_SIZE,
4457                                          HASH_MSG_LEN + 32,
4458                                          HASH_MSG_LEN + SHA256_DIGEST_SIZE,
4459                                          HASH_MSG_LEN + 64,
4460                                          HASH_MSG_LEN + SHA512_DIGEST_SIZE };
4461         dma_addr_t dma_addr;
4462         int i;
4463
4464         ctx->dev = caam_hash->dev;
4465
4466         if (alg->setkey) {
4467                 ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key,
4468                                                           ARRAY_SIZE(ctx->key),
4469                                                           DMA_TO_DEVICE,
4470                                                           DMA_ATTR_SKIP_CPU_SYNC);
4471                 if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) {
4472                         dev_err(ctx->dev, "unable to map key\n");
4473                         return -ENOMEM;
4474                 }
4475         }
4476
4477         dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc),
4478                                         DMA_BIDIRECTIONAL,
4479                                         DMA_ATTR_SKIP_CPU_SYNC);
4480         if (dma_mapping_error(ctx->dev, dma_addr)) {
4481                 dev_err(ctx->dev, "unable to map shared descriptors\n");
4482                 if (ctx->adata.key_dma)
4483                         dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4484                                                ARRAY_SIZE(ctx->key),
4485                                                DMA_TO_DEVICE,
4486                                                DMA_ATTR_SKIP_CPU_SYNC);
4487                 return -ENOMEM;
4488         }
4489
4490         for (i = 0; i < HASH_NUM_OP; i++)
4491                 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
4492
4493         /* copy descriptor header template value */
4494         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
4495
4496         ctx->ctx_len = runninglen[(ctx->adata.algtype &
4497                                    OP_ALG_ALGSEL_SUBMASK) >>
4498                                   OP_ALG_ALGSEL_SHIFT];
4499
4500         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
4501                                  sizeof(struct caam_hash_state));
4502
4503         /*
4504          * For keyed hash algorithms shared descriptors
4505          * will be created later in setkey() callback
4506          */
4507         return alg->setkey ? 0 : ahash_set_sh_desc(ahash);
4508 }
4509
4510 static void caam_hash_cra_exit(struct crypto_tfm *tfm)
4511 {
4512         struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
4513
4514         dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc),
4515                                DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);
4516         if (ctx->adata.key_dma)
4517                 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4518                                        ARRAY_SIZE(ctx->key), DMA_TO_DEVICE,
4519                                        DMA_ATTR_SKIP_CPU_SYNC);
4520 }
4521
4522 static struct caam_hash_alg *caam_hash_alloc(struct device *dev,
4523         struct caam_hash_template *template, bool keyed)
4524 {
4525         struct caam_hash_alg *t_alg;
4526         struct ahash_alg *halg;
4527         struct crypto_alg *alg;
4528
4529         t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
4530         if (!t_alg)
4531                 return ERR_PTR(-ENOMEM);
4532
4533         t_alg->ahash_alg = template->template_ahash;
4534         halg = &t_alg->ahash_alg;
4535         alg = &halg->halg.base;
4536
4537         if (keyed) {
4538                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4539                          template->hmac_name);
4540                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4541                          template->hmac_driver_name);
4542         } else {
4543                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4544                          template->name);
4545                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4546                          template->driver_name);
4547                 t_alg->ahash_alg.setkey = NULL;
4548         }
4549         alg->cra_module = THIS_MODULE;
4550         alg->cra_init = caam_hash_cra_init;
4551         alg->cra_exit = caam_hash_cra_exit;
4552         alg->cra_ctxsize = sizeof(struct caam_hash_ctx);
4553         alg->cra_priority = CAAM_CRA_PRIORITY;
4554         alg->cra_blocksize = template->blocksize;
4555         alg->cra_alignmask = 0;
4556         alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY;
4557
4558         t_alg->alg_type = template->alg_type;
4559         t_alg->dev = dev;
4560
4561         return t_alg;
4562 }
4563
4564 static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx)
4565 {
4566         struct dpaa2_caam_priv_per_cpu *ppriv;
4567
4568         ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx);
4569         napi_schedule_irqoff(&ppriv->napi);
4570 }
4571
4572 static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv)
4573 {
4574         struct device *dev = priv->dev;
4575         struct dpaa2_io_notification_ctx *nctx;
4576         struct dpaa2_caam_priv_per_cpu *ppriv;
4577         int err, i = 0, cpu;
4578
4579         for_each_online_cpu(cpu) {
4580                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4581                 ppriv->priv = priv;
4582                 nctx = &ppriv->nctx;
4583                 nctx->is_cdan = 0;
4584                 nctx->id = ppriv->rsp_fqid;
4585                 nctx->desired_cpu = cpu;
4586                 nctx->cb = dpaa2_caam_fqdan_cb;
4587
4588                 /* Register notification callbacks */
4589                 ppriv->dpio = dpaa2_io_service_select(cpu);
4590                 err = dpaa2_io_service_register(ppriv->dpio, nctx, dev);
4591                 if (unlikely(err)) {
4592                         dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu);
4593                         nctx->cb = NULL;
4594                         /*
4595                          * If no affine DPIO for this core, there's probably
4596                          * none available for next cores either. Signal we want
4597                          * to retry later, in case the DPIO devices weren't
4598                          * probed yet.
4599                          */
4600                         err = -EPROBE_DEFER;
4601                         goto err;
4602                 }
4603
4604                 ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE,
4605                                                      dev);
4606                 if (unlikely(!ppriv->store)) {
4607                         dev_err(dev, "dpaa2_io_store_create() failed\n");
4608                         err = -ENOMEM;
4609                         goto err;
4610                 }
4611
4612                 if (++i == priv->num_pairs)
4613                         break;
4614         }
4615
4616         return 0;
4617
4618 err:
4619         for_each_online_cpu(cpu) {
4620                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4621                 if (!ppriv->nctx.cb)
4622                         break;
4623                 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev);
4624         }
4625
4626         for_each_online_cpu(cpu) {
4627                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4628                 if (!ppriv->store)
4629                         break;
4630                 dpaa2_io_store_destroy(ppriv->store);
4631         }
4632
4633         return err;
4634 }
4635
4636 static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv)
4637 {
4638         struct dpaa2_caam_priv_per_cpu *ppriv;
4639         int i = 0, cpu;
4640
4641         for_each_online_cpu(cpu) {
4642                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4643                 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx,
4644                                             priv->dev);
4645                 dpaa2_io_store_destroy(ppriv->store);
4646
4647                 if (++i == priv->num_pairs)
4648                         return;
4649         }
4650 }
4651
4652 static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv)
4653 {
4654         struct dpseci_rx_queue_cfg rx_queue_cfg;
4655         struct device *dev = priv->dev;
4656         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4657         struct dpaa2_caam_priv_per_cpu *ppriv;
4658         int err = 0, i = 0, cpu;
4659
4660         /* Configure Rx queues */
4661         for_each_online_cpu(cpu) {
4662                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4663
4664                 rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST |
4665                                        DPSECI_QUEUE_OPT_USER_CTX;
4666                 rx_queue_cfg.order_preservation_en = 0;
4667                 rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO;
4668                 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id;
4669                 /*
4670                  * Rx priority (WQ) doesn't really matter, since we use
4671                  * pull mode, i.e. volatile dequeues from specific FQs
4672                  */
4673                 rx_queue_cfg.dest_cfg.priority = 0;
4674                 rx_queue_cfg.user_ctx = ppriv->nctx.qman64;
4675
4676                 err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
4677                                           &rx_queue_cfg);
4678                 if (err) {
4679                         dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n",
4680                                 err);
4681                         return err;
4682                 }
4683
4684                 if (++i == priv->num_pairs)
4685                         break;
4686         }
4687
4688         return err;
4689 }
4690
4691 static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv)
4692 {
4693         struct device *dev = priv->dev;
4694
4695         if (!priv->cscn_mem)
4696                 return;
4697
4698         dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4699         kfree(priv->cscn_mem);
4700 }
4701
4702 static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv)
4703 {
4704         struct device *dev = priv->dev;
4705         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4706         int err;
4707
4708         if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) {
4709                 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle);
4710                 if (err)
4711                         dev_err(dev, "dpseci_reset() failed\n");
4712         }
4713
4714         dpaa2_dpseci_congestion_free(priv);
4715         dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
4716 }
4717
4718 static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv,
4719                                   const struct dpaa2_fd *fd)
4720 {
4721         struct caam_request *req;
4722         u32 fd_err;
4723
4724         if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) {
4725                 dev_err(priv->dev, "Only Frame List FD format is supported!\n");
4726                 return;
4727         }
4728
4729         fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK;
4730         if (unlikely(fd_err))
4731                 dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err);
4732
4733         /*
4734          * FD[ADDR] is guaranteed to be valid, irrespective of errors reported
4735          * in FD[ERR] or FD[FRC].
4736          */
4737         req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd));
4738         dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt),
4739                          DMA_BIDIRECTIONAL);
4740         req->cbk(req->ctx, dpaa2_fd_get_frc(fd));
4741 }
4742
4743 static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv)
4744 {
4745         int err;
4746
4747         /* Retry while portal is busy */
4748         do {
4749                 err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid,
4750                                                ppriv->store);
4751         } while (err == -EBUSY);
4752
4753         if (unlikely(err))
4754                 dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err);
4755
4756         return err;
4757 }
4758
4759 static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv)
4760 {
4761         struct dpaa2_dq *dq;
4762         int cleaned = 0, is_last;
4763
4764         do {
4765                 dq = dpaa2_io_store_next(ppriv->store, &is_last);
4766                 if (unlikely(!dq)) {
4767                         if (unlikely(!is_last)) {
4768                                 dev_dbg(ppriv->priv->dev,
4769                                         "FQ %d returned no valid frames\n",
4770                                         ppriv->rsp_fqid);
4771                                 /*
4772                                  * MUST retry until we get some sort of
4773                                  * valid response token (be it "empty dequeue"
4774                                  * or a valid frame).
4775                                  */
4776                                 continue;
4777                         }
4778                         break;
4779                 }
4780
4781                 /* Process FD */
4782                 dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq));
4783                 cleaned++;
4784         } while (!is_last);
4785
4786         return cleaned;
4787 }
4788
4789 static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget)
4790 {
4791         struct dpaa2_caam_priv_per_cpu *ppriv;
4792         struct dpaa2_caam_priv *priv;
4793         int err, cleaned = 0, store_cleaned;
4794
4795         ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi);
4796         priv = ppriv->priv;
4797
4798         if (unlikely(dpaa2_caam_pull_fq(ppriv)))
4799                 return 0;
4800
4801         do {
4802                 store_cleaned = dpaa2_caam_store_consume(ppriv);
4803                 cleaned += store_cleaned;
4804
4805                 if (store_cleaned == 0 ||
4806                     cleaned > budget - DPAA2_CAAM_STORE_SIZE)
4807                         break;
4808
4809                 /* Try to dequeue some more */
4810                 err = dpaa2_caam_pull_fq(ppriv);
4811                 if (unlikely(err))
4812                         break;
4813         } while (1);
4814
4815         if (cleaned < budget) {
4816                 napi_complete_done(napi, cleaned);
4817                 err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx);
4818                 if (unlikely(err))
4819                         dev_err(priv->dev, "Notification rearm failed: %d\n",
4820                                 err);
4821         }
4822
4823         return cleaned;
4824 }
4825
4826 static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv,
4827                                          u16 token)
4828 {
4829         struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 };
4830         struct device *dev = priv->dev;
4831         int err;
4832
4833         /*
4834          * Congestion group feature supported starting with DPSECI API v5.1
4835          * and only when object has been created with this capability.
4836          */
4837         if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) ||
4838             !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG))
4839                 return 0;
4840
4841         priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN,
4842                                  GFP_KERNEL | GFP_DMA);
4843         if (!priv->cscn_mem)
4844                 return -ENOMEM;
4845
4846         priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN);
4847         priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned,
4848                                         DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4849         if (dma_mapping_error(dev, priv->cscn_dma)) {
4850                 dev_err(dev, "Error mapping CSCN memory area\n");
4851                 err = -ENOMEM;
4852                 goto err_dma_map;
4853         }
4854
4855         cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES;
4856         cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH;
4857         cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH;
4858         cong_notif_cfg.message_ctx = (uintptr_t)priv;
4859         cong_notif_cfg.message_iova = priv->cscn_dma;
4860         cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER |
4861                                         DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT |
4862                                         DPSECI_CGN_MODE_COHERENT_WRITE;
4863
4864         err = dpseci_set_congestion_notification(priv->mc_io, 0, token,
4865                                                  &cong_notif_cfg);
4866         if (err) {
4867                 dev_err(dev, "dpseci_set_congestion_notification failed\n");
4868                 goto err_set_cong;
4869         }
4870
4871         return 0;
4872
4873 err_set_cong:
4874         dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4875 err_dma_map:
4876         kfree(priv->cscn_mem);
4877
4878         return err;
4879 }
4880
4881 static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev)
4882 {
4883         struct device *dev = &ls_dev->dev;
4884         struct dpaa2_caam_priv *priv;
4885         struct dpaa2_caam_priv_per_cpu *ppriv;
4886         int err, cpu;
4887         u8 i;
4888
4889         priv = dev_get_drvdata(dev);
4890
4891         priv->dev = dev;
4892         priv->dpsec_id = ls_dev->obj_desc.id;
4893
4894         /* Get a handle for the DPSECI this interface is associate with */
4895         err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle);
4896         if (err) {
4897                 dev_err(dev, "dpseci_open() failed: %d\n", err);
4898                 goto err_open;
4899         }
4900
4901         err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver,
4902                                      &priv->minor_ver);
4903         if (err) {
4904                 dev_err(dev, "dpseci_get_api_version() failed\n");
4905                 goto err_get_vers;
4906         }
4907
4908         dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver);
4909
4910         if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) {
4911                 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle);
4912                 if (err) {
4913                         dev_err(dev, "dpseci_reset() failed\n");
4914                         goto err_get_vers;
4915                 }
4916         }
4917
4918         err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle,
4919                                     &priv->dpseci_attr);
4920         if (err) {
4921                 dev_err(dev, "dpseci_get_attributes() failed\n");
4922                 goto err_get_vers;
4923         }
4924
4925         err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle,
4926                                   &priv->sec_attr);
4927         if (err) {
4928                 dev_err(dev, "dpseci_get_sec_attr() failed\n");
4929                 goto err_get_vers;
4930         }
4931
4932         err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle);
4933         if (err) {
4934                 dev_err(dev, "setup_congestion() failed\n");
4935                 goto err_get_vers;
4936         }
4937
4938         priv->num_pairs = min(priv->dpseci_attr.num_rx_queues,
4939                               priv->dpseci_attr.num_tx_queues);
4940         if (priv->num_pairs > num_online_cpus()) {
4941                 dev_warn(dev, "%d queues won't be used\n",
4942                          priv->num_pairs - num_online_cpus());
4943                 priv->num_pairs = num_online_cpus();
4944         }
4945
4946         for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) {
4947                 err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
4948                                           &priv->rx_queue_attr[i]);
4949                 if (err) {
4950                         dev_err(dev, "dpseci_get_rx_queue() failed\n");
4951                         goto err_get_rx_queue;
4952                 }
4953         }
4954
4955         for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) {
4956                 err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
4957                                           &priv->tx_queue_attr[i]);
4958                 if (err) {
4959                         dev_err(dev, "dpseci_get_tx_queue() failed\n");
4960                         goto err_get_rx_queue;
4961                 }
4962         }
4963
4964         i = 0;
4965         for_each_online_cpu(cpu) {
4966                 u8 j;
4967
4968                 j = i % priv->num_pairs;
4969
4970                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4971                 ppriv->req_fqid = priv->tx_queue_attr[j].fqid;
4972
4973                 /*
4974                  * Allow all cores to enqueue, while only some of them
4975                  * will take part in dequeuing.
4976                  */
4977                 if (++i > priv->num_pairs)
4978                         continue;
4979
4980                 ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid;
4981                 ppriv->prio = j;
4982
4983                 dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j,
4984                         priv->rx_queue_attr[j].fqid,
4985                         priv->tx_queue_attr[j].fqid);
4986
4987                 ppriv->net_dev.dev = *dev;
4988                 INIT_LIST_HEAD(&ppriv->net_dev.napi_list);
4989                 netif_napi_add(&ppriv->net_dev, &ppriv->napi, dpaa2_dpseci_poll,
4990                                DPAA2_CAAM_NAPI_WEIGHT);
4991         }
4992
4993         return 0;
4994
4995 err_get_rx_queue:
4996         dpaa2_dpseci_congestion_free(priv);
4997 err_get_vers:
4998         dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
4999 err_open:
5000         return err;
5001 }
5002
5003 static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv)
5004 {
5005         struct device *dev = priv->dev;
5006         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
5007         struct dpaa2_caam_priv_per_cpu *ppriv;
5008         int i;
5009
5010         for (i = 0; i < priv->num_pairs; i++) {
5011                 ppriv = per_cpu_ptr(priv->ppriv, i);
5012                 napi_enable(&ppriv->napi);
5013         }
5014
5015         return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle);
5016 }
5017
5018 static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv)
5019 {
5020         struct device *dev = priv->dev;
5021         struct dpaa2_caam_priv_per_cpu *ppriv;
5022         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
5023         int i, err = 0, enabled;
5024
5025         err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle);
5026         if (err) {
5027                 dev_err(dev, "dpseci_disable() failed\n");
5028                 return err;
5029         }
5030
5031         err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled);
5032         if (err) {
5033                 dev_err(dev, "dpseci_is_enabled() failed\n");
5034                 return err;
5035         }
5036
5037         dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true");
5038
5039         for (i = 0; i < priv->num_pairs; i++) {
5040                 ppriv = per_cpu_ptr(priv->ppriv, i);
5041                 napi_disable(&ppriv->napi);
5042                 netif_napi_del(&ppriv->napi);
5043         }
5044
5045         return 0;
5046 }
5047
5048 static struct list_head hash_list;
5049
5050 static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev)
5051 {
5052         struct device *dev;
5053         struct dpaa2_caam_priv *priv;
5054         int i, err = 0;
5055         bool registered = false;
5056
5057         /*
5058          * There is no way to get CAAM endianness - there is no direct register
5059          * space access and MC f/w does not provide this attribute.
5060          * All DPAA2-based SoCs have little endian CAAM, thus hard-code this
5061          * property.
5062          */
5063         caam_little_end = true;
5064
5065         caam_imx = false;
5066
5067         dev = &dpseci_dev->dev;
5068
5069         priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
5070         if (!priv)
5071                 return -ENOMEM;
5072
5073         dev_set_drvdata(dev, priv);
5074
5075         priv->domain = iommu_get_domain_for_dev(dev);
5076
5077         qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE,
5078                                      0, SLAB_CACHE_DMA, NULL);
5079         if (!qi_cache) {
5080                 dev_err(dev, "Can't allocate SEC cache\n");
5081                 return -ENOMEM;
5082         }
5083
5084         err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49));
5085         if (err) {
5086                 dev_err(dev, "dma_set_mask_and_coherent() failed\n");
5087                 goto err_dma_mask;
5088         }
5089
5090         /* Obtain a MC portal */
5091         err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io);
5092         if (err) {
5093                 if (err == -ENXIO)
5094                         err = -EPROBE_DEFER;
5095                 else
5096                         dev_err(dev, "MC portal allocation failed\n");
5097
5098                 goto err_dma_mask;
5099         }
5100
5101         priv->ppriv = alloc_percpu(*priv->ppriv);
5102         if (!priv->ppriv) {
5103                 dev_err(dev, "alloc_percpu() failed\n");
5104                 err = -ENOMEM;
5105                 goto err_alloc_ppriv;
5106         }
5107
5108         /* DPSECI initialization */
5109         err = dpaa2_dpseci_setup(dpseci_dev);
5110         if (err) {
5111                 dev_err(dev, "dpaa2_dpseci_setup() failed\n");
5112                 goto err_dpseci_setup;
5113         }
5114
5115         /* DPIO */
5116         err = dpaa2_dpseci_dpio_setup(priv);
5117         if (err) {
5118                 if (err != -EPROBE_DEFER)
5119                         dev_err(dev, "dpaa2_dpseci_dpio_setup() failed\n");
5120                 goto err_dpio_setup;
5121         }
5122
5123         /* DPSECI binding to DPIO */
5124         err = dpaa2_dpseci_bind(priv);
5125         if (err) {
5126                 dev_err(dev, "dpaa2_dpseci_bind() failed\n");
5127                 goto err_bind;
5128         }
5129
5130         /* DPSECI enable */
5131         err = dpaa2_dpseci_enable(priv);
5132         if (err) {
5133                 dev_err(dev, "dpaa2_dpseci_enable() failed\n");
5134                 goto err_bind;
5135         }
5136
5137         dpaa2_dpseci_debugfs_init(priv);
5138
5139         /* register crypto algorithms the device supports */
5140         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5141                 struct caam_skcipher_alg *t_alg = driver_algs + i;
5142                 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
5143
5144                 /* Skip DES algorithms if not supported by device */
5145                 if (!priv->sec_attr.des_acc_num &&
5146                     (alg_sel == OP_ALG_ALGSEL_3DES ||
5147                      alg_sel == OP_ALG_ALGSEL_DES))
5148                         continue;
5149
5150                 /* Skip AES algorithms if not supported by device */
5151                 if (!priv->sec_attr.aes_acc_num &&
5152                     alg_sel == OP_ALG_ALGSEL_AES)
5153                         continue;
5154
5155                 /* Skip CHACHA20 algorithms if not supported by device */
5156                 if (alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5157                     !priv->sec_attr.ccha_acc_num)
5158                         continue;
5159
5160                 t_alg->caam.dev = dev;
5161                 caam_skcipher_alg_init(t_alg);
5162
5163                 err = crypto_register_skcipher(&t_alg->skcipher);
5164                 if (err) {
5165                         dev_warn(dev, "%s alg registration failed: %d\n",
5166                                  t_alg->skcipher.base.cra_driver_name, err);
5167                         continue;
5168                 }
5169
5170                 t_alg->registered = true;
5171                 registered = true;
5172         }
5173
5174         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5175                 struct caam_aead_alg *t_alg = driver_aeads + i;
5176                 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
5177                                  OP_ALG_ALGSEL_MASK;
5178                 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
5179                                  OP_ALG_ALGSEL_MASK;
5180
5181                 /* Skip DES algorithms if not supported by device */
5182                 if (!priv->sec_attr.des_acc_num &&
5183                     (c1_alg_sel == OP_ALG_ALGSEL_3DES ||
5184                      c1_alg_sel == OP_ALG_ALGSEL_DES))
5185                         continue;
5186
5187                 /* Skip AES algorithms if not supported by device */
5188                 if (!priv->sec_attr.aes_acc_num &&
5189                     c1_alg_sel == OP_ALG_ALGSEL_AES)
5190                         continue;
5191
5192                 /* Skip CHACHA20 algorithms if not supported by device */
5193                 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5194                     !priv->sec_attr.ccha_acc_num)
5195                         continue;
5196
5197                 /* Skip POLY1305 algorithms if not supported by device */
5198                 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 &&
5199                     !priv->sec_attr.ptha_acc_num)
5200                         continue;
5201
5202                 /*
5203                  * Skip algorithms requiring message digests
5204                  * if MD not supported by device.
5205                  */
5206                 if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 &&
5207                     !priv->sec_attr.md_acc_num)
5208                         continue;
5209
5210                 t_alg->caam.dev = dev;
5211                 caam_aead_alg_init(t_alg);
5212
5213                 err = crypto_register_aead(&t_alg->aead);
5214                 if (err) {
5215                         dev_warn(dev, "%s alg registration failed: %d\n",
5216                                  t_alg->aead.base.cra_driver_name, err);
5217                         continue;
5218                 }
5219
5220                 t_alg->registered = true;
5221                 registered = true;
5222         }
5223         if (registered)
5224                 dev_info(dev, "algorithms registered in /proc/crypto\n");
5225
5226         /* register hash algorithms the device supports */
5227         INIT_LIST_HEAD(&hash_list);
5228
5229         /*
5230          * Skip registration of any hashing algorithms if MD block
5231          * is not present.
5232          */
5233         if (!priv->sec_attr.md_acc_num)
5234                 return 0;
5235
5236         for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
5237                 struct caam_hash_alg *t_alg;
5238                 struct caam_hash_template *alg = driver_hash + i;
5239
5240                 /* register hmac version */
5241                 t_alg = caam_hash_alloc(dev, alg, true);
5242                 if (IS_ERR(t_alg)) {
5243                         err = PTR_ERR(t_alg);
5244                         dev_warn(dev, "%s hash alg allocation failed: %d\n",
5245                                  alg->hmac_driver_name, err);
5246                         continue;
5247                 }
5248
5249                 err = crypto_register_ahash(&t_alg->ahash_alg);
5250                 if (err) {
5251                         dev_warn(dev, "%s alg registration failed: %d\n",
5252                                  t_alg->ahash_alg.halg.base.cra_driver_name,
5253                                  err);
5254                         kfree(t_alg);
5255                 } else {
5256                         list_add_tail(&t_alg->entry, &hash_list);
5257                 }
5258
5259                 /* register unkeyed version */
5260                 t_alg = caam_hash_alloc(dev, alg, false);
5261                 if (IS_ERR(t_alg)) {
5262                         err = PTR_ERR(t_alg);
5263                         dev_warn(dev, "%s alg allocation failed: %d\n",
5264                                  alg->driver_name, err);
5265                         continue;
5266                 }
5267
5268                 err = crypto_register_ahash(&t_alg->ahash_alg);
5269                 if (err) {
5270                         dev_warn(dev, "%s alg registration failed: %d\n",
5271                                  t_alg->ahash_alg.halg.base.cra_driver_name,
5272                                  err);
5273                         kfree(t_alg);
5274                 } else {
5275                         list_add_tail(&t_alg->entry, &hash_list);
5276                 }
5277         }
5278         if (!list_empty(&hash_list))
5279                 dev_info(dev, "hash algorithms registered in /proc/crypto\n");
5280
5281         return err;
5282
5283 err_bind:
5284         dpaa2_dpseci_dpio_free(priv);
5285 err_dpio_setup:
5286         dpaa2_dpseci_free(priv);
5287 err_dpseci_setup:
5288         free_percpu(priv->ppriv);
5289 err_alloc_ppriv:
5290         fsl_mc_portal_free(priv->mc_io);
5291 err_dma_mask:
5292         kmem_cache_destroy(qi_cache);
5293
5294         return err;
5295 }
5296
5297 static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev)
5298 {
5299         struct device *dev;
5300         struct dpaa2_caam_priv *priv;
5301         int i;
5302
5303         dev = &ls_dev->dev;
5304         priv = dev_get_drvdata(dev);
5305
5306         dpaa2_dpseci_debugfs_exit(priv);
5307
5308         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5309                 struct caam_aead_alg *t_alg = driver_aeads + i;
5310
5311                 if (t_alg->registered)
5312                         crypto_unregister_aead(&t_alg->aead);
5313         }
5314
5315         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5316                 struct caam_skcipher_alg *t_alg = driver_algs + i;
5317
5318                 if (t_alg->registered)
5319                         crypto_unregister_skcipher(&t_alg->skcipher);
5320         }
5321
5322         if (hash_list.next) {
5323                 struct caam_hash_alg *t_hash_alg, *p;
5324
5325                 list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) {
5326                         crypto_unregister_ahash(&t_hash_alg->ahash_alg);
5327                         list_del(&t_hash_alg->entry);
5328                         kfree(t_hash_alg);
5329                 }
5330         }
5331
5332         dpaa2_dpseci_disable(priv);
5333         dpaa2_dpseci_dpio_free(priv);
5334         dpaa2_dpseci_free(priv);
5335         free_percpu(priv->ppriv);
5336         fsl_mc_portal_free(priv->mc_io);
5337         kmem_cache_destroy(qi_cache);
5338
5339         return 0;
5340 }
5341
5342 int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req)
5343 {
5344         struct dpaa2_fd fd;
5345         struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
5346         struct dpaa2_caam_priv_per_cpu *ppriv;
5347         int err = 0, i;
5348
5349         if (IS_ERR(req))
5350                 return PTR_ERR(req);
5351
5352         if (priv->cscn_mem) {
5353                 dma_sync_single_for_cpu(priv->dev, priv->cscn_dma,
5354                                         DPAA2_CSCN_SIZE,
5355                                         DMA_FROM_DEVICE);
5356                 if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) {
5357                         dev_dbg_ratelimited(dev, "Dropping request\n");
5358                         return -EBUSY;
5359                 }
5360         }
5361
5362         dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma);
5363
5364         req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt),
5365                                          DMA_BIDIRECTIONAL);
5366         if (dma_mapping_error(dev, req->fd_flt_dma)) {
5367                 dev_err(dev, "DMA mapping error for QI enqueue request\n");
5368                 goto err_out;
5369         }
5370
5371         memset(&fd, 0, sizeof(fd));
5372         dpaa2_fd_set_format(&fd, dpaa2_fd_list);
5373         dpaa2_fd_set_addr(&fd, req->fd_flt_dma);
5374         dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1]));
5375         dpaa2_fd_set_flc(&fd, req->flc_dma);
5376
5377         ppriv = this_cpu_ptr(priv->ppriv);
5378         for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) {
5379                 err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid,
5380                                                   &fd);
5381                 if (err != -EBUSY)
5382                         break;
5383
5384                 cpu_relax();
5385         }
5386
5387         if (unlikely(err)) {
5388                 dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err);
5389                 goto err_out;
5390         }
5391
5392         return -EINPROGRESS;
5393
5394 err_out:
5395         dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt),
5396                          DMA_BIDIRECTIONAL);
5397         return -EIO;
5398 }
5399 EXPORT_SYMBOL(dpaa2_caam_enqueue);
5400
5401 static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = {
5402         {
5403                 .vendor = FSL_MC_VENDOR_FREESCALE,
5404                 .obj_type = "dpseci",
5405         },
5406         { .vendor = 0x0 }
5407 };
5408
5409 static struct fsl_mc_driver dpaa2_caam_driver = {
5410         .driver = {
5411                 .name           = KBUILD_MODNAME,
5412                 .owner          = THIS_MODULE,
5413         },
5414         .probe          = dpaa2_caam_probe,
5415         .remove         = dpaa2_caam_remove,
5416         .match_id_table = dpaa2_caam_match_id_table
5417 };
5418
5419 MODULE_LICENSE("Dual BSD/GPL");
5420 MODULE_AUTHOR("Freescale Semiconductor, Inc");
5421 MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver");
5422
5423 module_fsl_mc_driver(dpaa2_caam_driver);