drivers/crypto/caam/caamalg_qi2.c

   1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
   2 /*
   3  * Copyright 2015-2016 Freescale Semiconductor Inc.
   4  * Copyright 2017-2019 NXP
   5  */
   6
   7 #include "compat.h"
   8 #include "regs.h"
   9 #include "caamalg_qi2.h"
  10 #include "dpseci_cmd.h"
  11 #include "desc_constr.h"
  12 #include "error.h"
  13 #include "sg_sw_sec4.h"
  14 #include "sg_sw_qm2.h"
  15 #include "key_gen.h"
  16 #include "caamalg_desc.h"
  17 #include "caamhash_desc.h"
  18 #include "dpseci-debugfs.h"
  19 #include <linux/dma-mapping.h>
  20 #include <linux/fsl/mc.h>
  21 #include <linux/kernel.h>
  22 #include <soc/fsl/dpaa2-io.h>
  23 #include <soc/fsl/dpaa2-fd.h>
  24 #include <crypto/xts.h>
  25 #include <asm/unaligned.h>
  26
  27 #define CAAM_CRA_PRIORITY       2000
  28
  29 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  30 #define CAAM_MAX_KEY_SIZE       (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \
  31                                  SHA512_DIGEST_SIZE * 2)
  32
  33 /*
  34  * This is a cache of buffers, from which the users of CAAM QI driver
  35  * can allocate short buffers. It's speedier than doing kmalloc on the hotpath.
  36  * NOTE: A more elegant solution would be to have some headroom in the frames
  37  *       being processed. This can be added by the dpaa2-eth driver. This would
  38  *       pose a problem for userspace application processing which cannot
  39  *       know of this limitation. So for now, this will work.
  40  * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here
  41  */
  42 static struct kmem_cache *qi_cache;
  43
  44 struct caam_alg_entry {
  45         struct device *dev;
  46         int class1_alg_type;
  47         int class2_alg_type;
  48         bool rfc3686;
  49         bool geniv;
  50         bool nodkp;
  51 };
  52
  53 struct caam_aead_alg {
  54         struct aead_alg aead;
  55         struct caam_alg_entry caam;
  56         bool registered;
  57 };
  58
  59 struct caam_skcipher_alg {
  60         struct skcipher_alg skcipher;
  61         struct caam_alg_entry caam;
  62         bool registered;
  63 };
  64
  65 /**
  66  * struct caam_ctx - per-session context
  67  * @flc: Flow Contexts array
  68  * @key:  [authentication key], encryption key
  69  * @flc_dma: I/O virtual addresses of the Flow Contexts
  70  * @key_dma: I/O virtual address of the key
  71  * @dir: DMA direction for mapping key and Flow Contexts
  72  * @dev: dpseci device
  73  * @adata: authentication algorithm details
  74  * @cdata: encryption algorithm details
  75  * @authsize: authentication tag (a.k.a. ICV / MAC) size
  76  * @xts_key_fallback: true if fallback tfm needs to be used due
  77  *                    to unsupported xts key lengths
  78  * @fallback: xts fallback tfm
  79  */
  80 struct caam_ctx {
  81         struct caam_flc flc[NUM_OP];
  82         u8 key[CAAM_MAX_KEY_SIZE];
  83         dma_addr_t flc_dma[NUM_OP];
  84         dma_addr_t key_dma;
  85         enum dma_data_direction dir;
  86         struct device *dev;
  87         struct alginfo adata;
  88         struct alginfo cdata;
  89         unsigned int authsize;
  90         bool xts_key_fallback;
  91         struct crypto_skcipher *fallback;
  92 };
  93
  94 static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv,
  95                                      dma_addr_t iova_addr)
  96 {
  97         phys_addr_t phys_addr;
  98
  99         phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) :
 100                                    iova_addr;
 101
 102         return phys_to_virt(phys_addr);
 103 }
 104
 105 /*
 106  * qi_cache_zalloc - Allocate buffers from CAAM-QI cache
 107  *
 108  * Allocate data on the hotpath. Instead of using kzalloc, one can use the
 109  * services of the CAAM QI memory cache (backed by kmem_cache). The buffers
 110  * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for
 111  * hosting 16 SG entries.
 112  *
 113  * @flags - flags that would be used for the equivalent kmalloc(..) call
 114  *
 115  * Returns a pointer to a retrieved buffer on success or NULL on failure.
 116  */
 117 static inline void *qi_cache_zalloc(gfp_t flags)
 118 {
 119         return kmem_cache_zalloc(qi_cache, flags);
 120 }
 121
 122 /*
 123  * qi_cache_free - Frees buffers allocated from CAAM-QI cache
 124  *
 125  * @obj - buffer previously allocated by qi_cache_zalloc
 126  *
 127  * No checking is being done, the call is a passthrough call to
 128  * kmem_cache_free(...)
 129  */
 130 static inline void qi_cache_free(void *obj)
 131 {
 132         kmem_cache_free(qi_cache, obj);
 133 }
 134
 135 static struct caam_request *to_caam_req(struct crypto_async_request *areq)
 136 {
 137         switch (crypto_tfm_alg_type(areq->tfm)) {
 138         case CRYPTO_ALG_TYPE_SKCIPHER:
 139                 return skcipher_request_ctx_dma(skcipher_request_cast(areq));
 140         case CRYPTO_ALG_TYPE_AEAD:
 141                 return aead_request_ctx_dma(
 142                         container_of(areq, struct aead_request, base));
 143         case CRYPTO_ALG_TYPE_AHASH:
 144                 return ahash_request_ctx_dma(ahash_request_cast(areq));
 145         default:
 146                 return ERR_PTR(-EINVAL);
 147         }
 148 }
 149
 150 static void caam_unmap(struct device *dev, struct scatterlist *src,
 151                        struct scatterlist *dst, int src_nents,
 152                        int dst_nents, dma_addr_t iv_dma, int ivsize,
 153                        enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
 154                        int qm_sg_bytes)
 155 {
 156         if (dst != src) {
 157                 if (src_nents)
 158                         dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 159                 if (dst_nents)
 160                         dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
 161         } else {
 162                 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 163         }
 164
 165         if (iv_dma)
 166                 dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
 167
 168         if (qm_sg_bytes)
 169                 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
 170 }
 171
 172 static int aead_set_sh_desc(struct crypto_aead *aead)
 173 {
 174         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 175                                                  typeof(*alg), aead);
 176         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 177         unsigned int ivsize = crypto_aead_ivsize(aead);
 178         struct device *dev = ctx->dev;
 179         struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
 180         struct caam_flc *flc;
 181         u32 *desc;
 182         u32 ctx1_iv_off = 0;
 183         u32 *nonce = NULL;
 184         unsigned int data_len[2];
 185         u32 inl_mask;
 186         const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
 187                                OP_ALG_AAI_CTR_MOD128);
 188         const bool is_rfc3686 = alg->caam.rfc3686;
 189
 190         if (!ctx->cdata.keylen || !ctx->authsize)
 191                 return 0;
 192
 193         /*
 194          * AES-CTR needs to load IV in CONTEXT1 reg
 195          * at an offset of 128bits (16bytes)
 196          * CONTEXT1[255:128] = IV
 197          */
 198         if (ctr_mode)
 199                 ctx1_iv_off = 16;
 200
 201         /*
 202          * RFC3686 specific:
 203          *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 204          */
 205         if (is_rfc3686) {
 206                 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 207                 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
 208                                 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
 209         }
 210
 211         /*
 212          * In case |user key| > |derived key|, using DKP<imm,imm> would result
 213          * in invalid opcodes (last bytes of user key) in the resulting
 214          * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
 215          * addresses are needed.
 216          */
 217         ctx->adata.key_virt = ctx->key;
 218         ctx->adata.key_dma = ctx->key_dma;
 219
 220         ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 221         ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 222
 223         data_len[0] = ctx->adata.keylen_pad;
 224         data_len[1] = ctx->cdata.keylen;
 225
 226         /* aead_encrypt shared descriptor */
 227         if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN :
 228                                                  DESC_QI_AEAD_ENC_LEN) +
 229                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 230                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 231                               ARRAY_SIZE(data_len)) < 0)
 232                 return -EINVAL;
 233
 234         ctx->adata.key_inline = !!(inl_mask & 1);
 235         ctx->cdata.key_inline = !!(inl_mask & 2);
 236
 237         flc = &ctx->flc[ENCRYPT];
 238         desc = flc->sh_desc;
 239
 240         if (alg->caam.geniv)
 241                 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata,
 242                                           ivsize, ctx->authsize, is_rfc3686,
 243                                           nonce, ctx1_iv_off, true,
 244                                           priv->sec_attr.era);
 245         else
 246                 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata,
 247                                        ivsize, ctx->authsize, is_rfc3686, nonce,
 248                                        ctx1_iv_off, true, priv->sec_attr.era);
 249
 250         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 251         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 252                                    sizeof(flc->flc) + desc_bytes(desc),
 253                                    ctx->dir);
 254
 255         /* aead_decrypt shared descriptor */
 256         if (desc_inline_query(DESC_QI_AEAD_DEC_LEN +
 257                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 258                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 259                               ARRAY_SIZE(data_len)) < 0)
 260                 return -EINVAL;
 261
 262         ctx->adata.key_inline = !!(inl_mask & 1);
 263         ctx->cdata.key_inline = !!(inl_mask & 2);
 264
 265         flc = &ctx->flc[DECRYPT];
 266         desc = flc->sh_desc;
 267         cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata,
 268                                ivsize, ctx->authsize, alg->caam.geniv,
 269                                is_rfc3686, nonce, ctx1_iv_off, true,
 270                                priv->sec_attr.era);
 271         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 272         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 273                                    sizeof(flc->flc) + desc_bytes(desc),
 274                                    ctx->dir);
 275
 276         return 0;
 277 }
 278
 279 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 280 {
 281         struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
 282
 283         ctx->authsize = authsize;
 284         aead_set_sh_desc(authenc);
 285
 286         return 0;
 287 }
 288
 289 static int aead_setkey(struct crypto_aead *aead, const u8 *key,
 290                        unsigned int keylen)
 291 {
 292         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 293         struct device *dev = ctx->dev;
 294         struct crypto_authenc_keys keys;
 295
 296         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 297                 goto badkey;
 298
 299         dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n",
 300                 keys.authkeylen + keys.enckeylen, keys.enckeylen,
 301                 keys.authkeylen);
 302         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 303                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 304
 305         ctx->adata.keylen = keys.authkeylen;
 306         ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
 307                                               OP_ALG_ALGSEL_MASK);
 308
 309         if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
 310                 goto badkey;
 311
 312         memcpy(ctx->key, keys.authkey, keys.authkeylen);
 313         memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
 314         dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad +
 315                                    keys.enckeylen, ctx->dir);
 316         print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
 317                              DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 318                              ctx->adata.keylen_pad + keys.enckeylen, 1);
 319
 320         ctx->cdata.keylen = keys.enckeylen;
 321
 322         memzero_explicit(&keys, sizeof(keys));
 323         return aead_set_sh_desc(aead);
 324 badkey:
 325         memzero_explicit(&keys, sizeof(keys));
 326         return -EINVAL;
 327 }
 328
 329 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
 330                             unsigned int keylen)
 331 {
 332         struct crypto_authenc_keys keys;
 333         int err;
 334
 335         err = crypto_authenc_extractkeys(&keys, key, keylen);
 336         if (unlikely(err))
 337                 goto out;
 338
 339         err = -EINVAL;
 340         if (keys.enckeylen != DES3_EDE_KEY_SIZE)
 341                 goto out;
 342
 343         err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?:
 344               aead_setkey(aead, key, keylen);
 345
 346 out:
 347         memzero_explicit(&keys, sizeof(keys));
 348         return err;
 349 }
 350
 351 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
 352                                            bool encrypt)
 353 {
 354         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 355         struct caam_request *req_ctx = aead_request_ctx_dma(req);
 356         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
 357         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
 358         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 359         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 360                                                  typeof(*alg), aead);
 361         struct device *dev = ctx->dev;
 362         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 363                       GFP_KERNEL : GFP_ATOMIC;
 364         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
 365         int src_len, dst_len = 0;
 366         struct aead_edesc *edesc;
 367         dma_addr_t qm_sg_dma, iv_dma = 0;
 368         int ivsize = 0;
 369         unsigned int authsize = ctx->authsize;
 370         int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes;
 371         int in_len, out_len;
 372         struct dpaa2_sg_entry *sg_table;
 373
 374         /* allocate space for base edesc, link tables and IV */
 375         edesc = qi_cache_zalloc(flags);
 376         if (unlikely(!edesc)) {
 377                 dev_err(dev, "could not allocate extended descriptor\n");
 378                 return ERR_PTR(-ENOMEM);
 379         }
 380
 381         if (unlikely(req->dst != req->src)) {
 382                 src_len = req->assoclen + req->cryptlen;
 383                 dst_len = src_len + (encrypt ? authsize : (-authsize));
 384
 385                 src_nents = sg_nents_for_len(req->src, src_len);
 386                 if (unlikely(src_nents < 0)) {
 387                         dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
 388                                 src_len);
 389                         qi_cache_free(edesc);
 390                         return ERR_PTR(src_nents);
 391                 }
 392
 393                 dst_nents = sg_nents_for_len(req->dst, dst_len);
 394                 if (unlikely(dst_nents < 0)) {
 395                         dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
 396                                 dst_len);
 397                         qi_cache_free(edesc);
 398                         return ERR_PTR(dst_nents);
 399                 }
 400
 401                 if (src_nents) {
 402                         mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
 403                                                       DMA_TO_DEVICE);
 404                         if (unlikely(!mapped_src_nents)) {
 405                                 dev_err(dev, "unable to map source\n");
 406                                 qi_cache_free(edesc);
 407                                 return ERR_PTR(-ENOMEM);
 408                         }
 409                 } else {
 410                         mapped_src_nents = 0;
 411                 }
 412
 413                 if (dst_nents) {
 414                         mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
 415                                                       DMA_FROM_DEVICE);
 416                         if (unlikely(!mapped_dst_nents)) {
 417                                 dev_err(dev, "unable to map destination\n");
 418                                 dma_unmap_sg(dev, req->src, src_nents,
 419                                              DMA_TO_DEVICE);
 420                                 qi_cache_free(edesc);
 421                                 return ERR_PTR(-ENOMEM);
 422                         }
 423                 } else {
 424                         mapped_dst_nents = 0;
 425                 }
 426         } else {
 427                 src_len = req->assoclen + req->cryptlen +
 428                           (encrypt ? authsize : 0);
 429
 430                 src_nents = sg_nents_for_len(req->src, src_len);
 431                 if (unlikely(src_nents < 0)) {
 432                         dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
 433                                 src_len);
 434                         qi_cache_free(edesc);
 435                         return ERR_PTR(src_nents);
 436                 }
 437
 438                 mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
 439                                               DMA_BIDIRECTIONAL);
 440                 if (unlikely(!mapped_src_nents)) {
 441                         dev_err(dev, "unable to map source\n");
 442                         qi_cache_free(edesc);
 443                         return ERR_PTR(-ENOMEM);
 444                 }
 445         }
 446
 447         if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv)
 448                 ivsize = crypto_aead_ivsize(aead);
 449
 450         /*
 451          * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
 452          * Input is not contiguous.
 453          * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
 454          * the end of the table by allocating more S/G entries. Logic:
 455          * if (src != dst && output S/G)
 456          *      pad output S/G, if needed
 457          * else if (src == dst && S/G)
 458          *      overlapping S/Gs; pad one of them
 459          * else if (input S/G) ...
 460          *      pad input S/G, if needed
 461          */
 462         qm_sg_nents = 1 + !!ivsize + mapped_src_nents;
 463         if (mapped_dst_nents > 1)
 464                 qm_sg_nents += pad_sg_nents(mapped_dst_nents);
 465         else if ((req->src == req->dst) && (mapped_src_nents > 1))
 466                 qm_sg_nents = max(pad_sg_nents(qm_sg_nents),
 467                                   1 + !!ivsize +
 468                                   pad_sg_nents(mapped_src_nents));
 469         else
 470                 qm_sg_nents = pad_sg_nents(qm_sg_nents);
 471
 472         sg_table = &edesc->sgt[0];
 473         qm_sg_bytes = qm_sg_nents * sizeof(*sg_table);
 474         if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
 475                      CAAM_QI_MEMCACHE_SIZE)) {
 476                 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
 477                         qm_sg_nents, ivsize);
 478                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
 479                            0, DMA_NONE, 0, 0);
 480                 qi_cache_free(edesc);
 481                 return ERR_PTR(-ENOMEM);
 482         }
 483
 484         if (ivsize) {
 485                 u8 *iv = (u8 *)(sg_table + qm_sg_nents);
 486
 487                 /* Make sure IV is located in a DMAable area */
 488                 memcpy(iv, req->iv, ivsize);
 489
 490                 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
 491                 if (dma_mapping_error(dev, iv_dma)) {
 492                         dev_err(dev, "unable to map IV\n");
 493                         caam_unmap(dev, req->src, req->dst, src_nents,
 494                                    dst_nents, 0, 0, DMA_NONE, 0, 0);
 495                         qi_cache_free(edesc);
 496                         return ERR_PTR(-ENOMEM);
 497                 }
 498         }
 499
 500         edesc->src_nents = src_nents;
 501         edesc->dst_nents = dst_nents;
 502         edesc->iv_dma = iv_dma;
 503
 504         if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) ==
 505             OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE)
 506                 /*
 507                  * The associated data comes already with the IV but we need
 508                  * to skip it when we authenticate or encrypt...
 509                  */
 510                 edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize);
 511         else
 512                 edesc->assoclen = cpu_to_caam32(req->assoclen);
 513         edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4,
 514                                              DMA_TO_DEVICE);
 515         if (dma_mapping_error(dev, edesc->assoclen_dma)) {
 516                 dev_err(dev, "unable to map assoclen\n");
 517                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
 518                            iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
 519                 qi_cache_free(edesc);
 520                 return ERR_PTR(-ENOMEM);
 521         }
 522
 523         dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0);
 524         qm_sg_index++;
 525         if (ivsize) {
 526                 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
 527                 qm_sg_index++;
 528         }
 529         sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
 530         qm_sg_index += mapped_src_nents;
 531
 532         if (mapped_dst_nents > 1)
 533                 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
 534
 535         qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
 536         if (dma_mapping_error(dev, qm_sg_dma)) {
 537                 dev_err(dev, "unable to map S/G table\n");
 538                 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
 539                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
 540                            iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
 541                 qi_cache_free(edesc);
 542                 return ERR_PTR(-ENOMEM);
 543         }
 544
 545         edesc->qm_sg_dma = qm_sg_dma;
 546         edesc->qm_sg_bytes = qm_sg_bytes;
 547
 548         out_len = req->assoclen + req->cryptlen +
 549                   (encrypt ? ctx->authsize : (-ctx->authsize));
 550         in_len = 4 + ivsize + req->assoclen + req->cryptlen;
 551
 552         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
 553         dpaa2_fl_set_final(in_fle, true);
 554         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
 555         dpaa2_fl_set_addr(in_fle, qm_sg_dma);
 556         dpaa2_fl_set_len(in_fle, in_len);
 557
 558         if (req->dst == req->src) {
 559                 if (mapped_src_nents == 1) {
 560                         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
 561                         dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src));
 562                 } else {
 563                         dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
 564                         dpaa2_fl_set_addr(out_fle, qm_sg_dma +
 565                                           (1 + !!ivsize) * sizeof(*sg_table));
 566                 }
 567         } else if (!mapped_dst_nents) {
 568                 /*
 569                  * crypto engine requires the output entry to be present when
 570                  * "frame list" FD is used.
 571                  * Since engine does not support FMT=2'b11 (unused entry type),
 572                  * leaving out_fle zeroized is the best option.
 573                  */
 574                 goto skip_out_fle;
 575         } else if (mapped_dst_nents == 1) {
 576                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
 577                 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
 578         } else {
 579                 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
 580                 dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index *
 581                                   sizeof(*sg_table));
 582         }
 583
 584         dpaa2_fl_set_len(out_fle, out_len);
 585
 586 skip_out_fle:
 587         return edesc;
 588 }
 589
 590 static int chachapoly_set_sh_desc(struct crypto_aead *aead)
 591 {
 592         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 593         unsigned int ivsize = crypto_aead_ivsize(aead);
 594         struct device *dev = ctx->dev;
 595         struct caam_flc *flc;
 596         u32 *desc;
 597
 598         if (!ctx->cdata.keylen || !ctx->authsize)
 599                 return 0;
 600
 601         flc = &ctx->flc[ENCRYPT];
 602         desc = flc->sh_desc;
 603         cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
 604                                ctx->authsize, true, true);
 605         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 606         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 607                                    sizeof(flc->flc) + desc_bytes(desc),
 608                                    ctx->dir);
 609
 610         flc = &ctx->flc[DECRYPT];
 611         desc = flc->sh_desc;
 612         cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
 613                                ctx->authsize, false, true);
 614         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 615         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 616                                    sizeof(flc->flc) + desc_bytes(desc),
 617                                    ctx->dir);
 618
 619         return 0;
 620 }
 621
 622 static int chachapoly_setauthsize(struct crypto_aead *aead,
 623                                   unsigned int authsize)
 624 {
 625         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 626
 627         if (authsize != POLY1305_DIGEST_SIZE)
 628                 return -EINVAL;
 629
 630         ctx->authsize = authsize;
 631         return chachapoly_set_sh_desc(aead);
 632 }
 633
 634 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
 635                              unsigned int keylen)
 636 {
 637         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 638         unsigned int ivsize = crypto_aead_ivsize(aead);
 639         unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize;
 640
 641         if (keylen != CHACHA_KEY_SIZE + saltlen)
 642                 return -EINVAL;
 643
 644         ctx->cdata.key_virt = key;
 645         ctx->cdata.keylen = keylen - saltlen;
 646
 647         return chachapoly_set_sh_desc(aead);
 648 }
 649
 650 static int gcm_set_sh_desc(struct crypto_aead *aead)
 651 {
 652         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 653         struct device *dev = ctx->dev;
 654         unsigned int ivsize = crypto_aead_ivsize(aead);
 655         struct caam_flc *flc;
 656         u32 *desc;
 657         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 658                         ctx->cdata.keylen;
 659
 660         if (!ctx->cdata.keylen || !ctx->authsize)
 661                 return 0;
 662
 663         /*
 664          * AES GCM encrypt shared descriptor
 665          * Job Descriptor and Shared Descriptor
 666          * must fit into the 64-word Descriptor h/w Buffer
 667          */
 668         if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {
 669                 ctx->cdata.key_inline = true;
 670                 ctx->cdata.key_virt = ctx->key;
 671         } else {
 672                 ctx->cdata.key_inline = false;
 673                 ctx->cdata.key_dma = ctx->key_dma;
 674         }
 675
 676         flc = &ctx->flc[ENCRYPT];
 677         desc = flc->sh_desc;
 678         cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
 679         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 680         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 681                                    sizeof(flc->flc) + desc_bytes(desc),
 682                                    ctx->dir);
 683
 684         /*
 685          * Job Descriptor and Shared Descriptors
 686          * must all fit into the 64-word Descriptor h/w Buffer
 687          */
 688         if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {
 689                 ctx->cdata.key_inline = true;
 690                 ctx->cdata.key_virt = ctx->key;
 691         } else {
 692                 ctx->cdata.key_inline = false;
 693                 ctx->cdata.key_dma = ctx->key_dma;
 694         }
 695
 696         flc = &ctx->flc[DECRYPT];
 697         desc = flc->sh_desc;
 698         cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
 699         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 700         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 701                                    sizeof(flc->flc) + desc_bytes(desc),
 702                                    ctx->dir);
 703
 704         return 0;
 705 }
 706
 707 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 708 {
 709         struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
 710         int err;
 711
 712         err = crypto_gcm_check_authsize(authsize);
 713         if (err)
 714                 return err;
 715
 716         ctx->authsize = authsize;
 717         gcm_set_sh_desc(authenc);
 718
 719         return 0;
 720 }
 721
 722 static int gcm_setkey(struct crypto_aead *aead,
 723                       const u8 *key, unsigned int keylen)
 724 {
 725         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 726         struct device *dev = ctx->dev;
 727         int ret;
 728
 729         ret = aes_check_keylen(keylen);
 730         if (ret)
 731                 return ret;
 732         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 733                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 734
 735         memcpy(ctx->key, key, keylen);
 736         dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir);
 737         ctx->cdata.keylen = keylen;
 738
 739         return gcm_set_sh_desc(aead);
 740 }
 741
 742 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 743 {
 744         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 745         struct device *dev = ctx->dev;
 746         unsigned int ivsize = crypto_aead_ivsize(aead);
 747         struct caam_flc *flc;
 748         u32 *desc;
 749         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 750                         ctx->cdata.keylen;
 751
 752         if (!ctx->cdata.keylen || !ctx->authsize)
 753                 return 0;
 754
 755         ctx->cdata.key_virt = ctx->key;
 756
 757         /*
 758          * RFC4106 encrypt shared descriptor
 759          * Job Descriptor and Shared Descriptor
 760          * must fit into the 64-word Descriptor h/w Buffer
 761          */
 762         if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {
 763                 ctx->cdata.key_inline = true;
 764         } else {
 765                 ctx->cdata.key_inline = false;
 766                 ctx->cdata.key_dma = ctx->key_dma;
 767         }
 768
 769         flc = &ctx->flc[ENCRYPT];
 770         desc = flc->sh_desc;
 771         cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
 772                                   true);
 773         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 774         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 775                                    sizeof(flc->flc) + desc_bytes(desc),
 776                                    ctx->dir);
 777
 778         /*
 779          * Job Descriptor and Shared Descriptors
 780          * must all fit into the 64-word Descriptor h/w Buffer
 781          */
 782         if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {
 783                 ctx->cdata.key_inline = true;
 784         } else {
 785                 ctx->cdata.key_inline = false;
 786                 ctx->cdata.key_dma = ctx->key_dma;
 787         }
 788
 789         flc = &ctx->flc[DECRYPT];
 790         desc = flc->sh_desc;
 791         cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
 792                                   true);
 793         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 794         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 795                                    sizeof(flc->flc) + desc_bytes(desc),
 796                                    ctx->dir);
 797
 798         return 0;
 799 }
 800
 801 static int rfc4106_setauthsize(struct crypto_aead *authenc,
 802                                unsigned int authsize)
 803 {
 804         struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
 805         int err;
 806
 807         err = crypto_rfc4106_check_authsize(authsize);
 808         if (err)
 809                 return err;
 810
 811         ctx->authsize = authsize;
 812         rfc4106_set_sh_desc(authenc);
 813
 814         return 0;
 815 }
 816
 817 static int rfc4106_setkey(struct crypto_aead *aead,
 818                           const u8 *key, unsigned int keylen)
 819 {
 820         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 821         struct device *dev = ctx->dev;
 822         int ret;
 823
 824         ret = aes_check_keylen(keylen - 4);
 825         if (ret)
 826                 return ret;
 827
 828         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 829                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 830
 831         memcpy(ctx->key, key, keylen);
 832         /*
 833          * The last four bytes of the key material are used as the salt value
 834          * in the nonce. Update the AES key length.
 835          */
 836         ctx->cdata.keylen = keylen - 4;
 837         dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
 838                                    ctx->dir);
 839
 840         return rfc4106_set_sh_desc(aead);
 841 }
 842
 843 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
 844 {
 845         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 846         struct device *dev = ctx->dev;
 847         unsigned int ivsize = crypto_aead_ivsize(aead);
 848         struct caam_flc *flc;
 849         u32 *desc;
 850         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 851                         ctx->cdata.keylen;
 852
 853         if (!ctx->cdata.keylen || !ctx->authsize)
 854                 return 0;
 855
 856         ctx->cdata.key_virt = ctx->key;
 857
 858         /*
 859          * RFC4543 encrypt shared descriptor
 860          * Job Descriptor and Shared Descriptor
 861          * must fit into the 64-word Descriptor h/w Buffer
 862          */
 863         if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {
 864                 ctx->cdata.key_inline = true;
 865         } else {
 866                 ctx->cdata.key_inline = false;
 867                 ctx->cdata.key_dma = ctx->key_dma;
 868         }
 869
 870         flc = &ctx->flc[ENCRYPT];
 871         desc = flc->sh_desc;
 872         cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
 873                                   true);
 874         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 875         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 876                                    sizeof(flc->flc) + desc_bytes(desc),
 877                                    ctx->dir);
 878
 879         /*
 880          * Job Descriptor and Shared Descriptors
 881          * must all fit into the 64-word Descriptor h/w Buffer
 882          */
 883         if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {
 884                 ctx->cdata.key_inline = true;
 885         } else {
 886                 ctx->cdata.key_inline = false;
 887                 ctx->cdata.key_dma = ctx->key_dma;
 888         }
 889
 890         flc = &ctx->flc[DECRYPT];
 891         desc = flc->sh_desc;
 892         cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
 893                                   true);
 894         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 895         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 896                                    sizeof(flc->flc) + desc_bytes(desc),
 897                                    ctx->dir);
 898
 899         return 0;
 900 }
 901
 902 static int rfc4543_setauthsize(struct crypto_aead *authenc,
 903                                unsigned int authsize)
 904 {
 905         struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
 906
 907         if (authsize != 16)
 908                 return -EINVAL;
 909
 910         ctx->authsize = authsize;
 911         rfc4543_set_sh_desc(authenc);
 912
 913         return 0;
 914 }
 915
 916 static int rfc4543_setkey(struct crypto_aead *aead,
 917                           const u8 *key, unsigned int keylen)
 918 {
 919         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 920         struct device *dev = ctx->dev;
 921         int ret;
 922
 923         ret = aes_check_keylen(keylen - 4);
 924         if (ret)
 925                 return ret;
 926
 927         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 928                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 929
 930         memcpy(ctx->key, key, keylen);
 931         /*
 932          * The last four bytes of the key material are used as the salt value
 933          * in the nonce. Update the AES key length.
 934          */
 935         ctx->cdata.keylen = keylen - 4;
 936         dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
 937                                    ctx->dir);
 938
 939         return rfc4543_set_sh_desc(aead);
 940 }
 941
 942 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 943                            unsigned int keylen, const u32 ctx1_iv_off)
 944 {
 945         struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
 946         struct caam_skcipher_alg *alg =
 947                 container_of(crypto_skcipher_alg(skcipher),
 948                              struct caam_skcipher_alg, skcipher);
 949         struct device *dev = ctx->dev;
 950         struct caam_flc *flc;
 951         unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
 952         u32 *desc;
 953         const bool is_rfc3686 = alg->caam.rfc3686;
 954
 955         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 956                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 957
 958         ctx->cdata.keylen = keylen;
 959         ctx->cdata.key_virt = key;
 960         ctx->cdata.key_inline = true;
 961
 962         /* skcipher_encrypt shared descriptor */
 963         flc = &ctx->flc[ENCRYPT];
 964         desc = flc->sh_desc;
 965         cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
 966                                    ctx1_iv_off);
 967         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 968         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 969                                    sizeof(flc->flc) + desc_bytes(desc),
 970                                    ctx->dir);
 971
 972         /* skcipher_decrypt shared descriptor */
 973         flc = &ctx->flc[DECRYPT];
 974         desc = flc->sh_desc;
 975         cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
 976                                    ctx1_iv_off);
 977         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 978         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 979                                    sizeof(flc->flc) + desc_bytes(desc),
 980                                    ctx->dir);
 981
 982         return 0;
 983 }
 984
 985 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
 986                                const u8 *key, unsigned int keylen)
 987 {
 988         int err;
 989
 990         err = aes_check_keylen(keylen);
 991         if (err)
 992                 return err;
 993
 994         return skcipher_setkey(skcipher, key, keylen, 0);
 995 }
 996
 997 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
 998                                    const u8 *key, unsigned int keylen)
 999 {
1000         u32 ctx1_iv_off;
1001         int err;
1002
1003         /*
1004          * RFC3686 specific:
1005          *      | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1006          *      | *key = {KEY, NONCE}
1007          */
1008         ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
1009         keylen -= CTR_RFC3686_NONCE_SIZE;
1010
1011         err = aes_check_keylen(keylen);
1012         if (err)
1013                 return err;
1014
1015         return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1016 }
1017
1018 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
1019                                const u8 *key, unsigned int keylen)
1020 {
1021         u32 ctx1_iv_off;
1022         int err;
1023
1024         /*
1025          * AES-CTR needs to load IV in CONTEXT1 reg
1026          * at an offset of 128bits (16bytes)
1027          * CONTEXT1[255:128] = IV
1028          */
1029         ctx1_iv_off = 16;
1030
1031         err = aes_check_keylen(keylen);
1032         if (err)
1033                 return err;
1034
1035         return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1036 }
1037
1038 static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher,
1039                                     const u8 *key, unsigned int keylen)
1040 {
1041         if (keylen != CHACHA_KEY_SIZE)
1042                 return -EINVAL;
1043
1044         return skcipher_setkey(skcipher, key, keylen, 0);
1045 }
1046
1047 static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
1048                                const u8 *key, unsigned int keylen)
1049 {
1050         return verify_skcipher_des_key(skcipher, key) ?:
1051                skcipher_setkey(skcipher, key, keylen, 0);
1052 }
1053
1054 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
1055                                 const u8 *key, unsigned int keylen)
1056 {
1057         return verify_skcipher_des3_key(skcipher, key) ?:
1058                skcipher_setkey(skcipher, key, keylen, 0);
1059 }
1060
1061 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
1062                                unsigned int keylen)
1063 {
1064         struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1065         struct device *dev = ctx->dev;
1066         struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
1067         struct caam_flc *flc;
1068         u32 *desc;
1069         int err;
1070
1071         err = xts_verify_key(skcipher, key, keylen);
1072         if (err) {
1073                 dev_dbg(dev, "key size mismatch\n");
1074                 return err;
1075         }
1076
1077         if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256)
1078                 ctx->xts_key_fallback = true;
1079
1080         if (priv->sec_attr.era <= 8 || ctx->xts_key_fallback) {
1081                 err = crypto_skcipher_setkey(ctx->fallback, key, keylen);
1082                 if (err)
1083                         return err;
1084         }
1085
1086         ctx->cdata.keylen = keylen;
1087         ctx->cdata.key_virt = key;
1088         ctx->cdata.key_inline = true;
1089
1090         /* xts_skcipher_encrypt shared descriptor */
1091         flc = &ctx->flc[ENCRYPT];
1092         desc = flc->sh_desc;
1093         cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata);
1094         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1095         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
1096                                    sizeof(flc->flc) + desc_bytes(desc),
1097                                    ctx->dir);
1098
1099         /* xts_skcipher_decrypt shared descriptor */
1100         flc = &ctx->flc[DECRYPT];
1101         desc = flc->sh_desc;
1102         cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata);
1103         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1104         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
1105                                    sizeof(flc->flc) + desc_bytes(desc),
1106                                    ctx->dir);
1107
1108         return 0;
1109 }
1110
1111 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
1112 {
1113         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1114         struct caam_request *req_ctx = skcipher_request_ctx_dma(req);
1115         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
1116         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
1117         struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1118         struct device *dev = ctx->dev;
1119         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1120                        GFP_KERNEL : GFP_ATOMIC;
1121         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1122         struct skcipher_edesc *edesc;
1123         dma_addr_t iv_dma;
1124         u8 *iv;
1125         int ivsize = crypto_skcipher_ivsize(skcipher);
1126         int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
1127         struct dpaa2_sg_entry *sg_table;
1128
1129         src_nents = sg_nents_for_len(req->src, req->cryptlen);
1130         if (unlikely(src_nents < 0)) {
1131                 dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
1132                         req->cryptlen);
1133                 return ERR_PTR(src_nents);
1134         }
1135
1136         if (unlikely(req->dst != req->src)) {
1137                 dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
1138                 if (unlikely(dst_nents < 0)) {
1139                         dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
1140                                 req->cryptlen);
1141                         return ERR_PTR(dst_nents);
1142                 }
1143
1144                 mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1145                                               DMA_TO_DEVICE);
1146                 if (unlikely(!mapped_src_nents)) {
1147                         dev_err(dev, "unable to map source\n");
1148                         return ERR_PTR(-ENOMEM);
1149                 }
1150
1151                 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
1152                                               DMA_FROM_DEVICE);
1153                 if (unlikely(!mapped_dst_nents)) {
1154                         dev_err(dev, "unable to map destination\n");
1155                         dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
1156                         return ERR_PTR(-ENOMEM);
1157                 }
1158         } else {
1159                 mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1160                                               DMA_BIDIRECTIONAL);
1161                 if (unlikely(!mapped_src_nents)) {
1162                         dev_err(dev, "unable to map source\n");
1163                         return ERR_PTR(-ENOMEM);
1164                 }
1165         }
1166
1167         qm_sg_ents = 1 + mapped_src_nents;
1168         dst_sg_idx = qm_sg_ents;
1169
1170         /*
1171          * Input, output HW S/G tables: [IV, src][dst, IV]
1172          * IV entries point to the same buffer
1173          * If src == dst, S/G entries are reused (S/G tables overlap)
1174          *
1175          * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1176          * the end of the table by allocating more S/G entries.
1177          */
1178         if (req->src != req->dst)
1179                 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
1180         else
1181                 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
1182
1183         qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry);
1184         if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
1185                      ivsize > CAAM_QI_MEMCACHE_SIZE)) {
1186                 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
1187                         qm_sg_ents, ivsize);
1188                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1189                            0, DMA_NONE, 0, 0);
1190                 return ERR_PTR(-ENOMEM);
1191         }
1192
1193         /* allocate space for base edesc, link tables and IV */
1194         edesc = qi_cache_zalloc(flags);
1195         if (unlikely(!edesc)) {
1196                 dev_err(dev, "could not allocate extended descriptor\n");
1197                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1198                            0, DMA_NONE, 0, 0);
1199                 return ERR_PTR(-ENOMEM);
1200         }
1201
1202         /* Make sure IV is located in a DMAable area */
1203         sg_table = &edesc->sgt[0];
1204         iv = (u8 *)(sg_table + qm_sg_ents);
1205         memcpy(iv, req->iv, ivsize);
1206
1207         iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL);
1208         if (dma_mapping_error(dev, iv_dma)) {
1209                 dev_err(dev, "unable to map IV\n");
1210                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1211                            0, DMA_NONE, 0, 0);
1212                 qi_cache_free(edesc);
1213                 return ERR_PTR(-ENOMEM);
1214         }
1215
1216         edesc->src_nents = src_nents;
1217         edesc->dst_nents = dst_nents;
1218         edesc->iv_dma = iv_dma;
1219         edesc->qm_sg_bytes = qm_sg_bytes;
1220
1221         dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
1222         sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
1223
1224         if (req->src != req->dst)
1225                 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
1226
1227         dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
1228                          ivsize, 0);
1229
1230         edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes,
1231                                           DMA_TO_DEVICE);
1232         if (dma_mapping_error(dev, edesc->qm_sg_dma)) {
1233                 dev_err(dev, "unable to map S/G table\n");
1234                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
1235                            iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
1236                 qi_cache_free(edesc);
1237                 return ERR_PTR(-ENOMEM);
1238         }
1239
1240         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
1241         dpaa2_fl_set_final(in_fle, true);
1242         dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize);
1243         dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize);
1244
1245         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
1246         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
1247
1248         dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
1249
1250         if (req->src == req->dst)
1251                 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma +
1252                                   sizeof(*sg_table));
1253         else
1254                 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx *
1255                                   sizeof(*sg_table));
1256
1257         return edesc;
1258 }
1259
1260 static void aead_unmap(struct device *dev, struct aead_edesc *edesc,
1261                        struct aead_request *req)
1262 {
1263         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1264         int ivsize = crypto_aead_ivsize(aead);
1265
1266         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1267                    edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
1268                    edesc->qm_sg_bytes);
1269         dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
1270 }
1271
1272 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
1273                            struct skcipher_request *req)
1274 {
1275         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1276         int ivsize = crypto_skcipher_ivsize(skcipher);
1277
1278         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1279                    edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
1280                    edesc->qm_sg_bytes);
1281 }
1282
1283 static void aead_encrypt_done(void *cbk_ctx, u32 status)
1284 {
1285         struct crypto_async_request *areq = cbk_ctx;
1286         struct aead_request *req = container_of(areq, struct aead_request,
1287                                                 base);
1288         struct caam_request *req_ctx = to_caam_req(areq);
1289         struct aead_edesc *edesc = req_ctx->edesc;
1290         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1291         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
1292         int ecode = 0;
1293
1294         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1295
1296         if (unlikely(status))
1297                 ecode = caam_qi2_strstatus(ctx->dev, status);
1298
1299         aead_unmap(ctx->dev, edesc, req);
1300         qi_cache_free(edesc);
1301         aead_request_complete(req, ecode);
1302 }
1303
1304 static void aead_decrypt_done(void *cbk_ctx, u32 status)
1305 {
1306         struct crypto_async_request *areq = cbk_ctx;
1307         struct aead_request *req = container_of(areq, struct aead_request,
1308                                                 base);
1309         struct caam_request *req_ctx = to_caam_req(areq);
1310         struct aead_edesc *edesc = req_ctx->edesc;
1311         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1312         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
1313         int ecode = 0;
1314
1315         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1316
1317         if (unlikely(status))
1318                 ecode = caam_qi2_strstatus(ctx->dev, status);
1319
1320         aead_unmap(ctx->dev, edesc, req);
1321         qi_cache_free(edesc);
1322         aead_request_complete(req, ecode);
1323 }
1324
1325 static int aead_encrypt(struct aead_request *req)
1326 {
1327         struct aead_edesc *edesc;
1328         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1329         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
1330         struct caam_request *caam_req = aead_request_ctx_dma(req);
1331         int ret;
1332
1333         /* allocate extended descriptor */
1334         edesc = aead_edesc_alloc(req, true);
1335         if (IS_ERR(edesc))
1336                 return PTR_ERR(edesc);
1337
1338         caam_req->flc = &ctx->flc[ENCRYPT];
1339         caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1340         caam_req->cbk = aead_encrypt_done;
1341         caam_req->ctx = &req->base;
1342         caam_req->edesc = edesc;
1343         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1344         if (ret != -EINPROGRESS &&
1345             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1346                 aead_unmap(ctx->dev, edesc, req);
1347                 qi_cache_free(edesc);
1348         }
1349
1350         return ret;
1351 }
1352
1353 static int aead_decrypt(struct aead_request *req)
1354 {
1355         struct aead_edesc *edesc;
1356         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1357         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
1358         struct caam_request *caam_req = aead_request_ctx_dma(req);
1359         int ret;
1360
1361         /* allocate extended descriptor */
1362         edesc = aead_edesc_alloc(req, false);
1363         if (IS_ERR(edesc))
1364                 return PTR_ERR(edesc);
1365
1366         caam_req->flc = &ctx->flc[DECRYPT];
1367         caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1368         caam_req->cbk = aead_decrypt_done;
1369         caam_req->ctx = &req->base;
1370         caam_req->edesc = edesc;
1371         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1372         if (ret != -EINPROGRESS &&
1373             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1374                 aead_unmap(ctx->dev, edesc, req);
1375                 qi_cache_free(edesc);
1376         }
1377
1378         return ret;
1379 }
1380
1381 static int ipsec_gcm_encrypt(struct aead_request *req)
1382 {
1383         return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req);
1384 }
1385
1386 static int ipsec_gcm_decrypt(struct aead_request *req)
1387 {
1388         return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req);
1389 }
1390
1391 static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
1392 {
1393         struct crypto_async_request *areq = cbk_ctx;
1394         struct skcipher_request *req = skcipher_request_cast(areq);
1395         struct caam_request *req_ctx = to_caam_req(areq);
1396         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1397         struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1398         struct skcipher_edesc *edesc = req_ctx->edesc;
1399         int ecode = 0;
1400         int ivsize = crypto_skcipher_ivsize(skcipher);
1401
1402         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1403
1404         if (unlikely(status))
1405                 ecode = caam_qi2_strstatus(ctx->dev, status);
1406
1407         print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1408                              DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1409                              edesc->src_nents > 1 ? 100 : ivsize, 1);
1410         caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1411                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1412                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1413
1414         skcipher_unmap(ctx->dev, edesc, req);
1415
1416         /*
1417          * The crypto API expects us to set the IV (req->iv) to the last
1418          * ciphertext block (CBC mode) or last counter (CTR mode).
1419          * This is used e.g. by the CTS mode.
1420          */
1421         if (!ecode)
1422                 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1423                        ivsize);
1424
1425         qi_cache_free(edesc);
1426         skcipher_request_complete(req, ecode);
1427 }
1428
1429 static void skcipher_decrypt_done(void *cbk_ctx, u32 status)
1430 {
1431         struct crypto_async_request *areq = cbk_ctx;
1432         struct skcipher_request *req = skcipher_request_cast(areq);
1433         struct caam_request *req_ctx = to_caam_req(areq);
1434         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1435         struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1436         struct skcipher_edesc *edesc = req_ctx->edesc;
1437         int ecode = 0;
1438         int ivsize = crypto_skcipher_ivsize(skcipher);
1439
1440         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1441
1442         if (unlikely(status))
1443                 ecode = caam_qi2_strstatus(ctx->dev, status);
1444
1445         print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1446                              DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1447                              edesc->src_nents > 1 ? 100 : ivsize, 1);
1448         caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1449                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1450                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1451
1452         skcipher_unmap(ctx->dev, edesc, req);
1453
1454         /*
1455          * The crypto API expects us to set the IV (req->iv) to the last
1456          * ciphertext block (CBC mode) or last counter (CTR mode).
1457          * This is used e.g. by the CTS mode.
1458          */
1459         if (!ecode)
1460                 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1461                        ivsize);
1462
1463         qi_cache_free(edesc);
1464         skcipher_request_complete(req, ecode);
1465 }
1466
1467 static inline bool xts_skcipher_ivsize(struct skcipher_request *req)
1468 {
1469         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1470         unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
1471
1472         return !!get_unaligned((u64 *)(req->iv + (ivsize / 2)));
1473 }
1474
1475 static int skcipher_encrypt(struct skcipher_request *req)
1476 {
1477         struct skcipher_edesc *edesc;
1478         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1479         struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1480         struct caam_request *caam_req = skcipher_request_ctx_dma(req);
1481         struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
1482         int ret;
1483
1484         /*
1485          * XTS is expected to return an error even for input length = 0
1486          * Note that the case input length < block size will be caught during
1487          * HW offloading and return an error.
1488          */
1489         if (!req->cryptlen && !ctx->fallback)
1490                 return 0;
1491
1492         if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) ||
1493                               ctx->xts_key_fallback)) {
1494                 skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback);
1495                 skcipher_request_set_callback(&caam_req->fallback_req,
1496                                               req->base.flags,
1497                                               req->base.complete,
1498                                               req->base.data);
1499                 skcipher_request_set_crypt(&caam_req->fallback_req, req->src,
1500                                            req->dst, req->cryptlen, req->iv);
1501
1502                 return crypto_skcipher_encrypt(&caam_req->fallback_req);
1503         }
1504
1505         /* allocate extended descriptor */
1506         edesc = skcipher_edesc_alloc(req);
1507         if (IS_ERR(edesc))
1508                 return PTR_ERR(edesc);
1509
1510         caam_req->flc = &ctx->flc[ENCRYPT];
1511         caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1512         caam_req->cbk = skcipher_encrypt_done;
1513         caam_req->ctx = &req->base;
1514         caam_req->edesc = edesc;
1515         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1516         if (ret != -EINPROGRESS &&
1517             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1518                 skcipher_unmap(ctx->dev, edesc, req);
1519                 qi_cache_free(edesc);
1520         }
1521
1522         return ret;
1523 }
1524
1525 static int skcipher_decrypt(struct skcipher_request *req)
1526 {
1527         struct skcipher_edesc *edesc;
1528         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1529         struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1530         struct caam_request *caam_req = skcipher_request_ctx_dma(req);
1531         struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
1532         int ret;
1533
1534         /*
1535          * XTS is expected to return an error even for input length = 0
1536          * Note that the case input length < block size will be caught during
1537          * HW offloading and return an error.
1538          */
1539         if (!req->cryptlen && !ctx->fallback)
1540                 return 0;
1541
1542         if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) ||
1543                               ctx->xts_key_fallback)) {
1544                 skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback);
1545                 skcipher_request_set_callback(&caam_req->fallback_req,
1546                                               req->base.flags,
1547                                               req->base.complete,
1548                                               req->base.data);
1549                 skcipher_request_set_crypt(&caam_req->fallback_req, req->src,
1550                                            req->dst, req->cryptlen, req->iv);
1551
1552                 return crypto_skcipher_decrypt(&caam_req->fallback_req);
1553         }
1554
1555         /* allocate extended descriptor */
1556         edesc = skcipher_edesc_alloc(req);
1557         if (IS_ERR(edesc))
1558                 return PTR_ERR(edesc);
1559
1560         caam_req->flc = &ctx->flc[DECRYPT];
1561         caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1562         caam_req->cbk = skcipher_decrypt_done;
1563         caam_req->ctx = &req->base;
1564         caam_req->edesc = edesc;
1565         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1566         if (ret != -EINPROGRESS &&
1567             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1568                 skcipher_unmap(ctx->dev, edesc, req);
1569                 qi_cache_free(edesc);
1570         }
1571
1572         return ret;
1573 }
1574
1575 static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam,
1576                          bool uses_dkp)
1577 {
1578         dma_addr_t dma_addr;
1579         int i;
1580
1581         /* copy descriptor header template value */
1582         ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
1583         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
1584
1585         ctx->dev = caam->dev;
1586         ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
1587
1588         dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc,
1589                                         offsetof(struct caam_ctx, flc_dma),
1590                                         ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
1591         if (dma_mapping_error(ctx->dev, dma_addr)) {
1592                 dev_err(ctx->dev, "unable to map key, shared descriptors\n");
1593                 return -ENOMEM;
1594         }
1595
1596         for (i = 0; i < NUM_OP; i++)
1597                 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
1598         ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]);
1599
1600         return 0;
1601 }
1602
1603 static int caam_cra_init_skcipher(struct crypto_skcipher *tfm)
1604 {
1605         struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1606         struct caam_skcipher_alg *caam_alg =
1607                 container_of(alg, typeof(*caam_alg), skcipher);
1608         struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
1609         u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
1610         int ret = 0;
1611
1612         if (alg_aai == OP_ALG_AAI_XTS) {
1613                 const char *tfm_name = crypto_tfm_alg_name(&tfm->base);
1614                 struct crypto_skcipher *fallback;
1615
1616                 fallback = crypto_alloc_skcipher(tfm_name, 0,
1617                                                  CRYPTO_ALG_NEED_FALLBACK);
1618                 if (IS_ERR(fallback)) {
1619                         dev_err(caam_alg->caam.dev,
1620                                 "Failed to allocate %s fallback: %ld\n",
1621                                 tfm_name, PTR_ERR(fallback));
1622                         return PTR_ERR(fallback);
1623                 }
1624
1625                 ctx->fallback = fallback;
1626                 crypto_skcipher_set_reqsize_dma(
1627                         tfm, sizeof(struct caam_request) +
1628                              crypto_skcipher_reqsize(fallback));
1629         } else {
1630                 crypto_skcipher_set_reqsize_dma(tfm,
1631                                                 sizeof(struct caam_request));
1632         }
1633
1634         ret = caam_cra_init(ctx, &caam_alg->caam, false);
1635         if (ret && ctx->fallback)
1636                 crypto_free_skcipher(ctx->fallback);
1637
1638         return ret;
1639 }
1640
1641 static int caam_cra_init_aead(struct crypto_aead *tfm)
1642 {
1643         struct aead_alg *alg = crypto_aead_alg(tfm);
1644         struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg),
1645                                                       aead);
1646
1647         crypto_aead_set_reqsize_dma(tfm, sizeof(struct caam_request));
1648         return caam_cra_init(crypto_aead_ctx_dma(tfm), &caam_alg->caam,
1649                              !caam_alg->caam.nodkp);
1650 }
1651
1652 static void caam_exit_common(struct caam_ctx *ctx)
1653 {
1654         dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0],
1655                                offsetof(struct caam_ctx, flc_dma), ctx->dir,
1656                                DMA_ATTR_SKIP_CPU_SYNC);
1657 }
1658
1659 static void caam_cra_exit(struct crypto_skcipher *tfm)
1660 {
1661         struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
1662
1663         if (ctx->fallback)
1664                 crypto_free_skcipher(ctx->fallback);
1665         caam_exit_common(ctx);
1666 }
1667
1668 static void caam_cra_exit_aead(struct crypto_aead *tfm)
1669 {
1670         caam_exit_common(crypto_aead_ctx_dma(tfm));
1671 }
1672
1673 static struct caam_skcipher_alg driver_algs[] = {
1674         {
1675                 .skcipher = {
1676                         .base = {
1677                                 .cra_name = "cbc(aes)",
1678                                 .cra_driver_name = "cbc-aes-caam-qi2",
1679                                 .cra_blocksize = AES_BLOCK_SIZE,
1680                         },
1681                         .setkey = aes_skcipher_setkey,
1682                         .encrypt = skcipher_encrypt,
1683                         .decrypt = skcipher_decrypt,
1684                         .min_keysize = AES_MIN_KEY_SIZE,
1685                         .max_keysize = AES_MAX_KEY_SIZE,
1686                         .ivsize = AES_BLOCK_SIZE,
1687                 },
1688                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1689         },
1690         {
1691                 .skcipher = {
1692                         .base = {
1693                                 .cra_name = "cbc(des3_ede)",
1694                                 .cra_driver_name = "cbc-3des-caam-qi2",
1695                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1696                         },
1697                         .setkey = des3_skcipher_setkey,
1698                         .encrypt = skcipher_encrypt,
1699                         .decrypt = skcipher_decrypt,
1700                         .min_keysize = DES3_EDE_KEY_SIZE,
1701                         .max_keysize = DES3_EDE_KEY_SIZE,
1702                         .ivsize = DES3_EDE_BLOCK_SIZE,
1703                 },
1704                 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1705         },
1706         {
1707                 .skcipher = {
1708                         .base = {
1709                                 .cra_name = "cbc(des)",
1710                                 .cra_driver_name = "cbc-des-caam-qi2",
1711                                 .cra_blocksize = DES_BLOCK_SIZE,
1712                         },
1713                         .setkey = des_skcipher_setkey,
1714                         .encrypt = skcipher_encrypt,
1715                         .decrypt = skcipher_decrypt,
1716                         .min_keysize = DES_KEY_SIZE,
1717                         .max_keysize = DES_KEY_SIZE,
1718                         .ivsize = DES_BLOCK_SIZE,
1719                 },
1720                 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1721         },
1722         {
1723                 .skcipher = {
1724                         .base = {
1725                                 .cra_name = "ctr(aes)",
1726                                 .cra_driver_name = "ctr-aes-caam-qi2",
1727                                 .cra_blocksize = 1,
1728                         },
1729                         .setkey = ctr_skcipher_setkey,
1730                         .encrypt = skcipher_encrypt,
1731                         .decrypt = skcipher_decrypt,
1732                         .min_keysize = AES_MIN_KEY_SIZE,
1733                         .max_keysize = AES_MAX_KEY_SIZE,
1734                         .ivsize = AES_BLOCK_SIZE,
1735                         .chunksize = AES_BLOCK_SIZE,
1736                 },
1737                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1738                                         OP_ALG_AAI_CTR_MOD128,
1739         },
1740         {
1741                 .skcipher = {
1742                         .base = {
1743                                 .cra_name = "rfc3686(ctr(aes))",
1744                                 .cra_driver_name = "rfc3686-ctr-aes-caam-qi2",
1745                                 .cra_blocksize = 1,
1746                         },
1747                         .setkey = rfc3686_skcipher_setkey,
1748                         .encrypt = skcipher_encrypt,
1749                         .decrypt = skcipher_decrypt,
1750                         .min_keysize = AES_MIN_KEY_SIZE +
1751                                        CTR_RFC3686_NONCE_SIZE,
1752                         .max_keysize = AES_MAX_KEY_SIZE +
1753                                        CTR_RFC3686_NONCE_SIZE,
1754                         .ivsize = CTR_RFC3686_IV_SIZE,
1755                         .chunksize = AES_BLOCK_SIZE,
1756                 },
1757                 .caam = {
1758                         .class1_alg_type = OP_ALG_ALGSEL_AES |
1759                                            OP_ALG_AAI_CTR_MOD128,
1760                         .rfc3686 = true,
1761                 },
1762         },
1763         {
1764                 .skcipher = {
1765                         .base = {
1766                                 .cra_name = "xts(aes)",
1767                                 .cra_driver_name = "xts-aes-caam-qi2",
1768                                 .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
1769                                 .cra_blocksize = AES_BLOCK_SIZE,
1770                         },
1771                         .setkey = xts_skcipher_setkey,
1772                         .encrypt = skcipher_encrypt,
1773                         .decrypt = skcipher_decrypt,
1774                         .min_keysize = 2 * AES_MIN_KEY_SIZE,
1775                         .max_keysize = 2 * AES_MAX_KEY_SIZE,
1776                         .ivsize = AES_BLOCK_SIZE,
1777                 },
1778                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
1779         },
1780         {
1781                 .skcipher = {
1782                         .base = {
1783                                 .cra_name = "chacha20",
1784                                 .cra_driver_name = "chacha20-caam-qi2",
1785                                 .cra_blocksize = 1,
1786                         },
1787                         .setkey = chacha20_skcipher_setkey,
1788                         .encrypt = skcipher_encrypt,
1789                         .decrypt = skcipher_decrypt,
1790                         .min_keysize = CHACHA_KEY_SIZE,
1791                         .max_keysize = CHACHA_KEY_SIZE,
1792                         .ivsize = CHACHA_IV_SIZE,
1793                 },
1794                 .caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20,
1795         },
1796 };
1797
1798 static struct caam_aead_alg driver_aeads[] = {
1799         {
1800                 .aead = {
1801                         .base = {
1802                                 .cra_name = "rfc4106(gcm(aes))",
1803                                 .cra_driver_name = "rfc4106-gcm-aes-caam-qi2",
1804                                 .cra_blocksize = 1,
1805                         },
1806                         .setkey = rfc4106_setkey,
1807                         .setauthsize = rfc4106_setauthsize,
1808                         .encrypt = ipsec_gcm_encrypt,
1809                         .decrypt = ipsec_gcm_decrypt,
1810                         .ivsize = 8,
1811                         .maxauthsize = AES_BLOCK_SIZE,
1812                 },
1813                 .caam = {
1814                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1815                         .nodkp = true,
1816                 },
1817         },
1818         {
1819                 .aead = {
1820                         .base = {
1821                                 .cra_name = "rfc4543(gcm(aes))",
1822                                 .cra_driver_name = "rfc4543-gcm-aes-caam-qi2",
1823                                 .cra_blocksize = 1,
1824                         },
1825                         .setkey = rfc4543_setkey,
1826                         .setauthsize = rfc4543_setauthsize,
1827                         .encrypt = ipsec_gcm_encrypt,
1828                         .decrypt = ipsec_gcm_decrypt,
1829                         .ivsize = 8,
1830                         .maxauthsize = AES_BLOCK_SIZE,
1831                 },
1832                 .caam = {
1833                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1834                         .nodkp = true,
1835                 },
1836         },
1837         /* Galois Counter Mode */
1838         {
1839                 .aead = {
1840                         .base = {
1841                                 .cra_name = "gcm(aes)",
1842                                 .cra_driver_name = "gcm-aes-caam-qi2",
1843                                 .cra_blocksize = 1,
1844                         },
1845                         .setkey = gcm_setkey,
1846                         .setauthsize = gcm_setauthsize,
1847                         .encrypt = aead_encrypt,
1848                         .decrypt = aead_decrypt,
1849                         .ivsize = 12,
1850                         .maxauthsize = AES_BLOCK_SIZE,
1851                 },
1852                 .caam = {
1853                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1854                         .nodkp = true,
1855                 }
1856         },
1857         /* single-pass ipsec_esp descriptor */
1858         {
1859                 .aead = {
1860                         .base = {
1861                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
1862                                 .cra_driver_name = "authenc-hmac-md5-"
1863                                                    "cbc-aes-caam-qi2",
1864                                 .cra_blocksize = AES_BLOCK_SIZE,
1865                         },
1866                         .setkey = aead_setkey,
1867                         .setauthsize = aead_setauthsize,
1868                         .encrypt = aead_encrypt,
1869                         .decrypt = aead_decrypt,
1870                         .ivsize = AES_BLOCK_SIZE,
1871                         .maxauthsize = MD5_DIGEST_SIZE,
1872                 },
1873                 .caam = {
1874                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1875                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1876                                            OP_ALG_AAI_HMAC_PRECOMP,
1877                 }
1878         },
1879         {
1880                 .aead = {
1881                         .base = {
1882                                 .cra_name = "echainiv(authenc(hmac(md5),"
1883                                             "cbc(aes)))",
1884                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
1885                                                    "cbc-aes-caam-qi2",
1886                                 .cra_blocksize = AES_BLOCK_SIZE,
1887                         },
1888                         .setkey = aead_setkey,
1889                         .setauthsize = aead_setauthsize,
1890                         .encrypt = aead_encrypt,
1891                         .decrypt = aead_decrypt,
1892                         .ivsize = AES_BLOCK_SIZE,
1893                         .maxauthsize = MD5_DIGEST_SIZE,
1894                 },
1895                 .caam = {
1896                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1897                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1898                                            OP_ALG_AAI_HMAC_PRECOMP,
1899                         .geniv = true,
1900                 }
1901         },
1902         {
1903                 .aead = {
1904                         .base = {
1905                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1906                                 .cra_driver_name = "authenc-hmac-sha1-"
1907                                                    "cbc-aes-caam-qi2",
1908                                 .cra_blocksize = AES_BLOCK_SIZE,
1909                         },
1910                         .setkey = aead_setkey,
1911                         .setauthsize = aead_setauthsize,
1912                         .encrypt = aead_encrypt,
1913                         .decrypt = aead_decrypt,
1914                         .ivsize = AES_BLOCK_SIZE,
1915                         .maxauthsize = SHA1_DIGEST_SIZE,
1916                 },
1917                 .caam = {
1918                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1919                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1920                                            OP_ALG_AAI_HMAC_PRECOMP,
1921                 }
1922         },
1923         {
1924                 .aead = {
1925                         .base = {
1926                                 .cra_name = "echainiv(authenc(hmac(sha1),"
1927                                             "cbc(aes)))",
1928                                 .cra_driver_name = "echainiv-authenc-"
1929                                                    "hmac-sha1-cbc-aes-caam-qi2",
1930                                 .cra_blocksize = AES_BLOCK_SIZE,
1931                         },
1932                         .setkey = aead_setkey,
1933                         .setauthsize = aead_setauthsize,
1934                         .encrypt = aead_encrypt,
1935                         .decrypt = aead_decrypt,
1936                         .ivsize = AES_BLOCK_SIZE,
1937                         .maxauthsize = SHA1_DIGEST_SIZE,
1938                 },
1939                 .caam = {
1940                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1941                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1942                                            OP_ALG_AAI_HMAC_PRECOMP,
1943                         .geniv = true,
1944                 },
1945         },
1946         {
1947                 .aead = {
1948                         .base = {
1949                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
1950                                 .cra_driver_name = "authenc-hmac-sha224-"
1951                                                    "cbc-aes-caam-qi2",
1952                                 .cra_blocksize = AES_BLOCK_SIZE,
1953                         },
1954                         .setkey = aead_setkey,
1955                         .setauthsize = aead_setauthsize,
1956                         .encrypt = aead_encrypt,
1957                         .decrypt = aead_decrypt,
1958                         .ivsize = AES_BLOCK_SIZE,
1959                         .maxauthsize = SHA224_DIGEST_SIZE,
1960                 },
1961                 .caam = {
1962                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1963                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1964                                            OP_ALG_AAI_HMAC_PRECOMP,
1965                 }
1966         },
1967         {
1968                 .aead = {
1969                         .base = {
1970                                 .cra_name = "echainiv(authenc(hmac(sha224),"
1971                                             "cbc(aes)))",
1972                                 .cra_driver_name = "echainiv-authenc-"
1973                                                    "hmac-sha224-cbc-aes-caam-qi2",
1974                                 .cra_blocksize = AES_BLOCK_SIZE,
1975                         },
1976                         .setkey = aead_setkey,
1977                         .setauthsize = aead_setauthsize,
1978                         .encrypt = aead_encrypt,
1979                         .decrypt = aead_decrypt,
1980                         .ivsize = AES_BLOCK_SIZE,
1981                         .maxauthsize = SHA224_DIGEST_SIZE,
1982                 },
1983                 .caam = {
1984                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1985                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1986                                            OP_ALG_AAI_HMAC_PRECOMP,
1987                         .geniv = true,
1988                 }
1989         },
1990         {
1991                 .aead = {
1992                         .base = {
1993                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1994                                 .cra_driver_name = "authenc-hmac-sha256-"
1995                                                    "cbc-aes-caam-qi2",
1996                                 .cra_blocksize = AES_BLOCK_SIZE,
1997                         },
1998                         .setkey = aead_setkey,
1999                         .setauthsize = aead_setauthsize,
2000                         .encrypt = aead_encrypt,
2001                         .decrypt = aead_decrypt,
2002                         .ivsize = AES_BLOCK_SIZE,
2003                         .maxauthsize = SHA256_DIGEST_SIZE,
2004                 },
2005                 .caam = {
2006                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2007                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2008                                            OP_ALG_AAI_HMAC_PRECOMP,
2009                 }
2010         },
2011         {
2012                 .aead = {
2013                         .base = {
2014                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2015                                             "cbc(aes)))",
2016                                 .cra_driver_name = "echainiv-authenc-"
2017                                                    "hmac-sha256-cbc-aes-"
2018                                                    "caam-qi2",
2019                                 .cra_blocksize = AES_BLOCK_SIZE,
2020                         },
2021                         .setkey = aead_setkey,
2022                         .setauthsize = aead_setauthsize,
2023                         .encrypt = aead_encrypt,
2024                         .decrypt = aead_decrypt,
2025                         .ivsize = AES_BLOCK_SIZE,
2026                         .maxauthsize = SHA256_DIGEST_SIZE,
2027                 },
2028                 .caam = {
2029                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2030                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2031                                            OP_ALG_AAI_HMAC_PRECOMP,
2032                         .geniv = true,
2033                 }
2034         },
2035         {
2036                 .aead = {
2037                         .base = {
2038                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2039                                 .cra_driver_name = "authenc-hmac-sha384-"
2040                                                    "cbc-aes-caam-qi2",
2041                                 .cra_blocksize = AES_BLOCK_SIZE,
2042                         },
2043                         .setkey = aead_setkey,
2044                         .setauthsize = aead_setauthsize,
2045                         .encrypt = aead_encrypt,
2046                         .decrypt = aead_decrypt,
2047                         .ivsize = AES_BLOCK_SIZE,
2048                         .maxauthsize = SHA384_DIGEST_SIZE,
2049                 },
2050                 .caam = {
2051                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2052                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2053                                            OP_ALG_AAI_HMAC_PRECOMP,
2054                 }
2055         },
2056         {
2057                 .aead = {
2058                         .base = {
2059                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2060                                             "cbc(aes)))",
2061                                 .cra_driver_name = "echainiv-authenc-"
2062                                                    "hmac-sha384-cbc-aes-"
2063                                                    "caam-qi2",
2064                                 .cra_blocksize = AES_BLOCK_SIZE,
2065                         },
2066                         .setkey = aead_setkey,
2067                         .setauthsize = aead_setauthsize,
2068                         .encrypt = aead_encrypt,
2069                         .decrypt = aead_decrypt,
2070                         .ivsize = AES_BLOCK_SIZE,
2071                         .maxauthsize = SHA384_DIGEST_SIZE,
2072                 },
2073                 .caam = {
2074                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2075                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2076                                            OP_ALG_AAI_HMAC_PRECOMP,
2077                         .geniv = true,
2078                 }
2079         },
2080         {
2081                 .aead = {
2082                         .base = {
2083                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2084                                 .cra_driver_name = "authenc-hmac-sha512-"
2085                                                    "cbc-aes-caam-qi2",
2086                                 .cra_blocksize = AES_BLOCK_SIZE,
2087                         },
2088                         .setkey = aead_setkey,
2089                         .setauthsize = aead_setauthsize,
2090                         .encrypt = aead_encrypt,
2091                         .decrypt = aead_decrypt,
2092                         .ivsize = AES_BLOCK_SIZE,
2093                         .maxauthsize = SHA512_DIGEST_SIZE,
2094                 },
2095                 .caam = {
2096                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2097                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2098                                            OP_ALG_AAI_HMAC_PRECOMP,
2099                 }
2100         },
2101         {
2102                 .aead = {
2103                         .base = {
2104                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2105                                             "cbc(aes)))",
2106                                 .cra_driver_name = "echainiv-authenc-"
2107                                                    "hmac-sha512-cbc-aes-"
2108                                                    "caam-qi2",
2109                                 .cra_blocksize = AES_BLOCK_SIZE,
2110                         },
2111                         .setkey = aead_setkey,
2112                         .setauthsize = aead_setauthsize,
2113                         .encrypt = aead_encrypt,
2114                         .decrypt = aead_decrypt,
2115                         .ivsize = AES_BLOCK_SIZE,
2116                         .maxauthsize = SHA512_DIGEST_SIZE,
2117                 },
2118                 .caam = {
2119                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2120                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2121                                            OP_ALG_AAI_HMAC_PRECOMP,
2122                         .geniv = true,
2123                 }
2124         },
2125         {
2126                 .aead = {
2127                         .base = {
2128                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2129                                 .cra_driver_name = "authenc-hmac-md5-"
2130                                                    "cbc-des3_ede-caam-qi2",
2131                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2132                         },
2133                         .setkey = des3_aead_setkey,
2134                         .setauthsize = aead_setauthsize,
2135                         .encrypt = aead_encrypt,
2136                         .decrypt = aead_decrypt,
2137                         .ivsize = DES3_EDE_BLOCK_SIZE,
2138                         .maxauthsize = MD5_DIGEST_SIZE,
2139                 },
2140                 .caam = {
2141                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2142                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2143                                            OP_ALG_AAI_HMAC_PRECOMP,
2144                 }
2145         },
2146         {
2147                 .aead = {
2148                         .base = {
2149                                 .cra_name = "echainiv(authenc(hmac(md5),"
2150                                             "cbc(des3_ede)))",
2151                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2152                                                    "cbc-des3_ede-caam-qi2",
2153                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2154                         },
2155                         .setkey = des3_aead_setkey,
2156                         .setauthsize = aead_setauthsize,
2157                         .encrypt = aead_encrypt,
2158                         .decrypt = aead_decrypt,
2159                         .ivsize = DES3_EDE_BLOCK_SIZE,
2160                         .maxauthsize = MD5_DIGEST_SIZE,
2161                 },
2162                 .caam = {
2163                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2164                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2165                                            OP_ALG_AAI_HMAC_PRECOMP,
2166                         .geniv = true,
2167                 }
2168         },
2169         {
2170                 .aead = {
2171                         .base = {
2172                                 .cra_name = "authenc(hmac(sha1),"
2173                                             "cbc(des3_ede))",
2174                                 .cra_driver_name = "authenc-hmac-sha1-"
2175                                                    "cbc-des3_ede-caam-qi2",
2176                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2177                         },
2178                         .setkey = des3_aead_setkey,
2179                         .setauthsize = aead_setauthsize,
2180                         .encrypt = aead_encrypt,
2181                         .decrypt = aead_decrypt,
2182                         .ivsize = DES3_EDE_BLOCK_SIZE,
2183                         .maxauthsize = SHA1_DIGEST_SIZE,
2184                 },
2185                 .caam = {
2186                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2187                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2188                                            OP_ALG_AAI_HMAC_PRECOMP,
2189                 },
2190         },
2191         {
2192                 .aead = {
2193                         .base = {
2194                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2195                                             "cbc(des3_ede)))",
2196                                 .cra_driver_name = "echainiv-authenc-"
2197                                                    "hmac-sha1-"
2198                                                    "cbc-des3_ede-caam-qi2",
2199                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2200                         },
2201                         .setkey = des3_aead_setkey,
2202                         .setauthsize = aead_setauthsize,
2203                         .encrypt = aead_encrypt,
2204                         .decrypt = aead_decrypt,
2205                         .ivsize = DES3_EDE_BLOCK_SIZE,
2206                         .maxauthsize = SHA1_DIGEST_SIZE,
2207                 },
2208                 .caam = {
2209                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2210                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2211                                            OP_ALG_AAI_HMAC_PRECOMP,
2212                         .geniv = true,
2213                 }
2214         },
2215         {
2216                 .aead = {
2217                         .base = {
2218                                 .cra_name = "authenc(hmac(sha224),"
2219                                             "cbc(des3_ede))",
2220                                 .cra_driver_name = "authenc-hmac-sha224-"
2221                                                    "cbc-des3_ede-caam-qi2",
2222                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2223                         },
2224                         .setkey = des3_aead_setkey,
2225                         .setauthsize = aead_setauthsize,
2226                         .encrypt = aead_encrypt,
2227                         .decrypt = aead_decrypt,
2228                         .ivsize = DES3_EDE_BLOCK_SIZE,
2229                         .maxauthsize = SHA224_DIGEST_SIZE,
2230                 },
2231                 .caam = {
2232                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2233                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2234                                            OP_ALG_AAI_HMAC_PRECOMP,
2235                 },
2236         },
2237         {
2238                 .aead = {
2239                         .base = {
2240                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2241                                             "cbc(des3_ede)))",
2242                                 .cra_driver_name = "echainiv-authenc-"
2243                                                    "hmac-sha224-"
2244                                                    "cbc-des3_ede-caam-qi2",
2245                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2246                         },
2247                         .setkey = des3_aead_setkey,
2248                         .setauthsize = aead_setauthsize,
2249                         .encrypt = aead_encrypt,
2250                         .decrypt = aead_decrypt,
2251                         .ivsize = DES3_EDE_BLOCK_SIZE,
2252                         .maxauthsize = SHA224_DIGEST_SIZE,
2253                 },
2254                 .caam = {
2255                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2256                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2257                                            OP_ALG_AAI_HMAC_PRECOMP,
2258                         .geniv = true,
2259                 }
2260         },
2261         {
2262                 .aead = {
2263                         .base = {
2264                                 .cra_name = "authenc(hmac(sha256),"
2265                                             "cbc(des3_ede))",
2266                                 .cra_driver_name = "authenc-hmac-sha256-"
2267                                                    "cbc-des3_ede-caam-qi2",
2268                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2269                         },
2270                         .setkey = des3_aead_setkey,
2271                         .setauthsize = aead_setauthsize,
2272                         .encrypt = aead_encrypt,
2273                         .decrypt = aead_decrypt,
2274                         .ivsize = DES3_EDE_BLOCK_SIZE,
2275                         .maxauthsize = SHA256_DIGEST_SIZE,
2276                 },
2277                 .caam = {
2278                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2279                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2280                                            OP_ALG_AAI_HMAC_PRECOMP,
2281                 },
2282         },
2283         {
2284                 .aead = {
2285                         .base = {
2286                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2287                                             "cbc(des3_ede)))",
2288                                 .cra_driver_name = "echainiv-authenc-"
2289                                                    "hmac-sha256-"
2290                                                    "cbc-des3_ede-caam-qi2",
2291                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2292                         },
2293                         .setkey = des3_aead_setkey,
2294                         .setauthsize = aead_setauthsize,
2295                         .encrypt = aead_encrypt,
2296                         .decrypt = aead_decrypt,
2297                         .ivsize = DES3_EDE_BLOCK_SIZE,
2298                         .maxauthsize = SHA256_DIGEST_SIZE,
2299                 },
2300                 .caam = {
2301                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2302                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2303                                            OP_ALG_AAI_HMAC_PRECOMP,
2304                         .geniv = true,
2305                 }
2306         },
2307         {
2308                 .aead = {
2309                         .base = {
2310                                 .cra_name = "authenc(hmac(sha384),"
2311                                             "cbc(des3_ede))",
2312                                 .cra_driver_name = "authenc-hmac-sha384-"
2313                                                    "cbc-des3_ede-caam-qi2",
2314                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2315                         },
2316                         .setkey = des3_aead_setkey,
2317                         .setauthsize = aead_setauthsize,
2318                         .encrypt = aead_encrypt,
2319                         .decrypt = aead_decrypt,
2320                         .ivsize = DES3_EDE_BLOCK_SIZE,
2321                         .maxauthsize = SHA384_DIGEST_SIZE,
2322                 },
2323                 .caam = {
2324                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2325                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2326                                            OP_ALG_AAI_HMAC_PRECOMP,
2327                 },
2328         },
2329         {
2330                 .aead = {
2331                         .base = {
2332                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2333                                             "cbc(des3_ede)))",
2334                                 .cra_driver_name = "echainiv-authenc-"
2335                                                    "hmac-sha384-"
2336                                                    "cbc-des3_ede-caam-qi2",
2337                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2338                         },
2339                         .setkey = des3_aead_setkey,
2340                         .setauthsize = aead_setauthsize,
2341                         .encrypt = aead_encrypt,
2342                         .decrypt = aead_decrypt,
2343                         .ivsize = DES3_EDE_BLOCK_SIZE,
2344                         .maxauthsize = SHA384_DIGEST_SIZE,
2345                 },
2346                 .caam = {
2347                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2348                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2349                                            OP_ALG_AAI_HMAC_PRECOMP,
2350                         .geniv = true,
2351                 }
2352         },
2353         {
2354                 .aead = {
2355                         .base = {
2356                                 .cra_name = "authenc(hmac(sha512),"
2357                                             "cbc(des3_ede))",
2358                                 .cra_driver_name = "authenc-hmac-sha512-"
2359                                                    "cbc-des3_ede-caam-qi2",
2360                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2361                         },
2362                         .setkey = des3_aead_setkey,
2363                         .setauthsize = aead_setauthsize,
2364                         .encrypt = aead_encrypt,
2365                         .decrypt = aead_decrypt,
2366                         .ivsize = DES3_EDE_BLOCK_SIZE,
2367                         .maxauthsize = SHA512_DIGEST_SIZE,
2368                 },
2369                 .caam = {
2370                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2371                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2372                                            OP_ALG_AAI_HMAC_PRECOMP,
2373                 },
2374         },
2375         {
2376                 .aead = {
2377                         .base = {
2378                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2379                                             "cbc(des3_ede)))",
2380                                 .cra_driver_name = "echainiv-authenc-"
2381                                                    "hmac-sha512-"
2382                                                    "cbc-des3_ede-caam-qi2",
2383                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2384                         },
2385                         .setkey = des3_aead_setkey,
2386                         .setauthsize = aead_setauthsize,
2387                         .encrypt = aead_encrypt,
2388                         .decrypt = aead_decrypt,
2389                         .ivsize = DES3_EDE_BLOCK_SIZE,
2390                         .maxauthsize = SHA512_DIGEST_SIZE,
2391                 },
2392                 .caam = {
2393                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2394                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2395                                            OP_ALG_AAI_HMAC_PRECOMP,
2396                         .geniv = true,
2397                 }
2398         },
2399         {
2400                 .aead = {
2401                         .base = {
2402                                 .cra_name = "authenc(hmac(md5),cbc(des))",
2403                                 .cra_driver_name = "authenc-hmac-md5-"
2404                                                    "cbc-des-caam-qi2",
2405                                 .cra_blocksize = DES_BLOCK_SIZE,
2406                         },
2407                         .setkey = aead_setkey,
2408                         .setauthsize = aead_setauthsize,
2409                         .encrypt = aead_encrypt,
2410                         .decrypt = aead_decrypt,
2411                         .ivsize = DES_BLOCK_SIZE,
2412                         .maxauthsize = MD5_DIGEST_SIZE,
2413                 },
2414                 .caam = {
2415                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2416                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2417                                            OP_ALG_AAI_HMAC_PRECOMP,
2418                 },
2419         },
2420         {
2421                 .aead = {
2422                         .base = {
2423                                 .cra_name = "echainiv(authenc(hmac(md5),"
2424                                             "cbc(des)))",
2425                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2426                                                    "cbc-des-caam-qi2",
2427                                 .cra_blocksize = DES_BLOCK_SIZE,
2428                         },
2429                         .setkey = aead_setkey,
2430                         .setauthsize = aead_setauthsize,
2431                         .encrypt = aead_encrypt,
2432                         .decrypt = aead_decrypt,
2433                         .ivsize = DES_BLOCK_SIZE,
2434                         .maxauthsize = MD5_DIGEST_SIZE,
2435                 },
2436                 .caam = {
2437                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2438                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2439                                            OP_ALG_AAI_HMAC_PRECOMP,
2440                         .geniv = true,
2441                 }
2442         },
2443         {
2444                 .aead = {
2445                         .base = {
2446                                 .cra_name = "authenc(hmac(sha1),cbc(des))",
2447                                 .cra_driver_name = "authenc-hmac-sha1-"
2448                                                    "cbc-des-caam-qi2",
2449                                 .cra_blocksize = DES_BLOCK_SIZE,
2450                         },
2451                         .setkey = aead_setkey,
2452                         .setauthsize = aead_setauthsize,
2453                         .encrypt = aead_encrypt,
2454                         .decrypt = aead_decrypt,
2455                         .ivsize = DES_BLOCK_SIZE,
2456                         .maxauthsize = SHA1_DIGEST_SIZE,
2457                 },
2458                 .caam = {
2459                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2460                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2461                                            OP_ALG_AAI_HMAC_PRECOMP,
2462                 },
2463         },
2464         {
2465                 .aead = {
2466                         .base = {
2467                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2468                                             "cbc(des)))",
2469                                 .cra_driver_name = "echainiv-authenc-"
2470                                                    "hmac-sha1-cbc-des-caam-qi2",
2471                                 .cra_blocksize = DES_BLOCK_SIZE,
2472                         },
2473                         .setkey = aead_setkey,
2474                         .setauthsize = aead_setauthsize,
2475                         .encrypt = aead_encrypt,
2476                         .decrypt = aead_decrypt,
2477                         .ivsize = DES_BLOCK_SIZE,
2478                         .maxauthsize = SHA1_DIGEST_SIZE,
2479                 },
2480                 .caam = {
2481                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2482                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2483                                            OP_ALG_AAI_HMAC_PRECOMP,
2484                         .geniv = true,
2485                 }
2486         },
2487         {
2488                 .aead = {
2489                         .base = {
2490                                 .cra_name = "authenc(hmac(sha224),cbc(des))",
2491                                 .cra_driver_name = "authenc-hmac-sha224-"
2492                                                    "cbc-des-caam-qi2",
2493                                 .cra_blocksize = DES_BLOCK_SIZE,
2494                         },
2495                         .setkey = aead_setkey,
2496                         .setauthsize = aead_setauthsize,
2497                         .encrypt = aead_encrypt,
2498                         .decrypt = aead_decrypt,
2499                         .ivsize = DES_BLOCK_SIZE,
2500                         .maxauthsize = SHA224_DIGEST_SIZE,
2501                 },
2502                 .caam = {
2503                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2504                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2505                                            OP_ALG_AAI_HMAC_PRECOMP,
2506                 },
2507         },
2508         {
2509                 .aead = {
2510                         .base = {
2511                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2512                                             "cbc(des)))",
2513                                 .cra_driver_name = "echainiv-authenc-"
2514                                                    "hmac-sha224-cbc-des-"
2515                                                    "caam-qi2",
2516                                 .cra_blocksize = DES_BLOCK_SIZE,
2517                         },
2518                         .setkey = aead_setkey,
2519                         .setauthsize = aead_setauthsize,
2520                         .encrypt = aead_encrypt,
2521                         .decrypt = aead_decrypt,
2522                         .ivsize = DES_BLOCK_SIZE,
2523                         .maxauthsize = SHA224_DIGEST_SIZE,
2524                 },
2525                 .caam = {
2526                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2527                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2528                                            OP_ALG_AAI_HMAC_PRECOMP,
2529                         .geniv = true,
2530                 }
2531         },
2532         {
2533                 .aead = {
2534                         .base = {
2535                                 .cra_name = "authenc(hmac(sha256),cbc(des))",
2536                                 .cra_driver_name = "authenc-hmac-sha256-"
2537                                                    "cbc-des-caam-qi2",
2538                                 .cra_blocksize = DES_BLOCK_SIZE,
2539                         },
2540                         .setkey = aead_setkey,
2541                         .setauthsize = aead_setauthsize,
2542                         .encrypt = aead_encrypt,
2543                         .decrypt = aead_decrypt,
2544                         .ivsize = DES_BLOCK_SIZE,
2545                         .maxauthsize = SHA256_DIGEST_SIZE,
2546                 },
2547                 .caam = {
2548                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2549                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2550                                            OP_ALG_AAI_HMAC_PRECOMP,
2551                 },
2552         },
2553         {
2554                 .aead = {
2555                         .base = {
2556                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2557                                             "cbc(des)))",
2558                                 .cra_driver_name = "echainiv-authenc-"
2559                                                    "hmac-sha256-cbc-des-"
2560                                                    "caam-qi2",
2561                                 .cra_blocksize = DES_BLOCK_SIZE,
2562                         },
2563                         .setkey = aead_setkey,
2564                         .setauthsize = aead_setauthsize,
2565                         .encrypt = aead_encrypt,
2566                         .decrypt = aead_decrypt,
2567                         .ivsize = DES_BLOCK_SIZE,
2568                         .maxauthsize = SHA256_DIGEST_SIZE,
2569                 },
2570                 .caam = {
2571                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2572                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2573                                            OP_ALG_AAI_HMAC_PRECOMP,
2574                         .geniv = true,
2575                 },
2576         },
2577         {
2578                 .aead = {
2579                         .base = {
2580                                 .cra_name = "authenc(hmac(sha384),cbc(des))",
2581                                 .cra_driver_name = "authenc-hmac-sha384-"
2582                                                    "cbc-des-caam-qi2",
2583                                 .cra_blocksize = DES_BLOCK_SIZE,
2584                         },
2585                         .setkey = aead_setkey,
2586                         .setauthsize = aead_setauthsize,
2587                         .encrypt = aead_encrypt,
2588                         .decrypt = aead_decrypt,
2589                         .ivsize = DES_BLOCK_SIZE,
2590                         .maxauthsize = SHA384_DIGEST_SIZE,
2591                 },
2592                 .caam = {
2593                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2594                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2595                                            OP_ALG_AAI_HMAC_PRECOMP,
2596                 },
2597         },
2598         {
2599                 .aead = {
2600                         .base = {
2601                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2602                                             "cbc(des)))",
2603                                 .cra_driver_name = "echainiv-authenc-"
2604                                                    "hmac-sha384-cbc-des-"
2605                                                    "caam-qi2",
2606                                 .cra_blocksize = DES_BLOCK_SIZE,
2607                         },
2608                         .setkey = aead_setkey,
2609                         .setauthsize = aead_setauthsize,
2610                         .encrypt = aead_encrypt,
2611                         .decrypt = aead_decrypt,
2612                         .ivsize = DES_BLOCK_SIZE,
2613                         .maxauthsize = SHA384_DIGEST_SIZE,
2614                 },
2615                 .caam = {
2616                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2617                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2618                                            OP_ALG_AAI_HMAC_PRECOMP,
2619                         .geniv = true,
2620                 }
2621         },
2622         {
2623                 .aead = {
2624                         .base = {
2625                                 .cra_name = "authenc(hmac(sha512),cbc(des))",
2626                                 .cra_driver_name = "authenc-hmac-sha512-"
2627                                                    "cbc-des-caam-qi2",
2628                                 .cra_blocksize = DES_BLOCK_SIZE,
2629                         },
2630                         .setkey = aead_setkey,
2631                         .setauthsize = aead_setauthsize,
2632                         .encrypt = aead_encrypt,
2633                         .decrypt = aead_decrypt,
2634                         .ivsize = DES_BLOCK_SIZE,
2635                         .maxauthsize = SHA512_DIGEST_SIZE,
2636                 },
2637                 .caam = {
2638                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2639                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2640                                            OP_ALG_AAI_HMAC_PRECOMP,
2641                 }
2642         },
2643         {
2644                 .aead = {
2645                         .base = {
2646                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2647                                             "cbc(des)))",
2648                                 .cra_driver_name = "echainiv-authenc-"
2649                                                    "hmac-sha512-cbc-des-"
2650                                                    "caam-qi2",
2651                                 .cra_blocksize = DES_BLOCK_SIZE,
2652                         },
2653                         .setkey = aead_setkey,
2654                         .setauthsize = aead_setauthsize,
2655                         .encrypt = aead_encrypt,
2656                         .decrypt = aead_decrypt,
2657                         .ivsize = DES_BLOCK_SIZE,
2658                         .maxauthsize = SHA512_DIGEST_SIZE,
2659                 },
2660                 .caam = {
2661                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2662                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2663                                            OP_ALG_AAI_HMAC_PRECOMP,
2664                         .geniv = true,
2665                 }
2666         },
2667         {
2668                 .aead = {
2669                         .base = {
2670                                 .cra_name = "authenc(hmac(md5),"
2671                                             "rfc3686(ctr(aes)))",
2672                                 .cra_driver_name = "authenc-hmac-md5-"
2673                                                    "rfc3686-ctr-aes-caam-qi2",
2674                                 .cra_blocksize = 1,
2675                         },
2676                         .setkey = aead_setkey,
2677                         .setauthsize = aead_setauthsize,
2678                         .encrypt = aead_encrypt,
2679                         .decrypt = aead_decrypt,
2680                         .ivsize = CTR_RFC3686_IV_SIZE,
2681                         .maxauthsize = MD5_DIGEST_SIZE,
2682                 },
2683                 .caam = {
2684                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2685                                            OP_ALG_AAI_CTR_MOD128,
2686                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2687                                            OP_ALG_AAI_HMAC_PRECOMP,
2688                         .rfc3686 = true,
2689                 },
2690         },
2691         {
2692                 .aead = {
2693                         .base = {
2694                                 .cra_name = "seqiv(authenc("
2695                                             "hmac(md5),rfc3686(ctr(aes))))",
2696                                 .cra_driver_name = "seqiv-authenc-hmac-md5-"
2697                                                    "rfc3686-ctr-aes-caam-qi2",
2698                                 .cra_blocksize = 1,
2699                         },
2700                         .setkey = aead_setkey,
2701                         .setauthsize = aead_setauthsize,
2702                         .encrypt = aead_encrypt,
2703                         .decrypt = aead_decrypt,
2704                         .ivsize = CTR_RFC3686_IV_SIZE,
2705                         .maxauthsize = MD5_DIGEST_SIZE,
2706                 },
2707                 .caam = {
2708                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2709                                            OP_ALG_AAI_CTR_MOD128,
2710                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2711                                            OP_ALG_AAI_HMAC_PRECOMP,
2712                         .rfc3686 = true,
2713                         .geniv = true,
2714                 },
2715         },
2716         {
2717                 .aead = {
2718                         .base = {
2719                                 .cra_name = "authenc(hmac(sha1),"
2720                                             "rfc3686(ctr(aes)))",
2721                                 .cra_driver_name = "authenc-hmac-sha1-"
2722                                                    "rfc3686-ctr-aes-caam-qi2",
2723                                 .cra_blocksize = 1,
2724                         },
2725                         .setkey = aead_setkey,
2726                         .setauthsize = aead_setauthsize,
2727                         .encrypt = aead_encrypt,
2728                         .decrypt = aead_decrypt,
2729                         .ivsize = CTR_RFC3686_IV_SIZE,
2730                         .maxauthsize = SHA1_DIGEST_SIZE,
2731                 },
2732                 .caam = {
2733                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2734                                            OP_ALG_AAI_CTR_MOD128,
2735                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2736                                            OP_ALG_AAI_HMAC_PRECOMP,
2737                         .rfc3686 = true,
2738                 },
2739         },
2740         {
2741                 .aead = {
2742                         .base = {
2743                                 .cra_name = "seqiv(authenc("
2744                                             "hmac(sha1),rfc3686(ctr(aes))))",
2745                                 .cra_driver_name = "seqiv-authenc-hmac-sha1-"
2746                                                    "rfc3686-ctr-aes-caam-qi2",
2747                                 .cra_blocksize = 1,
2748                         },
2749                         .setkey = aead_setkey,
2750                         .setauthsize = aead_setauthsize,
2751                         .encrypt = aead_encrypt,
2752                         .decrypt = aead_decrypt,
2753                         .ivsize = CTR_RFC3686_IV_SIZE,
2754                         .maxauthsize = SHA1_DIGEST_SIZE,
2755                 },
2756                 .caam = {
2757                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2758                                            OP_ALG_AAI_CTR_MOD128,
2759                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2760                                            OP_ALG_AAI_HMAC_PRECOMP,
2761                         .rfc3686 = true,
2762                         .geniv = true,
2763                 },
2764         },
2765         {
2766                 .aead = {
2767                         .base = {
2768                                 .cra_name = "authenc(hmac(sha224),"
2769                                             "rfc3686(ctr(aes)))",
2770                                 .cra_driver_name = "authenc-hmac-sha224-"
2771                                                    "rfc3686-ctr-aes-caam-qi2",
2772                                 .cra_blocksize = 1,
2773                         },
2774                         .setkey = aead_setkey,
2775                         .setauthsize = aead_setauthsize,
2776                         .encrypt = aead_encrypt,
2777                         .decrypt = aead_decrypt,
2778                         .ivsize = CTR_RFC3686_IV_SIZE,
2779                         .maxauthsize = SHA224_DIGEST_SIZE,
2780                 },
2781                 .caam = {
2782                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2783                                            OP_ALG_AAI_CTR_MOD128,
2784                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2785                                            OP_ALG_AAI_HMAC_PRECOMP,
2786                         .rfc3686 = true,
2787                 },
2788         },
2789         {
2790                 .aead = {
2791                         .base = {
2792                                 .cra_name = "seqiv(authenc("
2793                                             "hmac(sha224),rfc3686(ctr(aes))))",
2794                                 .cra_driver_name = "seqiv-authenc-hmac-sha224-"
2795                                                    "rfc3686-ctr-aes-caam-qi2",
2796                                 .cra_blocksize = 1,
2797                         },
2798                         .setkey = aead_setkey,
2799                         .setauthsize = aead_setauthsize,
2800                         .encrypt = aead_encrypt,
2801                         .decrypt = aead_decrypt,
2802                         .ivsize = CTR_RFC3686_IV_SIZE,
2803                         .maxauthsize = SHA224_DIGEST_SIZE,
2804                 },
2805                 .caam = {
2806                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2807                                            OP_ALG_AAI_CTR_MOD128,
2808                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2809                                            OP_ALG_AAI_HMAC_PRECOMP,
2810                         .rfc3686 = true,
2811                         .geniv = true,
2812                 },
2813         },
2814         {
2815                 .aead = {
2816                         .base = {
2817                                 .cra_name = "authenc(hmac(sha256),"
2818                                             "rfc3686(ctr(aes)))",
2819                                 .cra_driver_name = "authenc-hmac-sha256-"
2820                                                    "rfc3686-ctr-aes-caam-qi2",
2821                                 .cra_blocksize = 1,
2822                         },
2823                         .setkey = aead_setkey,
2824                         .setauthsize = aead_setauthsize,
2825                         .encrypt = aead_encrypt,
2826                         .decrypt = aead_decrypt,
2827                         .ivsize = CTR_RFC3686_IV_SIZE,
2828                         .maxauthsize = SHA256_DIGEST_SIZE,
2829                 },
2830                 .caam = {
2831                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2832                                            OP_ALG_AAI_CTR_MOD128,
2833                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2834                                            OP_ALG_AAI_HMAC_PRECOMP,
2835                         .rfc3686 = true,
2836                 },
2837         },
2838         {
2839                 .aead = {
2840                         .base = {
2841                                 .cra_name = "seqiv(authenc(hmac(sha256),"
2842                                             "rfc3686(ctr(aes))))",
2843                                 .cra_driver_name = "seqiv-authenc-hmac-sha256-"
2844                                                    "rfc3686-ctr-aes-caam-qi2",
2845                                 .cra_blocksize = 1,
2846                         },
2847                         .setkey = aead_setkey,
2848                         .setauthsize = aead_setauthsize,
2849                         .encrypt = aead_encrypt,
2850                         .decrypt = aead_decrypt,
2851                         .ivsize = CTR_RFC3686_IV_SIZE,
2852                         .maxauthsize = SHA256_DIGEST_SIZE,
2853                 },
2854                 .caam = {
2855                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2856                                            OP_ALG_AAI_CTR_MOD128,
2857                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2858                                            OP_ALG_AAI_HMAC_PRECOMP,
2859                         .rfc3686 = true,
2860                         .geniv = true,
2861                 },
2862         },
2863         {
2864                 .aead = {
2865                         .base = {
2866                                 .cra_name = "authenc(hmac(sha384),"
2867                                             "rfc3686(ctr(aes)))",
2868                                 .cra_driver_name = "authenc-hmac-sha384-"
2869                                                    "rfc3686-ctr-aes-caam-qi2",
2870                                 .cra_blocksize = 1,
2871                         },
2872                         .setkey = aead_setkey,
2873                         .setauthsize = aead_setauthsize,
2874                         .encrypt = aead_encrypt,
2875                         .decrypt = aead_decrypt,
2876                         .ivsize = CTR_RFC3686_IV_SIZE,
2877                         .maxauthsize = SHA384_DIGEST_SIZE,
2878                 },
2879                 .caam = {
2880                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2881                                            OP_ALG_AAI_CTR_MOD128,
2882                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2883                                            OP_ALG_AAI_HMAC_PRECOMP,
2884                         .rfc3686 = true,
2885                 },
2886         },
2887         {
2888                 .aead = {
2889                         .base = {
2890                                 .cra_name = "seqiv(authenc(hmac(sha384),"
2891                                             "rfc3686(ctr(aes))))",
2892                                 .cra_driver_name = "seqiv-authenc-hmac-sha384-"
2893                                                    "rfc3686-ctr-aes-caam-qi2",
2894                                 .cra_blocksize = 1,
2895                         },
2896                         .setkey = aead_setkey,
2897                         .setauthsize = aead_setauthsize,
2898                         .encrypt = aead_encrypt,
2899                         .decrypt = aead_decrypt,
2900                         .ivsize = CTR_RFC3686_IV_SIZE,
2901                         .maxauthsize = SHA384_DIGEST_SIZE,
2902                 },
2903                 .caam = {
2904                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2905                                            OP_ALG_AAI_CTR_MOD128,
2906                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2907                                            OP_ALG_AAI_HMAC_PRECOMP,
2908                         .rfc3686 = true,
2909                         .geniv = true,
2910                 },
2911         },
2912         {
2913                 .aead = {
2914                         .base = {
2915                                 .cra_name = "rfc7539(chacha20,poly1305)",
2916                                 .cra_driver_name = "rfc7539-chacha20-poly1305-"
2917                                                    "caam-qi2",
2918                                 .cra_blocksize = 1,
2919                         },
2920                         .setkey = chachapoly_setkey,
2921                         .setauthsize = chachapoly_setauthsize,
2922                         .encrypt = aead_encrypt,
2923                         .decrypt = aead_decrypt,
2924                         .ivsize = CHACHAPOLY_IV_SIZE,
2925                         .maxauthsize = POLY1305_DIGEST_SIZE,
2926                 },
2927                 .caam = {
2928                         .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2929                                            OP_ALG_AAI_AEAD,
2930                         .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2931                                            OP_ALG_AAI_AEAD,
2932                         .nodkp = true,
2933                 },
2934         },
2935         {
2936                 .aead = {
2937                         .base = {
2938                                 .cra_name = "rfc7539esp(chacha20,poly1305)",
2939                                 .cra_driver_name = "rfc7539esp-chacha20-"
2940                                                    "poly1305-caam-qi2",
2941                                 .cra_blocksize = 1,
2942                         },
2943                         .setkey = chachapoly_setkey,
2944                         .setauthsize = chachapoly_setauthsize,
2945                         .encrypt = aead_encrypt,
2946                         .decrypt = aead_decrypt,
2947                         .ivsize = 8,
2948                         .maxauthsize = POLY1305_DIGEST_SIZE,
2949                 },
2950                 .caam = {
2951                         .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2952                                            OP_ALG_AAI_AEAD,
2953                         .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2954                                            OP_ALG_AAI_AEAD,
2955                         .nodkp = true,
2956                 },
2957         },
2958         {
2959                 .aead = {
2960                         .base = {
2961                                 .cra_name = "authenc(hmac(sha512),"
2962                                             "rfc3686(ctr(aes)))",
2963                                 .cra_driver_name = "authenc-hmac-sha512-"
2964                                                    "rfc3686-ctr-aes-caam-qi2",
2965                                 .cra_blocksize = 1,
2966                         },
2967                         .setkey = aead_setkey,
2968                         .setauthsize = aead_setauthsize,
2969                         .encrypt = aead_encrypt,
2970                         .decrypt = aead_decrypt,
2971                         .ivsize = CTR_RFC3686_IV_SIZE,
2972                         .maxauthsize = SHA512_DIGEST_SIZE,
2973                 },
2974                 .caam = {
2975                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2976                                            OP_ALG_AAI_CTR_MOD128,
2977                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2978                                            OP_ALG_AAI_HMAC_PRECOMP,
2979                         .rfc3686 = true,
2980                 },
2981         },
2982         {
2983                 .aead = {
2984                         .base = {
2985                                 .cra_name = "seqiv(authenc(hmac(sha512),"
2986                                             "rfc3686(ctr(aes))))",
2987                                 .cra_driver_name = "seqiv-authenc-hmac-sha512-"
2988                                                    "rfc3686-ctr-aes-caam-qi2",
2989                                 .cra_blocksize = 1,
2990                         },
2991                         .setkey = aead_setkey,
2992                         .setauthsize = aead_setauthsize,
2993                         .encrypt = aead_encrypt,
2994                         .decrypt = aead_decrypt,
2995                         .ivsize = CTR_RFC3686_IV_SIZE,
2996                         .maxauthsize = SHA512_DIGEST_SIZE,
2997                 },
2998                 .caam = {
2999                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3000                                            OP_ALG_AAI_CTR_MOD128,
3001                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3002                                            OP_ALG_AAI_HMAC_PRECOMP,
3003                         .rfc3686 = true,
3004                         .geniv = true,
3005                 },
3006         },
3007 };
3008
3009 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
3010 {
3011         struct skcipher_alg *alg = &t_alg->skcipher;
3012
3013         alg->base.cra_module = THIS_MODULE;
3014         alg->base.cra_priority = CAAM_CRA_PRIORITY;
3015         alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding();
3016         alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3017                               CRYPTO_ALG_KERN_DRIVER_ONLY);
3018
3019         alg->init = caam_cra_init_skcipher;
3020         alg->exit = caam_cra_exit;
3021 }
3022
3023 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
3024 {
3025         struct aead_alg *alg = &t_alg->aead;
3026
3027         alg->base.cra_module = THIS_MODULE;
3028         alg->base.cra_priority = CAAM_CRA_PRIORITY;
3029         alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding();
3030         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3031                               CRYPTO_ALG_KERN_DRIVER_ONLY;
3032
3033         alg->init = caam_cra_init_aead;
3034         alg->exit = caam_cra_exit_aead;
3035 }
3036
3037 /* max hash key is max split key size */
3038 #define CAAM_MAX_HASH_KEY_SIZE          (SHA512_DIGEST_SIZE * 2)
3039
3040 #define CAAM_MAX_HASH_BLOCK_SIZE        SHA512_BLOCK_SIZE
3041
3042 /* caam context sizes for hashes: running digest + 8 */
3043 #define HASH_MSG_LEN                    8
3044 #define MAX_CTX_LEN                     (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
3045
3046 enum hash_optype {
3047         UPDATE = 0,
3048         UPDATE_FIRST,
3049         FINALIZE,
3050         DIGEST,
3051         HASH_NUM_OP
3052 };
3053
3054 /**
3055  * struct caam_hash_ctx - ahash per-session context
3056  * @flc: Flow Contexts array
3057  * @key: authentication key
3058  * @flc_dma: I/O virtual addresses of the Flow Contexts
3059  * @dev: dpseci device
3060  * @ctx_len: size of Context Register
3061  * @adata: hashing algorithm details
3062  */
3063 struct caam_hash_ctx {
3064         struct caam_flc flc[HASH_NUM_OP];
3065         u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
3066         dma_addr_t flc_dma[HASH_NUM_OP];
3067         struct device *dev;
3068         int ctx_len;
3069         struct alginfo adata;
3070 };
3071
3072 /* ahash state */
3073 struct caam_hash_state {
3074         struct caam_request caam_req;
3075         dma_addr_t buf_dma;
3076         dma_addr_t ctx_dma;
3077         int ctx_dma_len;
3078         u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
3079         int buflen;
3080         int next_buflen;
3081         u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
3082         int (*update)(struct ahash_request *req);
3083         int (*final)(struct ahash_request *req);
3084         int (*finup)(struct ahash_request *req);
3085 };
3086
3087 struct caam_export_state {
3088         u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
3089         u8 caam_ctx[MAX_CTX_LEN];
3090         int buflen;
3091         int (*update)(struct ahash_request *req);
3092         int (*final)(struct ahash_request *req);
3093         int (*finup)(struct ahash_request *req);
3094 };
3095
3096 /* Map current buffer in state (if length > 0) and put it in link table */
3097 static inline int buf_map_to_qm_sg(struct device *dev,
3098                                    struct dpaa2_sg_entry *qm_sg,
3099                                    struct caam_hash_state *state)
3100 {
3101         int buflen = state->buflen;
3102
3103         if (!buflen)
3104                 return 0;
3105
3106         state->buf_dma = dma_map_single(dev, state->buf, buflen,
3107                                         DMA_TO_DEVICE);
3108         if (dma_mapping_error(dev, state->buf_dma)) {
3109                 dev_err(dev, "unable to map buf\n");
3110                 state->buf_dma = 0;
3111                 return -ENOMEM;
3112         }
3113
3114         dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0);
3115
3116         return 0;
3117 }
3118
3119 /* Map state->caam_ctx, and add it to link table */
3120 static inline int ctx_map_to_qm_sg(struct device *dev,
3121                                    struct caam_hash_state *state, int ctx_len,
3122                                    struct dpaa2_sg_entry *qm_sg, u32 flag)
3123 {
3124         state->ctx_dma_len = ctx_len;
3125         state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag);
3126         if (dma_mapping_error(dev, state->ctx_dma)) {
3127                 dev_err(dev, "unable to map ctx\n");
3128                 state->ctx_dma = 0;
3129                 return -ENOMEM;
3130         }
3131
3132         dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0);
3133
3134         return 0;
3135 }
3136
3137 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
3138 {
3139         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3140         int digestsize = crypto_ahash_digestsize(ahash);
3141         struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
3142         struct caam_flc *flc;
3143         u32 *desc;
3144
3145         /* ahash_update shared descriptor */
3146         flc = &ctx->flc[UPDATE];
3147         desc = flc->sh_desc;
3148         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len,
3149                           ctx->ctx_len, true, priv->sec_attr.era);
3150         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3151         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE],
3152                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3153         print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ",
3154                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3155                              1);
3156
3157         /* ahash_update_first shared descriptor */
3158         flc = &ctx->flc[UPDATE_FIRST];
3159         desc = flc->sh_desc;
3160         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
3161                           ctx->ctx_len, false, priv->sec_attr.era);
3162         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3163         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST],
3164                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3165         print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ",
3166                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3167                              1);
3168
3169         /* ahash_final shared descriptor */
3170         flc = &ctx->flc[FINALIZE];
3171         desc = flc->sh_desc;
3172         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize,
3173                           ctx->ctx_len, true, priv->sec_attr.era);
3174         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3175         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE],
3176                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3177         print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ",
3178                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3179                              1);
3180
3181         /* ahash_digest shared descriptor */
3182         flc = &ctx->flc[DIGEST];
3183         desc = flc->sh_desc;
3184         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize,
3185                           ctx->ctx_len, false, priv->sec_attr.era);
3186         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3187         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST],
3188                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3189         print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ",
3190                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3191                              1);
3192
3193         return 0;
3194 }
3195
3196 struct split_key_sh_result {
3197         struct completion completion;
3198         int err;
3199         struct device *dev;
3200 };
3201
3202 static void split_key_sh_done(void *cbk_ctx, u32 err)
3203 {
3204         struct split_key_sh_result *res = cbk_ctx;
3205
3206         dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
3207
3208         res->err = err ? caam_qi2_strstatus(res->dev, err) : 0;
3209         complete(&res->completion);
3210 }
3211
3212 /* Digest hash size if it is too large */
3213 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
3214                            u32 digestsize)
3215 {
3216         struct caam_request *req_ctx;
3217         u32 *desc;
3218         struct split_key_sh_result result;
3219         dma_addr_t key_dma;
3220         struct caam_flc *flc;
3221         dma_addr_t flc_dma;
3222         int ret = -ENOMEM;
3223         struct dpaa2_fl_entry *in_fle, *out_fle;
3224
3225         req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL);
3226         if (!req_ctx)
3227                 return -ENOMEM;
3228
3229         in_fle = &req_ctx->fd_flt[1];
3230         out_fle = &req_ctx->fd_flt[0];
3231
3232         flc = kzalloc(sizeof(*flc), GFP_KERNEL);
3233         if (!flc)
3234                 goto err_flc;
3235
3236         key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL);
3237         if (dma_mapping_error(ctx->dev, key_dma)) {
3238                 dev_err(ctx->dev, "unable to map key memory\n");
3239                 goto err_key_dma;
3240         }
3241
3242         desc = flc->sh_desc;
3243
3244         init_sh_desc(desc, 0);
3245
3246         /* descriptor to perform unkeyed hash on key_in */
3247         append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
3248                          OP_ALG_AS_INITFINAL);
3249         append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
3250                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
3251         append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
3252                          LDST_SRCDST_BYTE_CONTEXT);
3253
3254         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3255         flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) +
3256                                  desc_bytes(desc), DMA_TO_DEVICE);
3257         if (dma_mapping_error(ctx->dev, flc_dma)) {
3258                 dev_err(ctx->dev, "unable to map shared descriptor\n");
3259                 goto err_flc_dma;
3260         }
3261
3262         dpaa2_fl_set_final(in_fle, true);
3263         dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3264         dpaa2_fl_set_addr(in_fle, key_dma);
3265         dpaa2_fl_set_len(in_fle, *keylen);
3266         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3267         dpaa2_fl_set_addr(out_fle, key_dma);
3268         dpaa2_fl_set_len(out_fle, digestsize);
3269
3270         print_hex_dump_debug("key_in@" __stringify(__LINE__)": ",
3271                              DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1);
3272         print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ",
3273                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3274                              1);
3275
3276         result.err = 0;
3277         init_completion(&result.completion);
3278         result.dev = ctx->dev;
3279
3280         req_ctx->flc = flc;
3281         req_ctx->flc_dma = flc_dma;
3282         req_ctx->cbk = split_key_sh_done;
3283         req_ctx->ctx = &result;
3284
3285         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3286         if (ret == -EINPROGRESS) {
3287                 /* in progress */
3288                 wait_for_completion(&result.completion);
3289                 ret = result.err;
3290                 print_hex_dump_debug("digested key@" __stringify(__LINE__)": ",
3291                                      DUMP_PREFIX_ADDRESS, 16, 4, key,
3292                                      digestsize, 1);
3293         }
3294
3295         dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc),
3296                          DMA_TO_DEVICE);
3297 err_flc_dma:
3298         dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL);
3299 err_key_dma:
3300         kfree(flc);
3301 err_flc:
3302         kfree(req_ctx);
3303
3304         *keylen = digestsize;
3305
3306         return ret;
3307 }
3308
3309 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
3310                         unsigned int keylen)
3311 {
3312         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3313         unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
3314         unsigned int digestsize = crypto_ahash_digestsize(ahash);
3315         int ret;
3316         u8 *hashed_key = NULL;
3317
3318         dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize);
3319
3320         if (keylen > blocksize) {
3321                 unsigned int aligned_len =
3322                         ALIGN(keylen, dma_get_cache_alignment());
3323
3324                 if (aligned_len < keylen)
3325                         return -EOVERFLOW;
3326
3327                 hashed_key = kmemdup(key, aligned_len, GFP_KERNEL);
3328                 if (!hashed_key)
3329                         return -ENOMEM;
3330                 ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize);
3331                 if (ret)
3332                         goto bad_free_key;
3333                 key = hashed_key;
3334         }
3335
3336         ctx->adata.keylen = keylen;
3337         ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
3338                                               OP_ALG_ALGSEL_MASK);
3339         if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE)
3340                 goto bad_free_key;
3341
3342         ctx->adata.key_virt = key;
3343         ctx->adata.key_inline = true;
3344
3345         /*
3346          * In case |user key| > |derived key|, using DKP<imm,imm> would result
3347          * in invalid opcodes (last bytes of user key) in the resulting
3348          * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
3349          * addresses are needed.
3350          */
3351         if (keylen > ctx->adata.keylen_pad) {
3352                 memcpy(ctx->key, key, keylen);
3353                 dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma,
3354                                            ctx->adata.keylen_pad,
3355                                            DMA_TO_DEVICE);
3356         }
3357
3358         ret = ahash_set_sh_desc(ahash);
3359         kfree(hashed_key);
3360         return ret;
3361 bad_free_key:
3362         kfree(hashed_key);
3363         return -EINVAL;
3364 }
3365
3366 static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc,
3367                                struct ahash_request *req)
3368 {
3369         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3370
3371         if (edesc->src_nents)
3372                 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
3373
3374         if (edesc->qm_sg_bytes)
3375                 dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes,
3376                                  DMA_TO_DEVICE);
3377
3378         if (state->buf_dma) {
3379                 dma_unmap_single(dev, state->buf_dma, state->buflen,
3380                                  DMA_TO_DEVICE);
3381                 state->buf_dma = 0;
3382         }
3383 }
3384
3385 static inline void ahash_unmap_ctx(struct device *dev,
3386                                    struct ahash_edesc *edesc,
3387                                    struct ahash_request *req, u32 flag)
3388 {
3389         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3390
3391         if (state->ctx_dma) {
3392                 dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag);
3393                 state->ctx_dma = 0;
3394         }
3395         ahash_unmap(dev, edesc, req);
3396 }
3397
3398 static void ahash_done(void *cbk_ctx, u32 status)
3399 {
3400         struct crypto_async_request *areq = cbk_ctx;
3401         struct ahash_request *req = ahash_request_cast(areq);
3402         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3403         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3404         struct ahash_edesc *edesc = state->caam_req.edesc;
3405         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3406         int digestsize = crypto_ahash_digestsize(ahash);
3407         int ecode = 0;
3408
3409         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3410
3411         if (unlikely(status))
3412                 ecode = caam_qi2_strstatus(ctx->dev, status);
3413
3414         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3415         memcpy(req->result, state->caam_ctx, digestsize);
3416         qi_cache_free(edesc);
3417
3418         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3419                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3420                              ctx->ctx_len, 1);
3421
3422         ahash_request_complete(req, ecode);
3423 }
3424
3425 static void ahash_done_bi(void *cbk_ctx, u32 status)
3426 {
3427         struct crypto_async_request *areq = cbk_ctx;
3428         struct ahash_request *req = ahash_request_cast(areq);
3429         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3430         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3431         struct ahash_edesc *edesc = state->caam_req.edesc;
3432         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3433         int ecode = 0;
3434
3435         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3436
3437         if (unlikely(status))
3438                 ecode = caam_qi2_strstatus(ctx->dev, status);
3439
3440         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3441         qi_cache_free(edesc);
3442
3443         scatterwalk_map_and_copy(state->buf, req->src,
3444                                  req->nbytes - state->next_buflen,
3445                                  state->next_buflen, 0);
3446         state->buflen = state->next_buflen;
3447
3448         print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3449                              DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
3450                              state->buflen, 1);
3451
3452         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3453                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3454                              ctx->ctx_len, 1);
3455         if (req->result)
3456                 print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3457                                      DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3458                                      crypto_ahash_digestsize(ahash), 1);
3459
3460         ahash_request_complete(req, ecode);
3461 }
3462
3463 static void ahash_done_ctx_src(void *cbk_ctx, u32 status)
3464 {
3465         struct crypto_async_request *areq = cbk_ctx;
3466         struct ahash_request *req = ahash_request_cast(areq);
3467         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3468         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3469         struct ahash_edesc *edesc = state->caam_req.edesc;
3470         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3471         int digestsize = crypto_ahash_digestsize(ahash);
3472         int ecode = 0;
3473
3474         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3475
3476         if (unlikely(status))
3477                 ecode = caam_qi2_strstatus(ctx->dev, status);
3478
3479         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3480         memcpy(req->result, state->caam_ctx, digestsize);
3481         qi_cache_free(edesc);
3482
3483         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3484                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3485                              ctx->ctx_len, 1);
3486
3487         ahash_request_complete(req, ecode);
3488 }
3489
3490 static void ahash_done_ctx_dst(void *cbk_ctx, u32 status)
3491 {
3492         struct crypto_async_request *areq = cbk_ctx;
3493         struct ahash_request *req = ahash_request_cast(areq);
3494         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3495         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3496         struct ahash_edesc *edesc = state->caam_req.edesc;
3497         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3498         int ecode = 0;
3499
3500         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3501
3502         if (unlikely(status))
3503                 ecode = caam_qi2_strstatus(ctx->dev, status);
3504
3505         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3506         qi_cache_free(edesc);
3507
3508         scatterwalk_map_and_copy(state->buf, req->src,
3509                                  req->nbytes - state->next_buflen,
3510                                  state->next_buflen, 0);
3511         state->buflen = state->next_buflen;
3512
3513         print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3514                              DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
3515                              state->buflen, 1);
3516
3517         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3518                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3519                              ctx->ctx_len, 1);
3520         if (req->result)
3521                 print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3522                                      DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3523                                      crypto_ahash_digestsize(ahash), 1);
3524
3525         ahash_request_complete(req, ecode);
3526 }
3527
3528 static int ahash_update_ctx(struct ahash_request *req)
3529 {
3530         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3531         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3532         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3533         struct caam_request *req_ctx = &state->caam_req;
3534         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3535         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3536         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3537                       GFP_KERNEL : GFP_ATOMIC;
3538         u8 *buf = state->buf;
3539         int *buflen = &state->buflen;
3540         int *next_buflen = &state->next_buflen;
3541         int in_len = *buflen + req->nbytes, to_hash;
3542         int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index;
3543         struct ahash_edesc *edesc;
3544         int ret = 0;
3545
3546         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
3547         to_hash = in_len - *next_buflen;
3548
3549         if (to_hash) {
3550                 struct dpaa2_sg_entry *sg_table;
3551                 int src_len = req->nbytes - *next_buflen;
3552
3553                 src_nents = sg_nents_for_len(req->src, src_len);
3554                 if (src_nents < 0) {
3555                         dev_err(ctx->dev, "Invalid number of src SG.\n");
3556                         return src_nents;
3557                 }
3558
3559                 if (src_nents) {
3560                         mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3561                                                   DMA_TO_DEVICE);
3562                         if (!mapped_nents) {
3563                                 dev_err(ctx->dev, "unable to DMA map source\n");
3564                                 return -ENOMEM;
3565                         }
3566                 } else {
3567                         mapped_nents = 0;
3568                 }
3569
3570                 /* allocate space for base edesc and link tables */
3571                 edesc = qi_cache_zalloc(flags);
3572                 if (!edesc) {
3573                         dma_unmap_sg(ctx->dev, req->src, src_nents,
3574                                      DMA_TO_DEVICE);
3575                         return -ENOMEM;
3576                 }
3577
3578                 edesc->src_nents = src_nents;
3579                 qm_sg_src_index = 1 + (*buflen ? 1 : 0);
3580                 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3581                               sizeof(*sg_table);
3582                 sg_table = &edesc->sgt[0];
3583
3584                 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3585                                        DMA_BIDIRECTIONAL);
3586                 if (ret)
3587                         goto unmap_ctx;
3588
3589                 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3590                 if (ret)
3591                         goto unmap_ctx;
3592
3593                 if (mapped_nents) {
3594                         sg_to_qm_sg_last(req->src, src_len,
3595                                          sg_table + qm_sg_src_index, 0);
3596                 } else {
3597                         dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1,
3598                                            true);
3599                 }
3600
3601                 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3602                                                   qm_sg_bytes, DMA_TO_DEVICE);
3603                 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3604                         dev_err(ctx->dev, "unable to map S/G table\n");
3605                         ret = -ENOMEM;
3606                         goto unmap_ctx;
3607                 }
3608                 edesc->qm_sg_bytes = qm_sg_bytes;
3609
3610                 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3611                 dpaa2_fl_set_final(in_fle, true);
3612                 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3613                 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3614                 dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash);
3615                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3616                 dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3617                 dpaa2_fl_set_len(out_fle, ctx->ctx_len);
3618
3619                 req_ctx->flc = &ctx->flc[UPDATE];
3620                 req_ctx->flc_dma = ctx->flc_dma[UPDATE];
3621                 req_ctx->cbk = ahash_done_bi;
3622                 req_ctx->ctx = &req->base;
3623                 req_ctx->edesc = edesc;
3624
3625                 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3626                 if (ret != -EINPROGRESS &&
3627                     !(ret == -EBUSY &&
3628                       req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3629                         goto unmap_ctx;
3630         } else if (*next_buflen) {
3631                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
3632                                          req->nbytes, 0);
3633                 *buflen = *next_buflen;
3634
3635                 print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3636                                      DUMP_PREFIX_ADDRESS, 16, 4, buf,
3637                                      *buflen, 1);
3638         }
3639
3640         return ret;
3641 unmap_ctx:
3642         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3643         qi_cache_free(edesc);
3644         return ret;
3645 }
3646
3647 static int ahash_final_ctx(struct ahash_request *req)
3648 {
3649         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3650         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3651         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3652         struct caam_request *req_ctx = &state->caam_req;
3653         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3654         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3655         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3656                       GFP_KERNEL : GFP_ATOMIC;
3657         int buflen = state->buflen;
3658         int qm_sg_bytes;
3659         int digestsize = crypto_ahash_digestsize(ahash);
3660         struct ahash_edesc *edesc;
3661         struct dpaa2_sg_entry *sg_table;
3662         int ret;
3663
3664         /* allocate space for base edesc and link tables */
3665         edesc = qi_cache_zalloc(flags);
3666         if (!edesc)
3667                 return -ENOMEM;
3668
3669         qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table);
3670         sg_table = &edesc->sgt[0];
3671
3672         ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3673                                DMA_BIDIRECTIONAL);
3674         if (ret)
3675                 goto unmap_ctx;
3676
3677         ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3678         if (ret)
3679                 goto unmap_ctx;
3680
3681         dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true);
3682
3683         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3684                                           DMA_TO_DEVICE);
3685         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3686                 dev_err(ctx->dev, "unable to map S/G table\n");
3687                 ret = -ENOMEM;
3688                 goto unmap_ctx;
3689         }
3690         edesc->qm_sg_bytes = qm_sg_bytes;
3691
3692         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3693         dpaa2_fl_set_final(in_fle, true);
3694         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3695         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3696         dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen);
3697         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3698         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3699         dpaa2_fl_set_len(out_fle, digestsize);
3700
3701         req_ctx->flc = &ctx->flc[FINALIZE];
3702         req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3703         req_ctx->cbk = ahash_done_ctx_src;
3704         req_ctx->ctx = &req->base;
3705         req_ctx->edesc = edesc;
3706
3707         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3708         if (ret == -EINPROGRESS ||
3709             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3710                 return ret;
3711
3712 unmap_ctx:
3713         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3714         qi_cache_free(edesc);
3715         return ret;
3716 }
3717
3718 static int ahash_finup_ctx(struct ahash_request *req)
3719 {
3720         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3721         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3722         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3723         struct caam_request *req_ctx = &state->caam_req;
3724         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3725         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3726         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3727                       GFP_KERNEL : GFP_ATOMIC;
3728         int buflen = state->buflen;
3729         int qm_sg_bytes, qm_sg_src_index;
3730         int src_nents, mapped_nents;
3731         int digestsize = crypto_ahash_digestsize(ahash);
3732         struct ahash_edesc *edesc;
3733         struct dpaa2_sg_entry *sg_table;
3734         int ret;
3735
3736         src_nents = sg_nents_for_len(req->src, req->nbytes);
3737         if (src_nents < 0) {
3738                 dev_err(ctx->dev, "Invalid number of src SG.\n");
3739                 return src_nents;
3740         }
3741
3742         if (src_nents) {
3743                 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3744                                           DMA_TO_DEVICE);
3745                 if (!mapped_nents) {
3746                         dev_err(ctx->dev, "unable to DMA map source\n");
3747                         return -ENOMEM;
3748                 }
3749         } else {
3750                 mapped_nents = 0;
3751         }
3752
3753         /* allocate space for base edesc and link tables */
3754         edesc = qi_cache_zalloc(flags);
3755         if (!edesc) {
3756                 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3757                 return -ENOMEM;
3758         }
3759
3760         edesc->src_nents = src_nents;
3761         qm_sg_src_index = 1 + (buflen ? 1 : 0);
3762         qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3763                       sizeof(*sg_table);
3764         sg_table = &edesc->sgt[0];
3765
3766         ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3767                                DMA_BIDIRECTIONAL);
3768         if (ret)
3769                 goto unmap_ctx;
3770
3771         ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3772         if (ret)
3773                 goto unmap_ctx;
3774
3775         sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0);
3776
3777         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3778                                           DMA_TO_DEVICE);
3779         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3780                 dev_err(ctx->dev, "unable to map S/G table\n");
3781                 ret = -ENOMEM;
3782                 goto unmap_ctx;
3783         }
3784         edesc->qm_sg_bytes = qm_sg_bytes;
3785
3786         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3787         dpaa2_fl_set_final(in_fle, true);
3788         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3789         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3790         dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes);
3791         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3792         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3793         dpaa2_fl_set_len(out_fle, digestsize);
3794
3795         req_ctx->flc = &ctx->flc[FINALIZE];
3796         req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3797         req_ctx->cbk = ahash_done_ctx_src;
3798         req_ctx->ctx = &req->base;
3799         req_ctx->edesc = edesc;
3800
3801         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3802         if (ret == -EINPROGRESS ||
3803             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3804                 return ret;
3805
3806 unmap_ctx:
3807         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3808         qi_cache_free(edesc);
3809         return ret;
3810 }
3811
3812 static int ahash_digest(struct ahash_request *req)
3813 {
3814         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3815         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3816         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3817         struct caam_request *req_ctx = &state->caam_req;
3818         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3819         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3820         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3821                       GFP_KERNEL : GFP_ATOMIC;
3822         int digestsize = crypto_ahash_digestsize(ahash);
3823         int src_nents, mapped_nents;
3824         struct ahash_edesc *edesc;
3825         int ret = -ENOMEM;
3826
3827         state->buf_dma = 0;
3828
3829         src_nents = sg_nents_for_len(req->src, req->nbytes);
3830         if (src_nents < 0) {
3831                 dev_err(ctx->dev, "Invalid number of src SG.\n");
3832                 return src_nents;
3833         }
3834
3835         if (src_nents) {
3836                 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3837                                           DMA_TO_DEVICE);
3838                 if (!mapped_nents) {
3839                         dev_err(ctx->dev, "unable to map source for DMA\n");
3840                         return ret;
3841                 }
3842         } else {
3843                 mapped_nents = 0;
3844         }
3845
3846         /* allocate space for base edesc and link tables */
3847         edesc = qi_cache_zalloc(flags);
3848         if (!edesc) {
3849                 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3850                 return ret;
3851         }
3852
3853         edesc->src_nents = src_nents;
3854         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3855
3856         if (mapped_nents > 1) {
3857                 int qm_sg_bytes;
3858                 struct dpaa2_sg_entry *sg_table = &edesc->sgt[0];
3859
3860                 qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table);
3861                 sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0);
3862                 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3863                                                   qm_sg_bytes, DMA_TO_DEVICE);
3864                 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3865                         dev_err(ctx->dev, "unable to map S/G table\n");
3866                         goto unmap;
3867                 }
3868                 edesc->qm_sg_bytes = qm_sg_bytes;
3869                 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3870                 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3871         } else {
3872                 dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3873                 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
3874         }
3875
3876         state->ctx_dma_len = digestsize;
3877         state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3878                                         DMA_FROM_DEVICE);
3879         if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3880                 dev_err(ctx->dev, "unable to map ctx\n");
3881                 state->ctx_dma = 0;
3882                 goto unmap;
3883         }
3884
3885         dpaa2_fl_set_final(in_fle, true);
3886         dpaa2_fl_set_len(in_fle, req->nbytes);
3887         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3888         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3889         dpaa2_fl_set_len(out_fle, digestsize);
3890
3891         req_ctx->flc = &ctx->flc[DIGEST];
3892         req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3893         req_ctx->cbk = ahash_done;
3894         req_ctx->ctx = &req->base;
3895         req_ctx->edesc = edesc;
3896         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3897         if (ret == -EINPROGRESS ||
3898             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3899                 return ret;
3900
3901 unmap:
3902         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3903         qi_cache_free(edesc);
3904         return ret;
3905 }
3906
3907 static int ahash_final_no_ctx(struct ahash_request *req)
3908 {
3909         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3910         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3911         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3912         struct caam_request *req_ctx = &state->caam_req;
3913         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3914         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3915         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3916                       GFP_KERNEL : GFP_ATOMIC;
3917         u8 *buf = state->buf;
3918         int buflen = state->buflen;
3919         int digestsize = crypto_ahash_digestsize(ahash);
3920         struct ahash_edesc *edesc;
3921         int ret = -ENOMEM;
3922
3923         /* allocate space for base edesc and link tables */
3924         edesc = qi_cache_zalloc(flags);
3925         if (!edesc)
3926                 return ret;
3927
3928         if (buflen) {
3929                 state->buf_dma = dma_map_single(ctx->dev, buf, buflen,
3930                                                 DMA_TO_DEVICE);
3931                 if (dma_mapping_error(ctx->dev, state->buf_dma)) {
3932                         dev_err(ctx->dev, "unable to map src\n");
3933                         goto unmap;
3934                 }
3935         }
3936
3937         state->ctx_dma_len = digestsize;
3938         state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3939                                         DMA_FROM_DEVICE);
3940         if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3941                 dev_err(ctx->dev, "unable to map ctx\n");
3942                 state->ctx_dma = 0;
3943                 goto unmap;
3944         }
3945
3946         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3947         dpaa2_fl_set_final(in_fle, true);
3948         /*
3949          * crypto engine requires the input entry to be present when
3950          * "frame list" FD is used.
3951          * Since engine does not support FMT=2'b11 (unused entry type), leaving
3952          * in_fle zeroized (except for "Final" flag) is the best option.
3953          */
3954         if (buflen) {
3955                 dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3956                 dpaa2_fl_set_addr(in_fle, state->buf_dma);
3957                 dpaa2_fl_set_len(in_fle, buflen);
3958         }
3959         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3960         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3961         dpaa2_fl_set_len(out_fle, digestsize);
3962
3963         req_ctx->flc = &ctx->flc[DIGEST];
3964         req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3965         req_ctx->cbk = ahash_done;
3966         req_ctx->ctx = &req->base;
3967         req_ctx->edesc = edesc;
3968
3969         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3970         if (ret == -EINPROGRESS ||
3971             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3972                 return ret;
3973
3974 unmap:
3975         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3976         qi_cache_free(edesc);
3977         return ret;
3978 }
3979
3980 static int ahash_update_no_ctx(struct ahash_request *req)
3981 {
3982         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3983         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3984         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3985         struct caam_request *req_ctx = &state->caam_req;
3986         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3987         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3988         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3989                       GFP_KERNEL : GFP_ATOMIC;
3990         u8 *buf = state->buf;
3991         int *buflen = &state->buflen;
3992         int *next_buflen = &state->next_buflen;
3993         int in_len = *buflen + req->nbytes, to_hash;
3994         int qm_sg_bytes, src_nents, mapped_nents;
3995         struct ahash_edesc *edesc;
3996         int ret = 0;
3997
3998         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
3999         to_hash = in_len - *next_buflen;
4000
4001         if (to_hash) {
4002                 struct dpaa2_sg_entry *sg_table;
4003                 int src_len = req->nbytes - *next_buflen;
4004
4005                 src_nents = sg_nents_for_len(req->src, src_len);
4006                 if (src_nents < 0) {
4007                         dev_err(ctx->dev, "Invalid number of src SG.\n");
4008                         return src_nents;
4009                 }
4010
4011                 if (src_nents) {
4012                         mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4013                                                   DMA_TO_DEVICE);
4014                         if (!mapped_nents) {
4015                                 dev_err(ctx->dev, "unable to DMA map source\n");
4016                                 return -ENOMEM;
4017                         }
4018                 } else {
4019                         mapped_nents = 0;
4020                 }
4021
4022                 /* allocate space for base edesc and link tables */
4023                 edesc = qi_cache_zalloc(flags);
4024                 if (!edesc) {
4025                         dma_unmap_sg(ctx->dev, req->src, src_nents,
4026                                      DMA_TO_DEVICE);
4027                         return -ENOMEM;
4028                 }
4029
4030                 edesc->src_nents = src_nents;
4031                 qm_sg_bytes = pad_sg_nents(1 + mapped_nents) *
4032                               sizeof(*sg_table);
4033                 sg_table = &edesc->sgt[0];
4034
4035                 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
4036                 if (ret)
4037                         goto unmap_ctx;
4038
4039                 sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0);
4040
4041                 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
4042                                                   qm_sg_bytes, DMA_TO_DEVICE);
4043                 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4044                         dev_err(ctx->dev, "unable to map S/G table\n");
4045                         ret = -ENOMEM;
4046                         goto unmap_ctx;
4047                 }
4048                 edesc->qm_sg_bytes = qm_sg_bytes;
4049
4050                 state->ctx_dma_len = ctx->ctx_len;
4051                 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
4052                                                 ctx->ctx_len, DMA_FROM_DEVICE);
4053                 if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4054                         dev_err(ctx->dev, "unable to map ctx\n");
4055                         state->ctx_dma = 0;
4056                         ret = -ENOMEM;
4057                         goto unmap_ctx;
4058                 }
4059
4060                 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4061                 dpaa2_fl_set_final(in_fle, true);
4062                 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4063                 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4064                 dpaa2_fl_set_len(in_fle, to_hash);
4065                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4066                 dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4067                 dpaa2_fl_set_len(out_fle, ctx->ctx_len);
4068
4069                 req_ctx->flc = &ctx->flc[UPDATE_FIRST];
4070                 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
4071                 req_ctx->cbk = ahash_done_ctx_dst;
4072                 req_ctx->ctx = &req->base;
4073                 req_ctx->edesc = edesc;
4074
4075                 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4076                 if (ret != -EINPROGRESS &&
4077                     !(ret == -EBUSY &&
4078                       req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
4079                         goto unmap_ctx;
4080
4081                 state->update = ahash_update_ctx;
4082                 state->finup = ahash_finup_ctx;
4083                 state->final = ahash_final_ctx;
4084         } else if (*next_buflen) {
4085                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
4086                                          req->nbytes, 0);
4087                 *buflen = *next_buflen;
4088
4089                 print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
4090                                      DUMP_PREFIX_ADDRESS, 16, 4, buf,
4091                                      *buflen, 1);
4092         }
4093
4094         return ret;
4095 unmap_ctx:
4096         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
4097         qi_cache_free(edesc);
4098         return ret;
4099 }
4100
4101 static int ahash_finup_no_ctx(struct ahash_request *req)
4102 {
4103         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
4104         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
4105         struct caam_hash_state *state = ahash_request_ctx_dma(req);
4106         struct caam_request *req_ctx = &state->caam_req;
4107         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
4108         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4109         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4110                       GFP_KERNEL : GFP_ATOMIC;
4111         int buflen = state->buflen;
4112         int qm_sg_bytes, src_nents, mapped_nents;
4113         int digestsize = crypto_ahash_digestsize(ahash);
4114         struct ahash_edesc *edesc;
4115         struct dpaa2_sg_entry *sg_table;
4116         int ret = -ENOMEM;
4117
4118         src_nents = sg_nents_for_len(req->src, req->nbytes);
4119         if (src_nents < 0) {
4120                 dev_err(ctx->dev, "Invalid number of src SG.\n");
4121                 return src_nents;
4122         }
4123
4124         if (src_nents) {
4125                 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4126                                           DMA_TO_DEVICE);
4127                 if (!mapped_nents) {
4128                         dev_err(ctx->dev, "unable to DMA map source\n");
4129                         return ret;
4130                 }
4131         } else {
4132                 mapped_nents = 0;
4133         }
4134
4135         /* allocate space for base edesc and link tables */
4136         edesc = qi_cache_zalloc(flags);
4137         if (!edesc) {
4138                 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
4139                 return ret;
4140         }
4141
4142         edesc->src_nents = src_nents;
4143         qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table);
4144         sg_table = &edesc->sgt[0];
4145
4146         ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
4147         if (ret)
4148                 goto unmap;
4149
4150         sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0);
4151
4152         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
4153                                           DMA_TO_DEVICE);
4154         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4155                 dev_err(ctx->dev, "unable to map S/G table\n");
4156                 ret = -ENOMEM;
4157                 goto unmap;
4158         }
4159         edesc->qm_sg_bytes = qm_sg_bytes;
4160
4161         state->ctx_dma_len = digestsize;
4162         state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
4163                                         DMA_FROM_DEVICE);
4164         if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4165                 dev_err(ctx->dev, "unable to map ctx\n");
4166                 state->ctx_dma = 0;
4167                 ret = -ENOMEM;
4168                 goto unmap;
4169         }
4170
4171         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4172         dpaa2_fl_set_final(in_fle, true);
4173         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4174         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4175         dpaa2_fl_set_len(in_fle, buflen + req->nbytes);
4176         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4177         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4178         dpaa2_fl_set_len(out_fle, digestsize);
4179
4180         req_ctx->flc = &ctx->flc[DIGEST];
4181         req_ctx->flc_dma = ctx->flc_dma[DIGEST];
4182         req_ctx->cbk = ahash_done;
4183         req_ctx->ctx = &req->base;
4184         req_ctx->edesc = edesc;
4185         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4186         if (ret != -EINPROGRESS &&
4187             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
4188                 goto unmap;
4189
4190         return ret;
4191 unmap:
4192         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
4193         qi_cache_free(edesc);
4194         return ret;
4195 }
4196
4197 static int ahash_update_first(struct ahash_request *req)
4198 {
4199         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
4200         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
4201         struct caam_hash_state *state = ahash_request_ctx_dma(req);
4202         struct caam_request *req_ctx = &state->caam_req;
4203         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
4204         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4205         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4206                       GFP_KERNEL : GFP_ATOMIC;
4207         u8 *buf = state->buf;
4208         int *buflen = &state->buflen;
4209         int *next_buflen = &state->next_buflen;
4210         int to_hash;
4211         int src_nents, mapped_nents;
4212         struct ahash_edesc *edesc;
4213         int ret = 0;
4214
4215         *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
4216                                       1);
4217         to_hash = req->nbytes - *next_buflen;
4218
4219         if (to_hash) {
4220                 struct dpaa2_sg_entry *sg_table;
4221                 int src_len = req->nbytes - *next_buflen;
4222
4223                 src_nents = sg_nents_for_len(req->src, src_len);
4224                 if (src_nents < 0) {
4225                         dev_err(ctx->dev, "Invalid number of src SG.\n");
4226                         return src_nents;
4227                 }
4228
4229                 if (src_nents) {
4230                         mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4231                                                   DMA_TO_DEVICE);
4232                         if (!mapped_nents) {
4233                                 dev_err(ctx->dev, "unable to map source for DMA\n");
4234                                 return -ENOMEM;
4235                         }
4236                 } else {
4237                         mapped_nents = 0;
4238                 }
4239
4240                 /* allocate space for base edesc and link tables */
4241                 edesc = qi_cache_zalloc(flags);
4242                 if (!edesc) {
4243                         dma_unmap_sg(ctx->dev, req->src, src_nents,
4244                                      DMA_TO_DEVICE);
4245                         return -ENOMEM;
4246                 }
4247
4248                 edesc->src_nents = src_nents;
4249                 sg_table = &edesc->sgt[0];
4250
4251                 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4252                 dpaa2_fl_set_final(in_fle, true);
4253                 dpaa2_fl_set_len(in_fle, to_hash);
4254
4255                 if (mapped_nents > 1) {
4256                         int qm_sg_bytes;
4257
4258                         sg_to_qm_sg_last(req->src, src_len, sg_table, 0);
4259                         qm_sg_bytes = pad_sg_nents(mapped_nents) *
4260                                       sizeof(*sg_table);
4261                         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
4262                                                           qm_sg_bytes,
4263                                                           DMA_TO_DEVICE);
4264                         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4265                                 dev_err(ctx->dev, "unable to map S/G table\n");
4266                                 ret = -ENOMEM;
4267                                 goto unmap_ctx;
4268                         }
4269                         edesc->qm_sg_bytes = qm_sg_bytes;
4270                         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4271                         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4272                 } else {
4273                         dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
4274                         dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
4275                 }
4276
4277                 state->ctx_dma_len = ctx->ctx_len;
4278                 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
4279                                                 ctx->ctx_len, DMA_FROM_DEVICE);
4280                 if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4281                         dev_err(ctx->dev, "unable to map ctx\n");
4282                         state->ctx_dma = 0;
4283                         ret = -ENOMEM;
4284                         goto unmap_ctx;
4285                 }
4286
4287                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4288                 dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4289                 dpaa2_fl_set_len(out_fle, ctx->ctx_len);
4290
4291                 req_ctx->flc = &ctx->flc[UPDATE_FIRST];
4292                 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
4293                 req_ctx->cbk = ahash_done_ctx_dst;
4294                 req_ctx->ctx = &req->base;
4295                 req_ctx->edesc = edesc;
4296
4297                 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4298                 if (ret != -EINPROGRESS &&
4299                     !(ret == -EBUSY && req->base.flags &
4300                       CRYPTO_TFM_REQ_MAY_BACKLOG))
4301                         goto unmap_ctx;
4302
4303                 state->update = ahash_update_ctx;
4304                 state->finup = ahash_finup_ctx;
4305                 state->final = ahash_final_ctx;
4306         } else if (*next_buflen) {
4307                 state->update = ahash_update_no_ctx;
4308                 state->finup = ahash_finup_no_ctx;
4309                 state->final = ahash_final_no_ctx;
4310                 scatterwalk_map_and_copy(buf, req->src, 0,
4311                                          req->nbytes, 0);
4312                 *buflen = *next_buflen;
4313
4314                 print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
4315                                      DUMP_PREFIX_ADDRESS, 16, 4, buf,
4316                                      *buflen, 1);
4317         }
4318
4319         return ret;
4320 unmap_ctx:
4321         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
4322         qi_cache_free(edesc);
4323         return ret;
4324 }
4325
4326 static int ahash_finup_first(struct ahash_request *req)
4327 {
4328         return ahash_digest(req);
4329 }
4330
4331 static int ahash_init(struct ahash_request *req)
4332 {
4333         struct caam_hash_state *state = ahash_request_ctx_dma(req);
4334
4335         state->update = ahash_update_first;
4336         state->finup = ahash_finup_first;
4337         state->final = ahash_final_no_ctx;
4338
4339         state->ctx_dma = 0;
4340         state->ctx_dma_len = 0;
4341         state->buf_dma = 0;
4342         state->buflen = 0;
4343         state->next_buflen = 0;
4344
4345         return 0;
4346 }
4347
4348 static int ahash_update(struct ahash_request *req)
4349 {
4350         struct caam_hash_state *state = ahash_request_ctx_dma(req);
4351
4352         return state->update(req);
4353 }
4354
4355 static int ahash_finup(struct ahash_request *req)
4356 {
4357         struct caam_hash_state *state = ahash_request_ctx_dma(req);
4358
4359         return state->finup(req);
4360 }
4361
4362 static int ahash_final(struct ahash_request *req)
4363 {
4364         struct caam_hash_state *state = ahash_request_ctx_dma(req);
4365
4366         return state->final(req);
4367 }
4368
4369 static int ahash_export(struct ahash_request *req, void *out)
4370 {
4371         struct caam_hash_state *state = ahash_request_ctx_dma(req);
4372         struct caam_export_state *export = out;
4373         u8 *buf = state->buf;
4374         int len = state->buflen;
4375
4376         memcpy(export->buf, buf, len);
4377         memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
4378         export->buflen = len;
4379         export->update = state->update;
4380         export->final = state->final;
4381         export->finup = state->finup;
4382
4383         return 0;
4384 }
4385
4386 static int ahash_import(struct ahash_request *req, const void *in)
4387 {
4388         struct caam_hash_state *state = ahash_request_ctx_dma(req);
4389         const struct caam_export_state *export = in;
4390
4391         memset(state, 0, sizeof(*state));
4392         memcpy(state->buf, export->buf, export->buflen);
4393         memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
4394         state->buflen = export->buflen;
4395         state->update = export->update;
4396         state->final = export->final;
4397         state->finup = export->finup;
4398
4399         return 0;
4400 }
4401
4402 struct caam_hash_template {
4403         char name[CRYPTO_MAX_ALG_NAME];
4404         char driver_name[CRYPTO_MAX_ALG_NAME];
4405         char hmac_name[CRYPTO_MAX_ALG_NAME];
4406         char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
4407         unsigned int blocksize;
4408         struct ahash_alg template_ahash;
4409         u32 alg_type;
4410 };
4411
4412 /* ahash descriptors */
4413 static struct caam_hash_template driver_hash[] = {
4414         {
4415                 .name = "sha1",
4416                 .driver_name = "sha1-caam-qi2",
4417                 .hmac_name = "hmac(sha1)",
4418                 .hmac_driver_name = "hmac-sha1-caam-qi2",
4419                 .blocksize = SHA1_BLOCK_SIZE,
4420                 .template_ahash = {
4421                         .init = ahash_init,
4422                         .update = ahash_update,
4423                         .final = ahash_final,
4424                         .finup = ahash_finup,
4425                         .digest = ahash_digest,
4426                         .export = ahash_export,
4427                         .import = ahash_import,
4428                         .setkey = ahash_setkey,
4429                         .halg = {
4430                                 .digestsize = SHA1_DIGEST_SIZE,
4431                                 .statesize = sizeof(struct caam_export_state),
4432                         },
4433                 },
4434                 .alg_type = OP_ALG_ALGSEL_SHA1,
4435         }, {
4436                 .name = "sha224",
4437                 .driver_name = "sha224-caam-qi2",
4438                 .hmac_name = "hmac(sha224)",
4439                 .hmac_driver_name = "hmac-sha224-caam-qi2",
4440                 .blocksize = SHA224_BLOCK_SIZE,
4441                 .template_ahash = {
4442                         .init = ahash_init,
4443                         .update = ahash_update,
4444                         .final = ahash_final,
4445                         .finup = ahash_finup,
4446                         .digest = ahash_digest,
4447                         .export = ahash_export,
4448                         .import = ahash_import,
4449                         .setkey = ahash_setkey,
4450                         .halg = {
4451                                 .digestsize = SHA224_DIGEST_SIZE,
4452                                 .statesize = sizeof(struct caam_export_state),
4453                         },
4454                 },
4455                 .alg_type = OP_ALG_ALGSEL_SHA224,
4456         }, {
4457                 .name = "sha256",
4458                 .driver_name = "sha256-caam-qi2",
4459                 .hmac_name = "hmac(sha256)",
4460                 .hmac_driver_name = "hmac-sha256-caam-qi2",
4461                 .blocksize = SHA256_BLOCK_SIZE,
4462                 .template_ahash = {
4463                         .init = ahash_init,
4464                         .update = ahash_update,
4465                         .final = ahash_final,
4466                         .finup = ahash_finup,
4467                         .digest = ahash_digest,
4468                         .export = ahash_export,
4469                         .import = ahash_import,
4470                         .setkey = ahash_setkey,
4471                         .halg = {
4472                                 .digestsize = SHA256_DIGEST_SIZE,
4473                                 .statesize = sizeof(struct caam_export_state),
4474                         },
4475                 },
4476                 .alg_type = OP_ALG_ALGSEL_SHA256,
4477         }, {
4478                 .name = "sha384",
4479                 .driver_name = "sha384-caam-qi2",
4480                 .hmac_name = "hmac(sha384)",
4481                 .hmac_driver_name = "hmac-sha384-caam-qi2",
4482                 .blocksize = SHA384_BLOCK_SIZE,
4483                 .template_ahash = {
4484                         .init = ahash_init,
4485                         .update = ahash_update,
4486                         .final = ahash_final,
4487                         .finup = ahash_finup,
4488                         .digest = ahash_digest,
4489                         .export = ahash_export,
4490                         .import = ahash_import,
4491                         .setkey = ahash_setkey,
4492                         .halg = {
4493                                 .digestsize = SHA384_DIGEST_SIZE,
4494                                 .statesize = sizeof(struct caam_export_state),
4495                         },
4496                 },
4497                 .alg_type = OP_ALG_ALGSEL_SHA384,
4498         }, {
4499                 .name = "sha512",
4500                 .driver_name = "sha512-caam-qi2",
4501                 .hmac_name = "hmac(sha512)",
4502                 .hmac_driver_name = "hmac-sha512-caam-qi2",
4503                 .blocksize = SHA512_BLOCK_SIZE,
4504                 .template_ahash = {
4505                         .init = ahash_init,
4506                         .update = ahash_update,
4507                         .final = ahash_final,
4508                         .finup = ahash_finup,
4509                         .digest = ahash_digest,
4510                         .export = ahash_export,
4511                         .import = ahash_import,
4512                         .setkey = ahash_setkey,
4513                         .halg = {
4514                                 .digestsize = SHA512_DIGEST_SIZE,
4515                                 .statesize = sizeof(struct caam_export_state),
4516                         },
4517                 },
4518                 .alg_type = OP_ALG_ALGSEL_SHA512,
4519         }, {
4520                 .name = "md5",
4521                 .driver_name = "md5-caam-qi2",
4522                 .hmac_name = "hmac(md5)",
4523                 .hmac_driver_name = "hmac-md5-caam-qi2",
4524                 .blocksize = MD5_BLOCK_WORDS * 4,
4525                 .template_ahash = {
4526                         .init = ahash_init,
4527                         .update = ahash_update,
4528                         .final = ahash_final,
4529                         .finup = ahash_finup,
4530                         .digest = ahash_digest,
4531                         .export = ahash_export,
4532                         .import = ahash_import,
4533                         .setkey = ahash_setkey,
4534                         .halg = {
4535                                 .digestsize = MD5_DIGEST_SIZE,
4536                                 .statesize = sizeof(struct caam_export_state),
4537                         },
4538                 },
4539                 .alg_type = OP_ALG_ALGSEL_MD5,
4540         }
4541 };
4542
4543 struct caam_hash_alg {
4544         struct list_head entry;
4545         struct device *dev;
4546         int alg_type;
4547         struct ahash_alg ahash_alg;
4548 };
4549
4550 static int caam_hash_cra_init(struct crypto_tfm *tfm)
4551 {
4552         struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
4553         struct crypto_alg *base = tfm->__crt_alg;
4554         struct hash_alg_common *halg =
4555                  container_of(base, struct hash_alg_common, base);
4556         struct ahash_alg *alg =
4557                  container_of(halg, struct ahash_alg, halg);
4558         struct caam_hash_alg *caam_hash =
4559                  container_of(alg, struct caam_hash_alg, ahash_alg);
4560         struct caam_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm);
4561         /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
4562         static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
4563                                          HASH_MSG_LEN + SHA1_DIGEST_SIZE,
4564                                          HASH_MSG_LEN + 32,
4565                                          HASH_MSG_LEN + SHA256_DIGEST_SIZE,
4566                                          HASH_MSG_LEN + 64,
4567                                          HASH_MSG_LEN + SHA512_DIGEST_SIZE };
4568         dma_addr_t dma_addr;
4569         int i;
4570
4571         ctx->dev = caam_hash->dev;
4572
4573         if (alg->setkey) {
4574                 ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key,
4575                                                           ARRAY_SIZE(ctx->key),
4576                                                           DMA_TO_DEVICE,
4577                                                           DMA_ATTR_SKIP_CPU_SYNC);
4578                 if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) {
4579                         dev_err(ctx->dev, "unable to map key\n");
4580                         return -ENOMEM;
4581                 }
4582         }
4583
4584         dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc),
4585                                         DMA_BIDIRECTIONAL,
4586                                         DMA_ATTR_SKIP_CPU_SYNC);
4587         if (dma_mapping_error(ctx->dev, dma_addr)) {
4588                 dev_err(ctx->dev, "unable to map shared descriptors\n");
4589                 if (ctx->adata.key_dma)
4590                         dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4591                                                ARRAY_SIZE(ctx->key),
4592                                                DMA_TO_DEVICE,
4593                                                DMA_ATTR_SKIP_CPU_SYNC);
4594                 return -ENOMEM;
4595         }
4596
4597         for (i = 0; i < HASH_NUM_OP; i++)
4598                 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
4599
4600         /* copy descriptor header template value */
4601         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
4602
4603         ctx->ctx_len = runninglen[(ctx->adata.algtype &
4604                                    OP_ALG_ALGSEL_SUBMASK) >>
4605                                   OP_ALG_ALGSEL_SHIFT];
4606
4607         crypto_ahash_set_reqsize_dma(ahash, sizeof(struct caam_hash_state));
4608
4609         /*
4610          * For keyed hash algorithms shared descriptors
4611          * will be created later in setkey() callback
4612          */
4613         return alg->setkey ? 0 : ahash_set_sh_desc(ahash);
4614 }
4615
4616 static void caam_hash_cra_exit(struct crypto_tfm *tfm)
4617 {
4618         struct caam_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm);
4619
4620         dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc),
4621                                DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);
4622         if (ctx->adata.key_dma)
4623                 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4624                                        ARRAY_SIZE(ctx->key), DMA_TO_DEVICE,
4625                                        DMA_ATTR_SKIP_CPU_SYNC);
4626 }
4627
4628 static struct caam_hash_alg *caam_hash_alloc(struct device *dev,
4629         struct caam_hash_template *template, bool keyed)
4630 {
4631         struct caam_hash_alg *t_alg;
4632         struct ahash_alg *halg;
4633         struct crypto_alg *alg;
4634
4635         t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
4636         if (!t_alg)
4637                 return ERR_PTR(-ENOMEM);
4638
4639         t_alg->ahash_alg = template->template_ahash;
4640         halg = &t_alg->ahash_alg;
4641         alg = &halg->halg.base;
4642
4643         if (keyed) {
4644                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4645                          template->hmac_name);
4646                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4647                          template->hmac_driver_name);
4648         } else {
4649                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4650                          template->name);
4651                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4652                          template->driver_name);
4653                 t_alg->ahash_alg.setkey = NULL;
4654         }
4655         alg->cra_module = THIS_MODULE;
4656         alg->cra_init = caam_hash_cra_init;
4657         alg->cra_exit = caam_hash_cra_exit;
4658         alg->cra_ctxsize = sizeof(struct caam_hash_ctx) + crypto_dma_padding();
4659         alg->cra_priority = CAAM_CRA_PRIORITY;
4660         alg->cra_blocksize = template->blocksize;
4661         alg->cra_alignmask = 0;
4662         alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY;
4663
4664         t_alg->alg_type = template->alg_type;
4665         t_alg->dev = dev;
4666
4667         return t_alg;
4668 }
4669
4670 static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx)
4671 {
4672         struct dpaa2_caam_priv_per_cpu *ppriv;
4673
4674         ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx);
4675         napi_schedule_irqoff(&ppriv->napi);
4676 }
4677
4678 static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv)
4679 {
4680         struct device *dev = priv->dev;
4681         struct dpaa2_io_notification_ctx *nctx;
4682         struct dpaa2_caam_priv_per_cpu *ppriv;
4683         int err, i = 0, cpu;
4684
4685         for_each_online_cpu(cpu) {
4686                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4687                 ppriv->priv = priv;
4688                 nctx = &ppriv->nctx;
4689                 nctx->is_cdan = 0;
4690                 nctx->id = ppriv->rsp_fqid;
4691                 nctx->desired_cpu = cpu;
4692                 nctx->cb = dpaa2_caam_fqdan_cb;
4693
4694                 /* Register notification callbacks */
4695                 ppriv->dpio = dpaa2_io_service_select(cpu);
4696                 err = dpaa2_io_service_register(ppriv->dpio, nctx, dev);
4697                 if (unlikely(err)) {
4698                         dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu);
4699                         nctx->cb = NULL;
4700                         /*
4701                          * If no affine DPIO for this core, there's probably
4702                          * none available for next cores either. Signal we want
4703                          * to retry later, in case the DPIO devices weren't
4704                          * probed yet.
4705                          */
4706                         err = -EPROBE_DEFER;
4707                         goto err;
4708                 }
4709
4710                 ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE,
4711                                                      dev);
4712                 if (unlikely(!ppriv->store)) {
4713                         dev_err(dev, "dpaa2_io_store_create() failed\n");
4714                         err = -ENOMEM;
4715                         goto err;
4716                 }
4717
4718                 if (++i == priv->num_pairs)
4719                         break;
4720         }
4721
4722         return 0;
4723
4724 err:
4725         for_each_online_cpu(cpu) {
4726                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4727                 if (!ppriv->nctx.cb)
4728                         break;
4729                 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev);
4730         }
4731
4732         for_each_online_cpu(cpu) {
4733                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4734                 if (!ppriv->store)
4735                         break;
4736                 dpaa2_io_store_destroy(ppriv->store);
4737         }
4738
4739         return err;
4740 }
4741
4742 static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv)
4743 {
4744         struct dpaa2_caam_priv_per_cpu *ppriv;
4745         int i = 0, cpu;
4746
4747         for_each_online_cpu(cpu) {
4748                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4749                 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx,
4750                                             priv->dev);
4751                 dpaa2_io_store_destroy(ppriv->store);
4752
4753                 if (++i == priv->num_pairs)
4754                         return;
4755         }
4756 }
4757
4758 static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv)
4759 {
4760         struct dpseci_rx_queue_cfg rx_queue_cfg;
4761         struct device *dev = priv->dev;
4762         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4763         struct dpaa2_caam_priv_per_cpu *ppriv;
4764         int err = 0, i = 0, cpu;
4765
4766         /* Configure Rx queues */
4767         for_each_online_cpu(cpu) {
4768                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4769
4770                 rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST |
4771                                        DPSECI_QUEUE_OPT_USER_CTX;
4772                 rx_queue_cfg.order_preservation_en = 0;
4773                 rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO;
4774                 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id;
4775                 /*
4776                  * Rx priority (WQ) doesn't really matter, since we use
4777                  * pull mode, i.e. volatile dequeues from specific FQs
4778                  */
4779                 rx_queue_cfg.dest_cfg.priority = 0;
4780                 rx_queue_cfg.user_ctx = ppriv->nctx.qman64;
4781
4782                 err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
4783                                           &rx_queue_cfg);
4784                 if (err) {
4785                         dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n",
4786                                 err);
4787                         return err;
4788                 }
4789
4790                 if (++i == priv->num_pairs)
4791                         break;
4792         }
4793
4794         return err;
4795 }
4796
4797 static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv)
4798 {
4799         struct device *dev = priv->dev;
4800
4801         if (!priv->cscn_mem)
4802                 return;
4803
4804         dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4805         kfree(priv->cscn_mem);
4806 }
4807
4808 static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv)
4809 {
4810         struct device *dev = priv->dev;
4811         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4812         int err;
4813
4814         if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) {
4815                 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle);
4816                 if (err)
4817                         dev_err(dev, "dpseci_reset() failed\n");
4818         }
4819
4820         dpaa2_dpseci_congestion_free(priv);
4821         dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
4822 }
4823
4824 static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv,
4825                                   const struct dpaa2_fd *fd)
4826 {
4827         struct caam_request *req;
4828         u32 fd_err;
4829
4830         if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) {
4831                 dev_err(priv->dev, "Only Frame List FD format is supported!\n");
4832                 return;
4833         }
4834
4835         fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK;
4836         if (unlikely(fd_err))
4837                 dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err);
4838
4839         /*
4840          * FD[ADDR] is guaranteed to be valid, irrespective of errors reported
4841          * in FD[ERR] or FD[FRC].
4842          */
4843         req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd));
4844         dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt),
4845                          DMA_BIDIRECTIONAL);
4846         req->cbk(req->ctx, dpaa2_fd_get_frc(fd));
4847 }
4848
4849 static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv)
4850 {
4851         int err;
4852
4853         /* Retry while portal is busy */
4854         do {
4855                 err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid,
4856                                                ppriv->store);
4857         } while (err == -EBUSY);
4858
4859         if (unlikely(err))
4860                 dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err);
4861
4862         return err;
4863 }
4864
4865 static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv)
4866 {
4867         struct dpaa2_dq *dq;
4868         int cleaned = 0, is_last;
4869
4870         do {
4871                 dq = dpaa2_io_store_next(ppriv->store, &is_last);
4872                 if (unlikely(!dq)) {
4873                         if (unlikely(!is_last)) {
4874                                 dev_dbg(ppriv->priv->dev,
4875                                         "FQ %d returned no valid frames\n",
4876                                         ppriv->rsp_fqid);
4877                                 /*
4878                                  * MUST retry until we get some sort of
4879                                  * valid response token (be it "empty dequeue"
4880                                  * or a valid frame).
4881                                  */
4882                                 continue;
4883                         }
4884                         break;
4885                 }
4886
4887                 /* Process FD */
4888                 dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq));
4889                 cleaned++;
4890         } while (!is_last);
4891
4892         return cleaned;
4893 }
4894
4895 static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget)
4896 {
4897         struct dpaa2_caam_priv_per_cpu *ppriv;
4898         struct dpaa2_caam_priv *priv;
4899         int err, cleaned = 0, store_cleaned;
4900
4901         ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi);
4902         priv = ppriv->priv;
4903
4904         if (unlikely(dpaa2_caam_pull_fq(ppriv)))
4905                 return 0;
4906
4907         do {
4908                 store_cleaned = dpaa2_caam_store_consume(ppriv);
4909                 cleaned += store_cleaned;
4910
4911                 if (store_cleaned == 0 ||
4912                     cleaned > budget - DPAA2_CAAM_STORE_SIZE)
4913                         break;
4914
4915                 /* Try to dequeue some more */
4916                 err = dpaa2_caam_pull_fq(ppriv);
4917                 if (unlikely(err))
4918                         break;
4919         } while (1);
4920
4921         if (cleaned < budget) {
4922                 napi_complete_done(napi, cleaned);
4923                 err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx);
4924                 if (unlikely(err))
4925                         dev_err(priv->dev, "Notification rearm failed: %d\n",
4926                                 err);
4927         }
4928
4929         return cleaned;
4930 }
4931
4932 static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv,
4933                                          u16 token)
4934 {
4935         struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 };
4936         struct device *dev = priv->dev;
4937         unsigned int alignmask;
4938         int err;
4939
4940         /*
4941          * Congestion group feature supported starting with DPSECI API v5.1
4942          * and only when object has been created with this capability.
4943          */
4944         if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) ||
4945             !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG))
4946                 return 0;
4947
4948         alignmask = DPAA2_CSCN_ALIGN - 1;
4949         alignmask |= dma_get_cache_alignment() - 1;
4950         priv->cscn_mem = kzalloc(ALIGN(DPAA2_CSCN_SIZE, alignmask + 1),
4951                                  GFP_KERNEL);
4952         if (!priv->cscn_mem)
4953                 return -ENOMEM;
4954
4955         priv->cscn_dma = dma_map_single(dev, priv->cscn_mem,
4956                                         DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4957         if (dma_mapping_error(dev, priv->cscn_dma)) {
4958                 dev_err(dev, "Error mapping CSCN memory area\n");
4959                 err = -ENOMEM;
4960                 goto err_dma_map;
4961         }
4962
4963         cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES;
4964         cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH;
4965         cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH;
4966         cong_notif_cfg.message_ctx = (uintptr_t)priv;
4967         cong_notif_cfg.message_iova = priv->cscn_dma;
4968         cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER |
4969                                         DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT |
4970                                         DPSECI_CGN_MODE_COHERENT_WRITE;
4971
4972         err = dpseci_set_congestion_notification(priv->mc_io, 0, token,
4973                                                  &cong_notif_cfg);
4974         if (err) {
4975                 dev_err(dev, "dpseci_set_congestion_notification failed\n");
4976                 goto err_set_cong;
4977         }
4978
4979         return 0;
4980
4981 err_set_cong:
4982         dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4983 err_dma_map:
4984         kfree(priv->cscn_mem);
4985
4986         return err;
4987 }
4988
4989 static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev)
4990 {
4991         struct device *dev = &ls_dev->dev;
4992         struct dpaa2_caam_priv *priv;
4993         struct dpaa2_caam_priv_per_cpu *ppriv;
4994         int err, cpu;
4995         u8 i;
4996
4997         priv = dev_get_drvdata(dev);
4998
4999         priv->dev = dev;
5000         priv->dpsec_id = ls_dev->obj_desc.id;
5001
5002         /* Get a handle for the DPSECI this interface is associate with */
5003         err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle);
5004         if (err) {
5005                 dev_err(dev, "dpseci_open() failed: %d\n", err);
5006                 goto err_open;
5007         }
5008
5009         err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver,
5010                                      &priv->minor_ver);
5011         if (err) {
5012                 dev_err(dev, "dpseci_get_api_version() failed\n");
5013                 goto err_get_vers;
5014         }
5015
5016         dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver);
5017
5018         if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) {
5019                 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle);
5020                 if (err) {
5021                         dev_err(dev, "dpseci_reset() failed\n");
5022                         goto err_get_vers;
5023                 }
5024         }
5025
5026         err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle,
5027                                     &priv->dpseci_attr);
5028         if (err) {
5029                 dev_err(dev, "dpseci_get_attributes() failed\n");
5030                 goto err_get_vers;
5031         }
5032
5033         err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle,
5034                                   &priv->sec_attr);
5035         if (err) {
5036                 dev_err(dev, "dpseci_get_sec_attr() failed\n");
5037                 goto err_get_vers;
5038         }
5039
5040         err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle);
5041         if (err) {
5042                 dev_err(dev, "setup_congestion() failed\n");
5043                 goto err_get_vers;
5044         }
5045
5046         priv->num_pairs = min(priv->dpseci_attr.num_rx_queues,
5047                               priv->dpseci_attr.num_tx_queues);
5048         if (priv->num_pairs > num_online_cpus()) {
5049                 dev_warn(dev, "%d queues won't be used\n",
5050                          priv->num_pairs - num_online_cpus());
5051                 priv->num_pairs = num_online_cpus();
5052         }
5053
5054         for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) {
5055                 err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
5056                                           &priv->rx_queue_attr[i]);
5057                 if (err) {
5058                         dev_err(dev, "dpseci_get_rx_queue() failed\n");
5059                         goto err_get_rx_queue;
5060                 }
5061         }
5062
5063         for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) {
5064                 err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
5065                                           &priv->tx_queue_attr[i]);
5066                 if (err) {
5067                         dev_err(dev, "dpseci_get_tx_queue() failed\n");
5068                         goto err_get_rx_queue;
5069                 }
5070         }
5071
5072         i = 0;
5073         for_each_online_cpu(cpu) {
5074                 u8 j;
5075
5076                 j = i % priv->num_pairs;
5077
5078                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
5079                 ppriv->req_fqid = priv->tx_queue_attr[j].fqid;
5080
5081                 /*
5082                  * Allow all cores to enqueue, while only some of them
5083                  * will take part in dequeuing.
5084                  */
5085                 if (++i > priv->num_pairs)
5086                         continue;
5087
5088                 ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid;
5089                 ppriv->prio = j;
5090
5091                 dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j,
5092                         priv->rx_queue_attr[j].fqid,
5093                         priv->tx_queue_attr[j].fqid);
5094
5095                 ppriv->net_dev.dev = *dev;
5096                 INIT_LIST_HEAD(&ppriv->net_dev.napi_list);
5097                 netif_napi_add_tx_weight(&ppriv->net_dev, &ppriv->napi,
5098                                          dpaa2_dpseci_poll,
5099                                          DPAA2_CAAM_NAPI_WEIGHT);
5100         }
5101
5102         return 0;
5103
5104 err_get_rx_queue:
5105         dpaa2_dpseci_congestion_free(priv);
5106 err_get_vers:
5107         dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
5108 err_open:
5109         return err;
5110 }
5111
5112 static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv)
5113 {
5114         struct device *dev = priv->dev;
5115         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
5116         struct dpaa2_caam_priv_per_cpu *ppriv;
5117         int i;
5118
5119         for (i = 0; i < priv->num_pairs; i++) {
5120                 ppriv = per_cpu_ptr(priv->ppriv, i);
5121                 napi_enable(&ppriv->napi);
5122         }
5123
5124         return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle);
5125 }
5126
5127 static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv)
5128 {
5129         struct device *dev = priv->dev;
5130         struct dpaa2_caam_priv_per_cpu *ppriv;
5131         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
5132         int i, err = 0, enabled;
5133
5134         err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle);
5135         if (err) {
5136                 dev_err(dev, "dpseci_disable() failed\n");
5137                 return err;
5138         }
5139
5140         err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled);
5141         if (err) {
5142                 dev_err(dev, "dpseci_is_enabled() failed\n");
5143                 return err;
5144         }
5145
5146         dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true");
5147
5148         for (i = 0; i < priv->num_pairs; i++) {
5149                 ppriv = per_cpu_ptr(priv->ppriv, i);
5150                 napi_disable(&ppriv->napi);
5151                 netif_napi_del(&ppriv->napi);
5152         }
5153
5154         return 0;
5155 }
5156
5157 static struct list_head hash_list;
5158
5159 static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev)
5160 {
5161         struct device *dev;
5162         struct dpaa2_caam_priv *priv;
5163         int i, err = 0;
5164         bool registered = false;
5165
5166         /*
5167          * There is no way to get CAAM endianness - there is no direct register
5168          * space access and MC f/w does not provide this attribute.
5169          * All DPAA2-based SoCs have little endian CAAM, thus hard-code this
5170          * property.
5171          */
5172         caam_little_end = true;
5173
5174         caam_imx = false;
5175
5176         dev = &dpseci_dev->dev;
5177
5178         priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
5179         if (!priv)
5180                 return -ENOMEM;
5181
5182         dev_set_drvdata(dev, priv);
5183
5184         priv->domain = iommu_get_domain_for_dev(dev);
5185
5186         qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE,
5187                                      0, 0, NULL);
5188         if (!qi_cache) {
5189                 dev_err(dev, "Can't allocate SEC cache\n");
5190                 return -ENOMEM;
5191         }
5192
5193         err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49));
5194         if (err) {
5195                 dev_err(dev, "dma_set_mask_and_coherent() failed\n");
5196                 goto err_dma_mask;
5197         }
5198
5199         /* Obtain a MC portal */
5200         err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io);
5201         if (err) {
5202                 if (err == -ENXIO)
5203                         err = -EPROBE_DEFER;
5204                 else
5205                         dev_err(dev, "MC portal allocation failed\n");
5206
5207                 goto err_dma_mask;
5208         }
5209
5210         priv->ppriv = alloc_percpu(*priv->ppriv);
5211         if (!priv->ppriv) {
5212                 dev_err(dev, "alloc_percpu() failed\n");
5213                 err = -ENOMEM;
5214                 goto err_alloc_ppriv;
5215         }
5216
5217         /* DPSECI initialization */
5218         err = dpaa2_dpseci_setup(dpseci_dev);
5219         if (err) {
5220                 dev_err(dev, "dpaa2_dpseci_setup() failed\n");
5221                 goto err_dpseci_setup;
5222         }
5223
5224         /* DPIO */
5225         err = dpaa2_dpseci_dpio_setup(priv);
5226         if (err) {
5227                 dev_err_probe(dev, err, "dpaa2_dpseci_dpio_setup() failed\n");
5228                 goto err_dpio_setup;
5229         }
5230
5231         /* DPSECI binding to DPIO */
5232         err = dpaa2_dpseci_bind(priv);
5233         if (err) {
5234                 dev_err(dev, "dpaa2_dpseci_bind() failed\n");
5235                 goto err_bind;
5236         }
5237
5238         /* DPSECI enable */
5239         err = dpaa2_dpseci_enable(priv);
5240         if (err) {
5241                 dev_err(dev, "dpaa2_dpseci_enable() failed\n");
5242                 goto err_bind;
5243         }
5244
5245         dpaa2_dpseci_debugfs_init(priv);
5246
5247         /* register crypto algorithms the device supports */
5248         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5249                 struct caam_skcipher_alg *t_alg = driver_algs + i;
5250                 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
5251
5252                 /* Skip DES algorithms if not supported by device */
5253                 if (!priv->sec_attr.des_acc_num &&
5254                     (alg_sel == OP_ALG_ALGSEL_3DES ||
5255                      alg_sel == OP_ALG_ALGSEL_DES))
5256                         continue;
5257
5258                 /* Skip AES algorithms if not supported by device */
5259                 if (!priv->sec_attr.aes_acc_num &&
5260                     alg_sel == OP_ALG_ALGSEL_AES)
5261                         continue;
5262
5263                 /* Skip CHACHA20 algorithms if not supported by device */
5264                 if (alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5265                     !priv->sec_attr.ccha_acc_num)
5266                         continue;
5267
5268                 t_alg->caam.dev = dev;
5269                 caam_skcipher_alg_init(t_alg);
5270
5271                 err = crypto_register_skcipher(&t_alg->skcipher);
5272                 if (err) {
5273                         dev_warn(dev, "%s alg registration failed: %d\n",
5274                                  t_alg->skcipher.base.cra_driver_name, err);
5275                         continue;
5276                 }
5277
5278                 t_alg->registered = true;
5279                 registered = true;
5280         }
5281
5282         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5283                 struct caam_aead_alg *t_alg = driver_aeads + i;
5284                 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
5285                                  OP_ALG_ALGSEL_MASK;
5286                 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
5287                                  OP_ALG_ALGSEL_MASK;
5288
5289                 /* Skip DES algorithms if not supported by device */
5290                 if (!priv->sec_attr.des_acc_num &&
5291                     (c1_alg_sel == OP_ALG_ALGSEL_3DES ||
5292                      c1_alg_sel == OP_ALG_ALGSEL_DES))
5293                         continue;
5294
5295                 /* Skip AES algorithms if not supported by device */
5296                 if (!priv->sec_attr.aes_acc_num &&
5297                     c1_alg_sel == OP_ALG_ALGSEL_AES)
5298                         continue;
5299
5300                 /* Skip CHACHA20 algorithms if not supported by device */
5301                 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5302                     !priv->sec_attr.ccha_acc_num)
5303                         continue;
5304
5305                 /* Skip POLY1305 algorithms if not supported by device */
5306                 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 &&
5307                     !priv->sec_attr.ptha_acc_num)
5308                         continue;
5309
5310                 /*
5311                  * Skip algorithms requiring message digests
5312                  * if MD not supported by device.
5313                  */
5314                 if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 &&
5315                     !priv->sec_attr.md_acc_num)
5316                         continue;
5317
5318                 t_alg->caam.dev = dev;
5319                 caam_aead_alg_init(t_alg);
5320
5321                 err = crypto_register_aead(&t_alg->aead);
5322                 if (err) {
5323                         dev_warn(dev, "%s alg registration failed: %d\n",
5324                                  t_alg->aead.base.cra_driver_name, err);
5325                         continue;
5326                 }
5327
5328                 t_alg->registered = true;
5329                 registered = true;
5330         }
5331         if (registered)
5332                 dev_info(dev, "algorithms registered in /proc/crypto\n");
5333
5334         /* register hash algorithms the device supports */
5335         INIT_LIST_HEAD(&hash_list);
5336
5337         /*
5338          * Skip registration of any hashing algorithms if MD block
5339          * is not present.
5340          */
5341         if (!priv->sec_attr.md_acc_num)
5342                 return 0;
5343
5344         for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
5345                 struct caam_hash_alg *t_alg;
5346                 struct caam_hash_template *alg = driver_hash + i;
5347
5348                 /* register hmac version */
5349                 t_alg = caam_hash_alloc(dev, alg, true);
5350                 if (IS_ERR(t_alg)) {
5351                         err = PTR_ERR(t_alg);
5352                         dev_warn(dev, "%s hash alg allocation failed: %d\n",
5353                                  alg->hmac_driver_name, err);
5354                         continue;
5355                 }
5356
5357                 err = crypto_register_ahash(&t_alg->ahash_alg);
5358                 if (err) {
5359                         dev_warn(dev, "%s alg registration failed: %d\n",
5360                                  t_alg->ahash_alg.halg.base.cra_driver_name,
5361                                  err);
5362                         kfree(t_alg);
5363                 } else {
5364                         list_add_tail(&t_alg->entry, &hash_list);
5365                 }
5366
5367                 /* register unkeyed version */
5368                 t_alg = caam_hash_alloc(dev, alg, false);
5369                 if (IS_ERR(t_alg)) {
5370                         err = PTR_ERR(t_alg);
5371                         dev_warn(dev, "%s alg allocation failed: %d\n",
5372                                  alg->driver_name, err);
5373                         continue;
5374                 }
5375
5376                 err = crypto_register_ahash(&t_alg->ahash_alg);
5377                 if (err) {
5378                         dev_warn(dev, "%s alg registration failed: %d\n",
5379                                  t_alg->ahash_alg.halg.base.cra_driver_name,
5380                                  err);
5381                         kfree(t_alg);
5382                 } else {
5383                         list_add_tail(&t_alg->entry, &hash_list);
5384                 }
5385         }
5386         if (!list_empty(&hash_list))
5387                 dev_info(dev, "hash algorithms registered in /proc/crypto\n");
5388
5389         return err;
5390
5391 err_bind:
5392         dpaa2_dpseci_dpio_free(priv);
5393 err_dpio_setup:
5394         dpaa2_dpseci_free(priv);
5395 err_dpseci_setup:
5396         free_percpu(priv->ppriv);
5397 err_alloc_ppriv:
5398         fsl_mc_portal_free(priv->mc_io);
5399 err_dma_mask:
5400         kmem_cache_destroy(qi_cache);
5401
5402         return err;
5403 }
5404
5405 static void __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev)
5406 {
5407         struct device *dev;
5408         struct dpaa2_caam_priv *priv;
5409         int i;
5410
5411         dev = &ls_dev->dev;
5412         priv = dev_get_drvdata(dev);
5413
5414         dpaa2_dpseci_debugfs_exit(priv);
5415
5416         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5417                 struct caam_aead_alg *t_alg = driver_aeads + i;
5418
5419                 if (t_alg->registered)
5420                         crypto_unregister_aead(&t_alg->aead);
5421         }
5422
5423         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5424                 struct caam_skcipher_alg *t_alg = driver_algs + i;
5425
5426                 if (t_alg->registered)
5427                         crypto_unregister_skcipher(&t_alg->skcipher);
5428         }
5429
5430         if (hash_list.next) {
5431                 struct caam_hash_alg *t_hash_alg, *p;
5432
5433                 list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) {
5434                         crypto_unregister_ahash(&t_hash_alg->ahash_alg);
5435                         list_del(&t_hash_alg->entry);
5436                         kfree(t_hash_alg);
5437                 }
5438         }
5439
5440         dpaa2_dpseci_disable(priv);
5441         dpaa2_dpseci_dpio_free(priv);
5442         dpaa2_dpseci_free(priv);
5443         free_percpu(priv->ppriv);
5444         fsl_mc_portal_free(priv->mc_io);
5445         kmem_cache_destroy(qi_cache);
5446 }
5447
5448 int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req)
5449 {
5450         struct dpaa2_fd fd;
5451         struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
5452         struct dpaa2_caam_priv_per_cpu *ppriv;
5453         int err = 0, i;
5454
5455         if (IS_ERR(req))
5456                 return PTR_ERR(req);
5457
5458         if (priv->cscn_mem) {
5459                 dma_sync_single_for_cpu(priv->dev, priv->cscn_dma,
5460                                         DPAA2_CSCN_SIZE,
5461                                         DMA_FROM_DEVICE);
5462                 if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem))) {
5463                         dev_dbg_ratelimited(dev, "Dropping request\n");
5464                         return -EBUSY;
5465                 }
5466         }
5467
5468         dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma);
5469
5470         req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt),
5471                                          DMA_BIDIRECTIONAL);
5472         if (dma_mapping_error(dev, req->fd_flt_dma)) {
5473                 dev_err(dev, "DMA mapping error for QI enqueue request\n");
5474                 goto err_out;
5475         }
5476
5477         memset(&fd, 0, sizeof(fd));
5478         dpaa2_fd_set_format(&fd, dpaa2_fd_list);
5479         dpaa2_fd_set_addr(&fd, req->fd_flt_dma);
5480         dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1]));
5481         dpaa2_fd_set_flc(&fd, req->flc_dma);
5482
5483         ppriv = raw_cpu_ptr(priv->ppriv);
5484         for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) {
5485                 err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid,
5486                                                   &fd);
5487                 if (err != -EBUSY)
5488                         break;
5489
5490                 cpu_relax();
5491         }
5492
5493         if (unlikely(err)) {
5494                 dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err);
5495                 goto err_out;
5496         }
5497
5498         return -EINPROGRESS;
5499
5500 err_out:
5501         dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt),
5502                          DMA_BIDIRECTIONAL);
5503         return -EIO;
5504 }
5505 EXPORT_SYMBOL(dpaa2_caam_enqueue);
5506
5507 static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = {
5508         {
5509                 .vendor = FSL_MC_VENDOR_FREESCALE,
5510                 .obj_type = "dpseci",
5511         },
5512         { .vendor = 0x0 }
5513 };
5514 MODULE_DEVICE_TABLE(fslmc, dpaa2_caam_match_id_table);
5515
5516 static struct fsl_mc_driver dpaa2_caam_driver = {
5517         .driver = {
5518                 .name           = KBUILD_MODNAME,
5519                 .owner          = THIS_MODULE,
5520         },
5521         .probe          = dpaa2_caam_probe,
5522         .remove         = dpaa2_caam_remove,
5523         .match_id_table = dpaa2_caam_match_id_table
5524 };
5525
5526 MODULE_LICENSE("Dual BSD/GPL");
5527 MODULE_AUTHOR("Freescale Semiconductor, Inc");
5528 MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver");
5529
5530 module_fsl_mc_driver(dpaa2_caam_driver);