drivers/crypto/chelsio/chcr_algo.c

   1 /*
   2  * This file is part of the Chelsio T6 Crypto driver for Linux.
   3  *
   4  * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  *
  34  * Written and Maintained by:
  35  *      Manoj Malviya (manojmalviya@chelsio.com)
  36  *      Atul Gupta (atul.gupta@chelsio.com)
  37  *      Jitendra Lulla (jlulla@chelsio.com)
  38  *      Yeshaswi M R Gowda (yeshaswi@chelsio.com)
  39  *      Harsh Jain (harsh@chelsio.com)
  40  */
  41
  42 #define pr_fmt(fmt) "chcr:" fmt
  43
  44 #include <linux/kernel.h>
  45 #include <linux/module.h>
  46 #include <linux/crypto.h>
  47 #include <linux/skbuff.h>
  48 #include <linux/rtnetlink.h>
  49 #include <linux/highmem.h>
  50 #include <linux/scatterlist.h>
  51
  52 #include <crypto/aes.h>
  53 #include <crypto/algapi.h>
  54 #include <crypto/hash.h>
  55 #include <crypto/gcm.h>
  56 #include <crypto/sha1.h>
  57 #include <crypto/sha2.h>
  58 #include <crypto/authenc.h>
  59 #include <crypto/ctr.h>
  60 #include <crypto/gf128mul.h>
  61 #include <crypto/internal/aead.h>
  62 #include <crypto/null.h>
  63 #include <crypto/internal/skcipher.h>
  64 #include <crypto/aead.h>
  65 #include <crypto/scatterwalk.h>
  66 #include <crypto/internal/hash.h>
  67
  68 #include "t4fw_api.h"
  69 #include "t4_msg.h"
  70 #include "chcr_core.h"
  71 #include "chcr_algo.h"
  72 #include "chcr_crypto.h"
  73
  74 #define IV AES_BLOCK_SIZE
  75
  76 static unsigned int sgl_ent_len[] = {
  77         0, 0, 16, 24, 40, 48, 64, 72, 88,
  78         96, 112, 120, 136, 144, 160, 168, 184,
  79         192, 208, 216, 232, 240, 256, 264, 280,
  80         288, 304, 312, 328, 336, 352, 360, 376
  81 };
  82
  83 static unsigned int dsgl_ent_len[] = {
  84         0, 32, 32, 48, 48, 64, 64, 80, 80,
  85         112, 112, 128, 128, 144, 144, 160, 160,
  86         192, 192, 208, 208, 224, 224, 240, 240,
  87         272, 272, 288, 288, 304, 304, 320, 320
  88 };
  89
  90 static u32 round_constant[11] = {
  91         0x01000000, 0x02000000, 0x04000000, 0x08000000,
  92         0x10000000, 0x20000000, 0x40000000, 0x80000000,
  93         0x1B000000, 0x36000000, 0x6C000000
  94 };
  95
  96 static int chcr_handle_cipher_resp(struct skcipher_request *req,
  97                                    unsigned char *input, int err);
  98
  99 static inline  struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
 100 {
 101         return ctx->crypto_ctx->aeadctx;
 102 }
 103
 104 static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx)
 105 {
 106         return ctx->crypto_ctx->ablkctx;
 107 }
 108
 109 static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx)
 110 {
 111         return ctx->crypto_ctx->hmacctx;
 112 }
 113
 114 static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx)
 115 {
 116         return gctx->ctx->gcm;
 117 }
 118
 119 static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx)
 120 {
 121         return gctx->ctx->authenc;
 122 }
 123
 124 static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx)
 125 {
 126         return container_of(ctx->dev, struct uld_ctx, dev);
 127 }
 128
 129 static inline int is_ofld_imm(const struct sk_buff *skb)
 130 {
 131         return (skb->len <= SGE_MAX_WR_LEN);
 132 }
 133
 134 static inline void chcr_init_hctx_per_wr(struct chcr_ahash_req_ctx *reqctx)
 135 {
 136         memset(&reqctx->hctx_wr, 0, sizeof(struct chcr_hctx_per_wr));
 137 }
 138
 139 static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen,
 140                          unsigned int entlen,
 141                          unsigned int skip)
 142 {
 143         int nents = 0;
 144         unsigned int less;
 145         unsigned int skip_len = 0;
 146
 147         while (sg && skip) {
 148                 if (sg_dma_len(sg) <= skip) {
 149                         skip -= sg_dma_len(sg);
 150                         skip_len = 0;
 151                         sg = sg_next(sg);
 152                 } else {
 153                         skip_len = skip;
 154                         skip = 0;
 155                 }
 156         }
 157
 158         while (sg && reqlen) {
 159                 less = min(reqlen, sg_dma_len(sg) - skip_len);
 160                 nents += DIV_ROUND_UP(less, entlen);
 161                 reqlen -= less;
 162                 skip_len = 0;
 163                 sg = sg_next(sg);
 164         }
 165         return nents;
 166 }
 167
 168 static inline int get_aead_subtype(struct crypto_aead *aead)
 169 {
 170         struct aead_alg *alg = crypto_aead_alg(aead);
 171         struct chcr_alg_template *chcr_crypto_alg =
 172                 container_of(alg, struct chcr_alg_template, alg.aead);
 173         return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
 174 }
 175
 176 void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
 177 {
 178         u8 temp[SHA512_DIGEST_SIZE];
 179         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 180         int authsize = crypto_aead_authsize(tfm);
 181         struct cpl_fw6_pld *fw6_pld;
 182         int cmp = 0;
 183
 184         fw6_pld = (struct cpl_fw6_pld *)input;
 185         if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) ||
 186             (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) {
 187                 cmp = crypto_memneq(&fw6_pld->data[2], (fw6_pld + 1), authsize);
 188         } else {
 189
 190                 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp,
 191                                 authsize, req->assoclen +
 192                                 req->cryptlen - authsize);
 193                 cmp = crypto_memneq(temp, (fw6_pld + 1), authsize);
 194         }
 195         if (cmp)
 196                 *err = -EBADMSG;
 197         else
 198                 *err = 0;
 199 }
 200
 201 static int chcr_inc_wrcount(struct chcr_dev *dev)
 202 {
 203         if (dev->state == CHCR_DETACH)
 204                 return 1;
 205         atomic_inc(&dev->inflight);
 206         return 0;
 207 }
 208
 209 static inline void chcr_dec_wrcount(struct chcr_dev *dev)
 210 {
 211         atomic_dec(&dev->inflight);
 212 }
 213
 214 static inline int chcr_handle_aead_resp(struct aead_request *req,
 215                                          unsigned char *input,
 216                                          int err)
 217 {
 218         struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
 219         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 220         struct chcr_dev *dev = a_ctx(tfm)->dev;
 221
 222         chcr_aead_common_exit(req);
 223         if (reqctx->verify == VERIFY_SW) {
 224                 chcr_verify_tag(req, input, &err);
 225                 reqctx->verify = VERIFY_HW;
 226         }
 227         chcr_dec_wrcount(dev);
 228         req->base.complete(&req->base, err);
 229
 230         return err;
 231 }
 232
 233 static void get_aes_decrypt_key(unsigned char *dec_key,
 234                                        const unsigned char *key,
 235                                        unsigned int keylength)
 236 {
 237         u32 temp;
 238         u32 w_ring[MAX_NK];
 239         int i, j, k;
 240         u8  nr, nk;
 241
 242         switch (keylength) {
 243         case AES_KEYLENGTH_128BIT:
 244                 nk = KEYLENGTH_4BYTES;
 245                 nr = NUMBER_OF_ROUNDS_10;
 246                 break;
 247         case AES_KEYLENGTH_192BIT:
 248                 nk = KEYLENGTH_6BYTES;
 249                 nr = NUMBER_OF_ROUNDS_12;
 250                 break;
 251         case AES_KEYLENGTH_256BIT:
 252                 nk = KEYLENGTH_8BYTES;
 253                 nr = NUMBER_OF_ROUNDS_14;
 254                 break;
 255         default:
 256                 return;
 257         }
 258         for (i = 0; i < nk; i++)
 259                 w_ring[i] = get_unaligned_be32(&key[i * 4]);
 260
 261         i = 0;
 262         temp = w_ring[nk - 1];
 263         while (i + nk < (nr + 1) * 4) {
 264                 if (!(i % nk)) {
 265                         /* RotWord(temp) */
 266                         temp = (temp << 8) | (temp >> 24);
 267                         temp = aes_ks_subword(temp);
 268                         temp ^= round_constant[i / nk];
 269                 } else if (nk == 8 && (i % 4 == 0)) {
 270                         temp = aes_ks_subword(temp);
 271                 }
 272                 w_ring[i % nk] ^= temp;
 273                 temp = w_ring[i % nk];
 274                 i++;
 275         }
 276         i--;
 277         for (k = 0, j = i % nk; k < nk; k++) {
 278                 put_unaligned_be32(w_ring[j], &dec_key[k * 4]);
 279                 j--;
 280                 if (j < 0)
 281                         j += nk;
 282         }
 283 }
 284
 285 static struct crypto_shash *chcr_alloc_shash(unsigned int ds)
 286 {
 287         struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
 288
 289         switch (ds) {
 290         case SHA1_DIGEST_SIZE:
 291                 base_hash = crypto_alloc_shash("sha1", 0, 0);
 292                 break;
 293         case SHA224_DIGEST_SIZE:
 294                 base_hash = crypto_alloc_shash("sha224", 0, 0);
 295                 break;
 296         case SHA256_DIGEST_SIZE:
 297                 base_hash = crypto_alloc_shash("sha256", 0, 0);
 298                 break;
 299         case SHA384_DIGEST_SIZE:
 300                 base_hash = crypto_alloc_shash("sha384", 0, 0);
 301                 break;
 302         case SHA512_DIGEST_SIZE:
 303                 base_hash = crypto_alloc_shash("sha512", 0, 0);
 304                 break;
 305         }
 306
 307         return base_hash;
 308 }
 309
 310 static int chcr_compute_partial_hash(struct shash_desc *desc,
 311                                      char *iopad, char *result_hash,
 312                                      int digest_size)
 313 {
 314         struct sha1_state sha1_st;
 315         struct sha256_state sha256_st;
 316         struct sha512_state sha512_st;
 317         int error;
 318
 319         if (digest_size == SHA1_DIGEST_SIZE) {
 320                 error = crypto_shash_init(desc) ?:
 321                         crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?:
 322                         crypto_shash_export(desc, (void *)&sha1_st);
 323                 memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE);
 324         } else if (digest_size == SHA224_DIGEST_SIZE) {
 325                 error = crypto_shash_init(desc) ?:
 326                         crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
 327                         crypto_shash_export(desc, (void *)&sha256_st);
 328                 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
 329
 330         } else if (digest_size == SHA256_DIGEST_SIZE) {
 331                 error = crypto_shash_init(desc) ?:
 332                         crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
 333                         crypto_shash_export(desc, (void *)&sha256_st);
 334                 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
 335
 336         } else if (digest_size == SHA384_DIGEST_SIZE) {
 337                 error = crypto_shash_init(desc) ?:
 338                         crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
 339                         crypto_shash_export(desc, (void *)&sha512_st);
 340                 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
 341
 342         } else if (digest_size == SHA512_DIGEST_SIZE) {
 343                 error = crypto_shash_init(desc) ?:
 344                         crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
 345                         crypto_shash_export(desc, (void *)&sha512_st);
 346                 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
 347         } else {
 348                 error = -EINVAL;
 349                 pr_err("Unknown digest size %d\n", digest_size);
 350         }
 351         return error;
 352 }
 353
 354 static void chcr_change_order(char *buf, int ds)
 355 {
 356         int i;
 357
 358         if (ds == SHA512_DIGEST_SIZE) {
 359                 for (i = 0; i < (ds / sizeof(u64)); i++)
 360                         *((__be64 *)buf + i) =
 361                                 cpu_to_be64(*((u64 *)buf + i));
 362         } else {
 363                 for (i = 0; i < (ds / sizeof(u32)); i++)
 364                         *((__be32 *)buf + i) =
 365                                 cpu_to_be32(*((u32 *)buf + i));
 366         }
 367 }
 368
 369 static inline int is_hmac(struct crypto_tfm *tfm)
 370 {
 371         struct crypto_alg *alg = tfm->__crt_alg;
 372         struct chcr_alg_template *chcr_crypto_alg =
 373                 container_of(__crypto_ahash_alg(alg), struct chcr_alg_template,
 374                              alg.hash);
 375         if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC)
 376                 return 1;
 377         return 0;
 378 }
 379
 380 static inline void dsgl_walk_init(struct dsgl_walk *walk,
 381                                    struct cpl_rx_phys_dsgl *dsgl)
 382 {
 383         walk->dsgl = dsgl;
 384         walk->nents = 0;
 385         walk->to = (struct phys_sge_pairs *)(dsgl + 1);
 386 }
 387
 388 static inline void dsgl_walk_end(struct dsgl_walk *walk, unsigned short qid,
 389                                  int pci_chan_id)
 390 {
 391         struct cpl_rx_phys_dsgl *phys_cpl;
 392
 393         phys_cpl = walk->dsgl;
 394
 395         phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL)
 396                                     | CPL_RX_PHYS_DSGL_ISRDMA_V(0));
 397         phys_cpl->pcirlxorder_to_noofsgentr =
 398                 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) |
 399                       CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) |
 400                       CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) |
 401                       CPL_RX_PHYS_DSGL_PCITPHNT_V(0) |
 402                       CPL_RX_PHYS_DSGL_DCAID_V(0) |
 403                       CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk->nents));
 404         phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
 405         phys_cpl->rss_hdr_int.qid = htons(qid);
 406         phys_cpl->rss_hdr_int.hash_val = 0;
 407         phys_cpl->rss_hdr_int.channel = pci_chan_id;
 408 }
 409
 410 static inline void dsgl_walk_add_page(struct dsgl_walk *walk,
 411                                         size_t size,
 412                                         dma_addr_t addr)
 413 {
 414         int j;
 415
 416         if (!size)
 417                 return;
 418         j = walk->nents;
 419         walk->to->len[j % 8] = htons(size);
 420         walk->to->addr[j % 8] = cpu_to_be64(addr);
 421         j++;
 422         if ((j % 8) == 0)
 423                 walk->to++;
 424         walk->nents = j;
 425 }
 426
 427 static void  dsgl_walk_add_sg(struct dsgl_walk *walk,
 428                            struct scatterlist *sg,
 429                               unsigned int slen,
 430                               unsigned int skip)
 431 {
 432         int skip_len = 0;
 433         unsigned int left_size = slen, len = 0;
 434         unsigned int j = walk->nents;
 435         int offset, ent_len;
 436
 437         if (!slen)
 438                 return;
 439         while (sg && skip) {
 440                 if (sg_dma_len(sg) <= skip) {
 441                         skip -= sg_dma_len(sg);
 442                         skip_len = 0;
 443                         sg = sg_next(sg);
 444                 } else {
 445                         skip_len = skip;
 446                         skip = 0;
 447                 }
 448         }
 449
 450         while (left_size && sg) {
 451                 len = min_t(u32, left_size, sg_dma_len(sg) - skip_len);
 452                 offset = 0;
 453                 while (len) {
 454                         ent_len =  min_t(u32, len, CHCR_DST_SG_SIZE);
 455                         walk->to->len[j % 8] = htons(ent_len);
 456                         walk->to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) +
 457                                                       offset + skip_len);
 458                         offset += ent_len;
 459                         len -= ent_len;
 460                         j++;
 461                         if ((j % 8) == 0)
 462                                 walk->to++;
 463                 }
 464                 walk->last_sg = sg;
 465                 walk->last_sg_len = min_t(u32, left_size, sg_dma_len(sg) -
 466                                           skip_len) + skip_len;
 467                 left_size -= min_t(u32, left_size, sg_dma_len(sg) - skip_len);
 468                 skip_len = 0;
 469                 sg = sg_next(sg);
 470         }
 471         walk->nents = j;
 472 }
 473
 474 static inline void ulptx_walk_init(struct ulptx_walk *walk,
 475                                    struct ulptx_sgl *ulp)
 476 {
 477         walk->sgl = ulp;
 478         walk->nents = 0;
 479         walk->pair_idx = 0;
 480         walk->pair = ulp->sge;
 481         walk->last_sg = NULL;
 482         walk->last_sg_len = 0;
 483 }
 484
 485 static inline void ulptx_walk_end(struct ulptx_walk *walk)
 486 {
 487         walk->sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) |
 488                               ULPTX_NSGE_V(walk->nents));
 489 }
 490
 491
 492 static inline void ulptx_walk_add_page(struct ulptx_walk *walk,
 493                                         size_t size,
 494                                         dma_addr_t addr)
 495 {
 496         if (!size)
 497                 return;
 498
 499         if (walk->nents == 0) {
 500                 walk->sgl->len0 = cpu_to_be32(size);
 501                 walk->sgl->addr0 = cpu_to_be64(addr);
 502         } else {
 503                 walk->pair->addr[walk->pair_idx] = cpu_to_be64(addr);
 504                 walk->pair->len[walk->pair_idx] = cpu_to_be32(size);
 505                 walk->pair_idx = !walk->pair_idx;
 506                 if (!walk->pair_idx)
 507                         walk->pair++;
 508         }
 509         walk->nents++;
 510 }
 511
 512 static void  ulptx_walk_add_sg(struct ulptx_walk *walk,
 513                                         struct scatterlist *sg,
 514                                unsigned int len,
 515                                unsigned int skip)
 516 {
 517         int small;
 518         int skip_len = 0;
 519         unsigned int sgmin;
 520
 521         if (!len)
 522                 return;
 523         while (sg && skip) {
 524                 if (sg_dma_len(sg) <= skip) {
 525                         skip -= sg_dma_len(sg);
 526                         skip_len = 0;
 527                         sg = sg_next(sg);
 528                 } else {
 529                         skip_len = skip;
 530                         skip = 0;
 531                 }
 532         }
 533         WARN(!sg, "SG should not be null here\n");
 534         if (sg && (walk->nents == 0)) {
 535                 small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len);
 536                 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE);
 537                 walk->sgl->len0 = cpu_to_be32(sgmin);
 538                 walk->sgl->addr0 = cpu_to_be64(sg_dma_address(sg) + skip_len);
 539                 walk->nents++;
 540                 len -= sgmin;
 541                 walk->last_sg = sg;
 542                 walk->last_sg_len = sgmin + skip_len;
 543                 skip_len += sgmin;
 544                 if (sg_dma_len(sg) == skip_len) {
 545                         sg = sg_next(sg);
 546                         skip_len = 0;
 547                 }
 548         }
 549
 550         while (sg && len) {
 551                 small = min(sg_dma_len(sg) - skip_len, len);
 552                 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE);
 553                 walk->pair->len[walk->pair_idx] = cpu_to_be32(sgmin);
 554                 walk->pair->addr[walk->pair_idx] =
 555                         cpu_to_be64(sg_dma_address(sg) + skip_len);
 556                 walk->pair_idx = !walk->pair_idx;
 557                 walk->nents++;
 558                 if (!walk->pair_idx)
 559                         walk->pair++;
 560                 len -= sgmin;
 561                 skip_len += sgmin;
 562                 walk->last_sg = sg;
 563                 walk->last_sg_len = skip_len;
 564                 if (sg_dma_len(sg) == skip_len) {
 565                         sg = sg_next(sg);
 566                         skip_len = 0;
 567                 }
 568         }
 569 }
 570
 571 static inline int get_cryptoalg_subtype(struct crypto_skcipher *tfm)
 572 {
 573         struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
 574         struct chcr_alg_template *chcr_crypto_alg =
 575                 container_of(alg, struct chcr_alg_template, alg.skcipher);
 576
 577         return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
 578 }
 579
 580 static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx)
 581 {
 582         struct adapter *adap = netdev2adap(dev);
 583         struct sge_uld_txq_info *txq_info =
 584                 adap->sge.uld_txq_info[CXGB4_TX_CRYPTO];
 585         struct sge_uld_txq *txq;
 586         int ret = 0;
 587
 588         local_bh_disable();
 589         txq = &txq_info->uldtxq[idx];
 590         spin_lock(&txq->sendq.lock);
 591         if (txq->full)
 592                 ret = -1;
 593         spin_unlock(&txq->sendq.lock);
 594         local_bh_enable();
 595         return ret;
 596 }
 597
 598 static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
 599                                struct _key_ctx *key_ctx)
 600 {
 601         if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
 602                 memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len);
 603         } else {
 604                 memcpy(key_ctx->key,
 605                        ablkctx->key + (ablkctx->enckey_len >> 1),
 606                        ablkctx->enckey_len >> 1);
 607                 memcpy(key_ctx->key + (ablkctx->enckey_len >> 1),
 608                        ablkctx->rrkey, ablkctx->enckey_len >> 1);
 609         }
 610         return 0;
 611 }
 612
 613 static int chcr_hash_ent_in_wr(struct scatterlist *src,
 614                              unsigned int minsg,
 615                              unsigned int space,
 616                              unsigned int srcskip)
 617 {
 618         int srclen = 0;
 619         int srcsg = minsg;
 620         int soffset = 0, sless;
 621
 622         if (sg_dma_len(src) == srcskip) {
 623                 src = sg_next(src);
 624                 srcskip = 0;
 625         }
 626         while (src && space > (sgl_ent_len[srcsg + 1])) {
 627                 sless = min_t(unsigned int, sg_dma_len(src) - soffset - srcskip,
 628                                                         CHCR_SRC_SG_SIZE);
 629                 srclen += sless;
 630                 soffset += sless;
 631                 srcsg++;
 632                 if (sg_dma_len(src) == (soffset + srcskip)) {
 633                         src = sg_next(src);
 634                         soffset = 0;
 635                         srcskip = 0;
 636                 }
 637         }
 638         return srclen;
 639 }
 640
 641 static int chcr_sg_ent_in_wr(struct scatterlist *src,
 642                              struct scatterlist *dst,
 643                              unsigned int minsg,
 644                              unsigned int space,
 645                              unsigned int srcskip,
 646                              unsigned int dstskip)
 647 {
 648         int srclen = 0, dstlen = 0;
 649         int srcsg = minsg, dstsg = minsg;
 650         int offset = 0, soffset = 0, less, sless = 0;
 651
 652         if (sg_dma_len(src) == srcskip) {
 653                 src = sg_next(src);
 654                 srcskip = 0;
 655         }
 656         if (sg_dma_len(dst) == dstskip) {
 657                 dst = sg_next(dst);
 658                 dstskip = 0;
 659         }
 660
 661         while (src && dst &&
 662                space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) {
 663                 sless = min_t(unsigned int, sg_dma_len(src) - srcskip - soffset,
 664                                 CHCR_SRC_SG_SIZE);
 665                 srclen += sless;
 666                 srcsg++;
 667                 offset = 0;
 668                 while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) &&
 669                        space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) {
 670                         if (srclen <= dstlen)
 671                                 break;
 672                         less = min_t(unsigned int, sg_dma_len(dst) - offset -
 673                                      dstskip, CHCR_DST_SG_SIZE);
 674                         dstlen += less;
 675                         offset += less;
 676                         if ((offset + dstskip) == sg_dma_len(dst)) {
 677                                 dst = sg_next(dst);
 678                                 offset = 0;
 679                         }
 680                         dstsg++;
 681                         dstskip = 0;
 682                 }
 683                 soffset += sless;
 684                 if ((soffset + srcskip) == sg_dma_len(src)) {
 685                         src = sg_next(src);
 686                         srcskip = 0;
 687                         soffset = 0;
 688                 }
 689
 690         }
 691         return min(srclen, dstlen);
 692 }
 693
 694 static int chcr_cipher_fallback(struct crypto_skcipher *cipher,
 695                                 struct skcipher_request *req,
 696                                 u8 *iv,
 697                                 unsigned short op_type)
 698 {
 699         struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
 700         int err;
 701
 702         skcipher_request_set_tfm(&reqctx->fallback_req, cipher);
 703         skcipher_request_set_callback(&reqctx->fallback_req, req->base.flags,
 704                                       req->base.complete, req->base.data);
 705         skcipher_request_set_crypt(&reqctx->fallback_req, req->src, req->dst,
 706                                    req->cryptlen, iv);
 707
 708         err = op_type ? crypto_skcipher_decrypt(&reqctx->fallback_req) :
 709                         crypto_skcipher_encrypt(&reqctx->fallback_req);
 710
 711         return err;
 712
 713 }
 714
 715 static inline int get_qidxs(struct crypto_async_request *req,
 716                             unsigned int *txqidx, unsigned int *rxqidx)
 717 {
 718         struct crypto_tfm *tfm = req->tfm;
 719         int ret = 0;
 720
 721         switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
 722         case CRYPTO_ALG_TYPE_AEAD:
 723         {
 724                 struct aead_request *aead_req =
 725                         container_of(req, struct aead_request, base);
 726                 struct chcr_aead_reqctx *reqctx = aead_request_ctx(aead_req);
 727                 *txqidx = reqctx->txqidx;
 728                 *rxqidx = reqctx->rxqidx;
 729                 break;
 730         }
 731         case CRYPTO_ALG_TYPE_SKCIPHER:
 732         {
 733                 struct skcipher_request *sk_req =
 734                         container_of(req, struct skcipher_request, base);
 735                 struct chcr_skcipher_req_ctx *reqctx =
 736                         skcipher_request_ctx(sk_req);
 737                 *txqidx = reqctx->txqidx;
 738                 *rxqidx = reqctx->rxqidx;
 739                 break;
 740         }
 741         case CRYPTO_ALG_TYPE_AHASH:
 742         {
 743                 struct ahash_request *ahash_req =
 744                         container_of(req, struct ahash_request, base);
 745                 struct chcr_ahash_req_ctx *reqctx =
 746                         ahash_request_ctx(ahash_req);
 747                 *txqidx = reqctx->txqidx;
 748                 *rxqidx = reqctx->rxqidx;
 749                 break;
 750         }
 751         default:
 752                 ret = -EINVAL;
 753                 /* should never get here */
 754                 BUG();
 755                 break;
 756         }
 757         return ret;
 758 }
 759
 760 static inline void create_wreq(struct chcr_context *ctx,
 761                                struct chcr_wr *chcr_req,
 762                                struct crypto_async_request *req,
 763                                unsigned int imm,
 764                                int hash_sz,
 765                                unsigned int len16,
 766                                unsigned int sc_len,
 767                                unsigned int lcb)
 768 {
 769         struct uld_ctx *u_ctx = ULD_CTX(ctx);
 770         unsigned int tx_channel_id, rx_channel_id;
 771         unsigned int txqidx = 0, rxqidx = 0;
 772         unsigned int qid, fid;
 773
 774         get_qidxs(req, &txqidx, &rxqidx);
 775         qid = u_ctx->lldi.rxq_ids[rxqidx];
 776         fid = u_ctx->lldi.rxq_ids[0];
 777         tx_channel_id = txqidx / ctx->txq_perchan;
 778         rx_channel_id = rxqidx / ctx->rxq_perchan;
 779
 780
 781         chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE;
 782         chcr_req->wreq.pld_size_hash_size =
 783                 htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz));
 784         chcr_req->wreq.len16_pkd =
 785                 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16, 16)));
 786         chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req);
 787         chcr_req->wreq.rx_chid_to_rx_q_id = FILL_WR_RX_Q_ID(rx_channel_id, qid,
 788                                                             !!lcb, txqidx);
 789
 790         chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(tx_channel_id, fid);
 791         chcr_req->ulptx.len = htonl((DIV_ROUND_UP(len16, 16) -
 792                                 ((sizeof(chcr_req->wreq)) >> 4)));
 793         chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(!imm);
 794         chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
 795                                            sizeof(chcr_req->key_ctx) + sc_len);
 796 }
 797
 798 /**
 799  *      create_cipher_wr - form the WR for cipher operations
 800  *      @req: cipher req.
 801  *      @ctx: crypto driver context of the request.
 802  *      @qid: ingress qid where response of this WR should be received.
 803  *      @op_type:       encryption or decryption
 804  */
 805 static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam)
 806 {
 807         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(wrparam->req);
 808         struct chcr_context *ctx = c_ctx(tfm);
 809         struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
 810         struct sk_buff *skb = NULL;
 811         struct chcr_wr *chcr_req;
 812         struct cpl_rx_phys_dsgl *phys_cpl;
 813         struct ulptx_sgl *ulptx;
 814         struct chcr_skcipher_req_ctx *reqctx =
 815                 skcipher_request_ctx(wrparam->req);
 816         unsigned int temp = 0, transhdr_len, dst_size;
 817         int error;
 818         int nents;
 819         unsigned int kctx_len;
 820         gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
 821                         GFP_KERNEL : GFP_ATOMIC;
 822         struct adapter *adap = padap(ctx->dev);
 823         unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
 824
 825         nents = sg_nents_xlen(reqctx->dstsg,  wrparam->bytes, CHCR_DST_SG_SIZE,
 826                               reqctx->dst_ofst);
 827         dst_size = get_space_for_phys_dsgl(nents);
 828         kctx_len = roundup(ablkctx->enckey_len, 16);
 829         transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
 830         nents = sg_nents_xlen(reqctx->srcsg, wrparam->bytes,
 831                                   CHCR_SRC_SG_SIZE, reqctx->src_ofst);
 832         temp = reqctx->imm ? roundup(wrparam->bytes, 16) :
 833                                      (sgl_len(nents) * 8);
 834         transhdr_len += temp;
 835         transhdr_len = roundup(transhdr_len, 16);
 836         skb = alloc_skb(SGE_MAX_WR_LEN, flags);
 837         if (!skb) {
 838                 error = -ENOMEM;
 839                 goto err;
 840         }
 841         chcr_req = __skb_put_zero(skb, transhdr_len);
 842         chcr_req->sec_cpl.op_ivinsrtofst =
 843                         FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1);
 844
 845         chcr_req->sec_cpl.pldlen = htonl(IV + wrparam->bytes);
 846         chcr_req->sec_cpl.aadstart_cipherstop_hi =
 847                         FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV + 1, 0);
 848
 849         chcr_req->sec_cpl.cipherstop_lo_authinsert =
 850                         FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0);
 851         chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0,
 852                                                          ablkctx->ciph_mode,
 853                                                          0, 0, IV >> 1);
 854         chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
 855                                                           0, 1, dst_size);
 856
 857         chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr;
 858         if ((reqctx->op == CHCR_DECRYPT_OP) &&
 859             (!(get_cryptoalg_subtype(tfm) ==
 860                CRYPTO_ALG_SUB_TYPE_CTR)) &&
 861             (!(get_cryptoalg_subtype(tfm) ==
 862                CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) {
 863                 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx);
 864         } else {
 865                 if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) ||
 866                     (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) {
 867                         memcpy(chcr_req->key_ctx.key, ablkctx->key,
 868                                ablkctx->enckey_len);
 869                 } else {
 870                         memcpy(chcr_req->key_ctx.key, ablkctx->key +
 871                                (ablkctx->enckey_len >> 1),
 872                                ablkctx->enckey_len >> 1);
 873                         memcpy(chcr_req->key_ctx.key +
 874                                (ablkctx->enckey_len >> 1),
 875                                ablkctx->key,
 876                                ablkctx->enckey_len >> 1);
 877                 }
 878         }
 879         phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
 880         ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
 881         chcr_add_cipher_src_ent(wrparam->req, ulptx, wrparam);
 882         chcr_add_cipher_dst_ent(wrparam->req, phys_cpl, wrparam, wrparam->qid);
 883
 884         atomic_inc(&adap->chcr_stats.cipher_rqst);
 885         temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len + IV
 886                 + (reqctx->imm ? (wrparam->bytes) : 0);
 887         create_wreq(c_ctx(tfm), chcr_req, &(wrparam->req->base), reqctx->imm, 0,
 888                     transhdr_len, temp,
 889                         ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC);
 890         reqctx->skb = skb;
 891
 892         if (reqctx->op && (ablkctx->ciph_mode ==
 893                            CHCR_SCMD_CIPHER_MODE_AES_CBC))
 894                 sg_pcopy_to_buffer(wrparam->req->src,
 895                         sg_nents(wrparam->req->src), wrparam->req->iv, 16,
 896                         reqctx->processed + wrparam->bytes - AES_BLOCK_SIZE);
 897
 898         return skb;
 899 err:
 900         return ERR_PTR(error);
 901 }
 902
 903 static inline int chcr_keyctx_ck_size(unsigned int keylen)
 904 {
 905         int ck_size = 0;
 906
 907         if (keylen == AES_KEYSIZE_128)
 908                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
 909         else if (keylen == AES_KEYSIZE_192)
 910                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
 911         else if (keylen == AES_KEYSIZE_256)
 912                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
 913         else
 914                 ck_size = 0;
 915
 916         return ck_size;
 917 }
 918 static int chcr_cipher_fallback_setkey(struct crypto_skcipher *cipher,
 919                                        const u8 *key,
 920                                        unsigned int keylen)
 921 {
 922         struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
 923
 924         crypto_skcipher_clear_flags(ablkctx->sw_cipher,
 925                                 CRYPTO_TFM_REQ_MASK);
 926         crypto_skcipher_set_flags(ablkctx->sw_cipher,
 927                                 cipher->base.crt_flags & CRYPTO_TFM_REQ_MASK);
 928         return crypto_skcipher_setkey(ablkctx->sw_cipher, key, keylen);
 929 }
 930
 931 static int chcr_aes_cbc_setkey(struct crypto_skcipher *cipher,
 932                                const u8 *key,
 933                                unsigned int keylen)
 934 {
 935         struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
 936         unsigned int ck_size, context_size;
 937         u16 alignment = 0;
 938         int err;
 939
 940         err = chcr_cipher_fallback_setkey(cipher, key, keylen);
 941         if (err)
 942                 goto badkey_err;
 943
 944         ck_size = chcr_keyctx_ck_size(keylen);
 945         alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0;
 946         memcpy(ablkctx->key, key, keylen);
 947         ablkctx->enckey_len = keylen;
 948         get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3);
 949         context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
 950                         keylen + alignment) >> 4;
 951
 952         ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
 953                                                 0, 0, context_size);
 954         ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC;
 955         return 0;
 956 badkey_err:
 957         ablkctx->enckey_len = 0;
 958
 959         return err;
 960 }
 961
 962 static int chcr_aes_ctr_setkey(struct crypto_skcipher *cipher,
 963                                    const u8 *key,
 964                                    unsigned int keylen)
 965 {
 966         struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
 967         unsigned int ck_size, context_size;
 968         u16 alignment = 0;
 969         int err;
 970
 971         err = chcr_cipher_fallback_setkey(cipher, key, keylen);
 972         if (err)
 973                 goto badkey_err;
 974         ck_size = chcr_keyctx_ck_size(keylen);
 975         alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
 976         memcpy(ablkctx->key, key, keylen);
 977         ablkctx->enckey_len = keylen;
 978         context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
 979                         keylen + alignment) >> 4;
 980
 981         ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
 982                                                 0, 0, context_size);
 983         ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
 984
 985         return 0;
 986 badkey_err:
 987         ablkctx->enckey_len = 0;
 988
 989         return err;
 990 }
 991
 992 static int chcr_aes_rfc3686_setkey(struct crypto_skcipher *cipher,
 993                                    const u8 *key,
 994                                    unsigned int keylen)
 995 {
 996         struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
 997         unsigned int ck_size, context_size;
 998         u16 alignment = 0;
 999         int err;
1000
1001         if (keylen < CTR_RFC3686_NONCE_SIZE)
1002                 return -EINVAL;
1003         memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE),
1004                CTR_RFC3686_NONCE_SIZE);
1005
1006         keylen -= CTR_RFC3686_NONCE_SIZE;
1007         err = chcr_cipher_fallback_setkey(cipher, key, keylen);
1008         if (err)
1009                 goto badkey_err;
1010
1011         ck_size = chcr_keyctx_ck_size(keylen);
1012         alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
1013         memcpy(ablkctx->key, key, keylen);
1014         ablkctx->enckey_len = keylen;
1015         context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
1016                         keylen + alignment) >> 4;
1017
1018         ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
1019                                                 0, 0, context_size);
1020         ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
1021
1022         return 0;
1023 badkey_err:
1024         ablkctx->enckey_len = 0;
1025
1026         return err;
1027 }
1028 static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add)
1029 {
1030         unsigned int size = AES_BLOCK_SIZE;
1031         __be32 *b = (__be32 *)(dstiv + size);
1032         u32 c, prev;
1033
1034         memcpy(dstiv, srciv, AES_BLOCK_SIZE);
1035         for (; size >= 4; size -= 4) {
1036                 prev = be32_to_cpu(*--b);
1037                 c = prev + add;
1038                 *b = cpu_to_be32(c);
1039                 if (prev < c)
1040                         break;
1041                 add = 1;
1042         }
1043
1044 }
1045
1046 static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes)
1047 {
1048         __be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE);
1049         u64 c;
1050         u32 temp = be32_to_cpu(*--b);
1051
1052         temp = ~temp;
1053         c = (u64)temp +  1; // No of block can processed without overflow
1054         if ((bytes / AES_BLOCK_SIZE) >= c)
1055                 bytes = c * AES_BLOCK_SIZE;
1056         return bytes;
1057 }
1058
1059 static int chcr_update_tweak(struct skcipher_request *req, u8 *iv,
1060                              u32 isfinal)
1061 {
1062         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1063         struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
1064         struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1065         struct crypto_aes_ctx aes;
1066         int ret, i;
1067         u8 *key;
1068         unsigned int keylen;
1069         int round = reqctx->last_req_len / AES_BLOCK_SIZE;
1070         int round8 = round / 8;
1071
1072         memcpy(iv, reqctx->iv, AES_BLOCK_SIZE);
1073
1074         keylen = ablkctx->enckey_len / 2;
1075         key = ablkctx->key + keylen;
1076         /* For a 192 bit key remove the padded zeroes which was
1077          * added in chcr_xts_setkey
1078          */
1079         if (KEY_CONTEXT_CK_SIZE_G(ntohl(ablkctx->key_ctx_hdr))
1080                         == CHCR_KEYCTX_CIPHER_KEY_SIZE_192)
1081                 ret = aes_expandkey(&aes, key, keylen - 8);
1082         else
1083                 ret = aes_expandkey(&aes, key, keylen);
1084         if (ret)
1085                 return ret;
1086         aes_encrypt(&aes, iv, iv);
1087         for (i = 0; i < round8; i++)
1088                 gf128mul_x8_ble((le128 *)iv, (le128 *)iv);
1089
1090         for (i = 0; i < (round % 8); i++)
1091                 gf128mul_x_ble((le128 *)iv, (le128 *)iv);
1092
1093         if (!isfinal)
1094                 aes_decrypt(&aes, iv, iv);
1095
1096         memzero_explicit(&aes, sizeof(aes));
1097         return 0;
1098 }
1099
1100 static int chcr_update_cipher_iv(struct skcipher_request *req,
1101                                    struct cpl_fw6_pld *fw6_pld, u8 *iv)
1102 {
1103         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1104         struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1105         int subtype = get_cryptoalg_subtype(tfm);
1106         int ret = 0;
1107
1108         if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
1109                 ctr_add_iv(iv, req->iv, (reqctx->processed /
1110                            AES_BLOCK_SIZE));
1111         else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686)
1112                 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
1113                         CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed /
1114                                                 AES_BLOCK_SIZE) + 1);
1115         else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
1116                 ret = chcr_update_tweak(req, iv, 0);
1117         else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
1118                 if (reqctx->op)
1119                         /*Updated before sending last WR*/
1120                         memcpy(iv, req->iv, AES_BLOCK_SIZE);
1121                 else
1122                         memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
1123         }
1124
1125         return ret;
1126
1127 }
1128
1129 /* We need separate function for final iv because in rfc3686  Initial counter
1130  * starts from 1 and buffer size of iv is 8 byte only which remains constant
1131  * for subsequent update requests
1132  */
1133
1134 static int chcr_final_cipher_iv(struct skcipher_request *req,
1135                                    struct cpl_fw6_pld *fw6_pld, u8 *iv)
1136 {
1137         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1138         struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1139         int subtype = get_cryptoalg_subtype(tfm);
1140         int ret = 0;
1141
1142         if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
1143                 ctr_add_iv(iv, req->iv, DIV_ROUND_UP(reqctx->processed,
1144                                                        AES_BLOCK_SIZE));
1145         else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) {
1146                 if (!reqctx->partial_req)
1147                         memcpy(iv, reqctx->iv, AES_BLOCK_SIZE);
1148                 else
1149                         ret = chcr_update_tweak(req, iv, 1);
1150         }
1151         else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
1152                 /*Already updated for Decrypt*/
1153                 if (!reqctx->op)
1154                         memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
1155
1156         }
1157         return ret;
1158
1159 }
1160
1161 static int chcr_handle_cipher_resp(struct skcipher_request *req,
1162                                    unsigned char *input, int err)
1163 {
1164         struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1165         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1166         struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input;
1167         struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
1168         struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
1169         struct chcr_dev *dev = c_ctx(tfm)->dev;
1170         struct chcr_context *ctx = c_ctx(tfm);
1171         struct adapter *adap = padap(ctx->dev);
1172         struct cipher_wr_param wrparam;
1173         struct sk_buff *skb;
1174         int bytes;
1175
1176         if (err)
1177                 goto unmap;
1178         if (req->cryptlen == reqctx->processed) {
1179                 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1180                                       req);
1181                 err = chcr_final_cipher_iv(req, fw6_pld, req->iv);
1182                 goto complete;
1183         }
1184
1185         if (!reqctx->imm) {
1186                 bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 0,
1187                                           CIP_SPACE_LEFT(ablkctx->enckey_len),
1188                                           reqctx->src_ofst, reqctx->dst_ofst);
1189                 if ((bytes + reqctx->processed) >= req->cryptlen)
1190                         bytes  = req->cryptlen - reqctx->processed;
1191                 else
1192                         bytes = rounddown(bytes, 16);
1193         } else {
1194                 /*CTR mode counter overfloa*/
1195                 bytes  = req->cryptlen - reqctx->processed;
1196         }
1197         err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv);
1198         if (err)
1199                 goto unmap;
1200
1201         if (unlikely(bytes == 0)) {
1202                 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1203                                       req);
1204                 memcpy(req->iv, reqctx->init_iv, IV);
1205                 atomic_inc(&adap->chcr_stats.fallback);
1206                 err = chcr_cipher_fallback(ablkctx->sw_cipher, req, req->iv,
1207                                            reqctx->op);
1208                 goto complete;
1209         }
1210
1211         if (get_cryptoalg_subtype(tfm) ==
1212             CRYPTO_ALG_SUB_TYPE_CTR)
1213                 bytes = adjust_ctr_overflow(reqctx->iv, bytes);
1214         wrparam.qid = u_ctx->lldi.rxq_ids[reqctx->rxqidx];
1215         wrparam.req = req;
1216         wrparam.bytes = bytes;
1217         skb = create_cipher_wr(&wrparam);
1218         if (IS_ERR(skb)) {
1219                 pr_err("%s : Failed to form WR. No memory\n", __func__);
1220                 err = PTR_ERR(skb);
1221                 goto unmap;
1222         }
1223         skb->dev = u_ctx->lldi.ports[0];
1224         set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
1225         chcr_send_wr(skb);
1226         reqctx->last_req_len = bytes;
1227         reqctx->processed += bytes;
1228         if (get_cryptoalg_subtype(tfm) ==
1229                 CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags ==
1230                         CRYPTO_TFM_REQ_MAY_SLEEP ) {
1231                 complete(&ctx->cbc_aes_aio_done);
1232         }
1233         return 0;
1234 unmap:
1235         chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
1236 complete:
1237         if (get_cryptoalg_subtype(tfm) ==
1238                 CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags ==
1239                         CRYPTO_TFM_REQ_MAY_SLEEP ) {
1240                 complete(&ctx->cbc_aes_aio_done);
1241         }
1242         chcr_dec_wrcount(dev);
1243         req->base.complete(&req->base, err);
1244         return err;
1245 }
1246
1247 static int process_cipher(struct skcipher_request *req,
1248                                   unsigned short qid,
1249                                   struct sk_buff **skb,
1250                                   unsigned short op_type)
1251 {
1252         struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1253         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1254         unsigned int ivsize = crypto_skcipher_ivsize(tfm);
1255         struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
1256         struct adapter *adap = padap(c_ctx(tfm)->dev);
1257         struct  cipher_wr_param wrparam;
1258         int bytes, err = -EINVAL;
1259         int subtype;
1260
1261         reqctx->processed = 0;
1262         reqctx->partial_req = 0;
1263         if (!req->iv)
1264                 goto error;
1265         subtype = get_cryptoalg_subtype(tfm);
1266         if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) ||
1267             (req->cryptlen == 0) ||
1268             (req->cryptlen % crypto_skcipher_blocksize(tfm))) {
1269                 if (req->cryptlen == 0 && subtype != CRYPTO_ALG_SUB_TYPE_XTS)
1270                         goto fallback;
1271                 else if (req->cryptlen % crypto_skcipher_blocksize(tfm) &&
1272                          subtype == CRYPTO_ALG_SUB_TYPE_XTS)
1273                         goto fallback;
1274                 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n",
1275                        ablkctx->enckey_len, req->cryptlen, ivsize);
1276                 goto error;
1277         }
1278
1279         err = chcr_cipher_dma_map(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
1280         if (err)
1281                 goto error;
1282         if (req->cryptlen < (SGE_MAX_WR_LEN - (sizeof(struct chcr_wr) +
1283                                             AES_MIN_KEY_SIZE +
1284                                             sizeof(struct cpl_rx_phys_dsgl) +
1285                                         /*Min dsgl size*/
1286                                             32))) {
1287                 /* Can be sent as Imm*/
1288                 unsigned int dnents = 0, transhdr_len, phys_dsgl, kctx_len;
1289
1290                 dnents = sg_nents_xlen(req->dst, req->cryptlen,
1291                                        CHCR_DST_SG_SIZE, 0);
1292                 phys_dsgl = get_space_for_phys_dsgl(dnents);
1293                 kctx_len = roundup(ablkctx->enckey_len, 16);
1294                 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
1295                 reqctx->imm = (transhdr_len + IV + req->cryptlen) <=
1296                         SGE_MAX_WR_LEN;
1297                 bytes = IV + req->cryptlen;
1298
1299         } else {
1300                 reqctx->imm = 0;
1301         }
1302
1303         if (!reqctx->imm) {
1304                 bytes = chcr_sg_ent_in_wr(req->src, req->dst, 0,
1305                                           CIP_SPACE_LEFT(ablkctx->enckey_len),
1306                                           0, 0);
1307                 if ((bytes + reqctx->processed) >= req->cryptlen)
1308                         bytes  = req->cryptlen - reqctx->processed;
1309                 else
1310                         bytes = rounddown(bytes, 16);
1311         } else {
1312                 bytes = req->cryptlen;
1313         }
1314         if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) {
1315                 bytes = adjust_ctr_overflow(req->iv, bytes);
1316         }
1317         if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) {
1318                 memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE);
1319                 memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->iv,
1320                                 CTR_RFC3686_IV_SIZE);
1321
1322                 /* initialize counter portion of counter block */
1323                 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
1324                         CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
1325                 memcpy(reqctx->init_iv, reqctx->iv, IV);
1326
1327         } else {
1328
1329                 memcpy(reqctx->iv, req->iv, IV);
1330                 memcpy(reqctx->init_iv, req->iv, IV);
1331         }
1332         if (unlikely(bytes == 0)) {
1333                 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1334                                       req);
1335 fallback:       atomic_inc(&adap->chcr_stats.fallback);
1336                 err = chcr_cipher_fallback(ablkctx->sw_cipher, req,
1337                                            subtype ==
1338                                            CRYPTO_ALG_SUB_TYPE_CTR_RFC3686 ?
1339                                            reqctx->iv : req->iv,
1340                                            op_type);
1341                 goto error;
1342         }
1343         reqctx->op = op_type;
1344         reqctx->srcsg = req->src;
1345         reqctx->dstsg = req->dst;
1346         reqctx->src_ofst = 0;
1347         reqctx->dst_ofst = 0;
1348         wrparam.qid = qid;
1349         wrparam.req = req;
1350         wrparam.bytes = bytes;
1351         *skb = create_cipher_wr(&wrparam);
1352         if (IS_ERR(*skb)) {
1353                 err = PTR_ERR(*skb);
1354                 goto unmap;
1355         }
1356         reqctx->processed = bytes;
1357         reqctx->last_req_len = bytes;
1358         reqctx->partial_req = !!(req->cryptlen - reqctx->processed);
1359
1360         return 0;
1361 unmap:
1362         chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
1363 error:
1364         return err;
1365 }
1366
1367 static int chcr_aes_encrypt(struct skcipher_request *req)
1368 {
1369         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1370         struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1371         struct chcr_dev *dev = c_ctx(tfm)->dev;
1372         struct sk_buff *skb = NULL;
1373         int err;
1374         struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
1375         struct chcr_context *ctx = c_ctx(tfm);
1376         unsigned int cpu;
1377
1378         cpu = get_cpu();
1379         reqctx->txqidx = cpu % ctx->ntxq;
1380         reqctx->rxqidx = cpu % ctx->nrxq;
1381         put_cpu();
1382
1383         err = chcr_inc_wrcount(dev);
1384         if (err)
1385                 return -ENXIO;
1386         if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1387                                                 reqctx->txqidx) &&
1388                 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1389                         err = -ENOSPC;
1390                         goto error;
1391         }
1392
1393         err = process_cipher(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx],
1394                              &skb, CHCR_ENCRYPT_OP);
1395         if (err || !skb)
1396                 return  err;
1397         skb->dev = u_ctx->lldi.ports[0];
1398         set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
1399         chcr_send_wr(skb);
1400         if (get_cryptoalg_subtype(tfm) ==
1401                 CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags ==
1402                         CRYPTO_TFM_REQ_MAY_SLEEP ) {
1403                         reqctx->partial_req = 1;
1404                         wait_for_completion(&ctx->cbc_aes_aio_done);
1405         }
1406         return -EINPROGRESS;
1407 error:
1408         chcr_dec_wrcount(dev);
1409         return err;
1410 }
1411
1412 static int chcr_aes_decrypt(struct skcipher_request *req)
1413 {
1414         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1415         struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1416         struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
1417         struct chcr_dev *dev = c_ctx(tfm)->dev;
1418         struct sk_buff *skb = NULL;
1419         int err;
1420         struct chcr_context *ctx = c_ctx(tfm);
1421         unsigned int cpu;
1422
1423         cpu = get_cpu();
1424         reqctx->txqidx = cpu % ctx->ntxq;
1425         reqctx->rxqidx = cpu % ctx->nrxq;
1426         put_cpu();
1427
1428         err = chcr_inc_wrcount(dev);
1429         if (err)
1430                 return -ENXIO;
1431
1432         if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1433                                                 reqctx->txqidx) &&
1434                 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))))
1435                         return -ENOSPC;
1436         err = process_cipher(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx],
1437                              &skb, CHCR_DECRYPT_OP);
1438         if (err || !skb)
1439                 return err;
1440         skb->dev = u_ctx->lldi.ports[0];
1441         set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
1442         chcr_send_wr(skb);
1443         return -EINPROGRESS;
1444 }
1445 static int chcr_device_init(struct chcr_context *ctx)
1446 {
1447         struct uld_ctx *u_ctx = NULL;
1448         int txq_perchan, ntxq;
1449         int err = 0, rxq_perchan;
1450
1451         if (!ctx->dev) {
1452                 u_ctx = assign_chcr_device();
1453                 if (!u_ctx) {
1454                         err = -ENXIO;
1455                         pr_err("chcr device assignment fails\n");
1456                         goto out;
1457                 }
1458                 ctx->dev = &u_ctx->dev;
1459                 ntxq = u_ctx->lldi.ntxq;
1460                 rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan;
1461                 txq_perchan = ntxq / u_ctx->lldi.nchan;
1462                 ctx->ntxq = ntxq;
1463                 ctx->nrxq = u_ctx->lldi.nrxq;
1464                 ctx->rxq_perchan = rxq_perchan;
1465                 ctx->txq_perchan = txq_perchan;
1466         }
1467 out:
1468         return err;
1469 }
1470
1471 static int chcr_init_tfm(struct crypto_skcipher *tfm)
1472 {
1473         struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1474         struct chcr_context *ctx = crypto_skcipher_ctx(tfm);
1475         struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1476
1477         ablkctx->sw_cipher = crypto_alloc_skcipher(alg->base.cra_name, 0,
1478                                 CRYPTO_ALG_NEED_FALLBACK);
1479         if (IS_ERR(ablkctx->sw_cipher)) {
1480                 pr_err("failed to allocate fallback for %s\n", alg->base.cra_name);
1481                 return PTR_ERR(ablkctx->sw_cipher);
1482         }
1483         init_completion(&ctx->cbc_aes_aio_done);
1484         crypto_skcipher_set_reqsize(tfm, sizeof(struct chcr_skcipher_req_ctx) +
1485                                          crypto_skcipher_reqsize(ablkctx->sw_cipher));
1486
1487         return chcr_device_init(ctx);
1488 }
1489
1490 static int chcr_rfc3686_init(struct crypto_skcipher *tfm)
1491 {
1492         struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1493         struct chcr_context *ctx = crypto_skcipher_ctx(tfm);
1494         struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1495
1496         /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes))
1497          * cannot be used as fallback in chcr_handle_cipher_response
1498          */
1499         ablkctx->sw_cipher = crypto_alloc_skcipher("ctr(aes)", 0,
1500                                 CRYPTO_ALG_NEED_FALLBACK);
1501         if (IS_ERR(ablkctx->sw_cipher)) {
1502                 pr_err("failed to allocate fallback for %s\n", alg->base.cra_name);
1503                 return PTR_ERR(ablkctx->sw_cipher);
1504         }
1505         crypto_skcipher_set_reqsize(tfm, sizeof(struct chcr_skcipher_req_ctx) +
1506                                     crypto_skcipher_reqsize(ablkctx->sw_cipher));
1507         return chcr_device_init(ctx);
1508 }
1509
1510
1511 static void chcr_exit_tfm(struct crypto_skcipher *tfm)
1512 {
1513         struct chcr_context *ctx = crypto_skcipher_ctx(tfm);
1514         struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1515
1516         crypto_free_skcipher(ablkctx->sw_cipher);
1517 }
1518
1519 static int get_alg_config(struct algo_param *params,
1520                           unsigned int auth_size)
1521 {
1522         switch (auth_size) {
1523         case SHA1_DIGEST_SIZE:
1524                 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
1525                 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1;
1526                 params->result_size = SHA1_DIGEST_SIZE;
1527                 break;
1528         case SHA224_DIGEST_SIZE:
1529                 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1530                 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224;
1531                 params->result_size = SHA256_DIGEST_SIZE;
1532                 break;
1533         case SHA256_DIGEST_SIZE:
1534                 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1535                 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256;
1536                 params->result_size = SHA256_DIGEST_SIZE;
1537                 break;
1538         case SHA384_DIGEST_SIZE:
1539                 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1540                 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384;
1541                 params->result_size = SHA512_DIGEST_SIZE;
1542                 break;
1543         case SHA512_DIGEST_SIZE:
1544                 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1545                 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512;
1546                 params->result_size = SHA512_DIGEST_SIZE;
1547                 break;
1548         default:
1549                 pr_err("ERROR, unsupported digest size\n");
1550                 return -EINVAL;
1551         }
1552         return 0;
1553 }
1554
1555 static inline void chcr_free_shash(struct crypto_shash *base_hash)
1556 {
1557                 crypto_free_shash(base_hash);
1558 }
1559
1560 /**
1561  *      create_hash_wr - Create hash work request
1562  *      @req - Cipher req base
1563  */
1564 static struct sk_buff *create_hash_wr(struct ahash_request *req,
1565                                       struct hash_wr_param *param)
1566 {
1567         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1568         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1569         struct chcr_context *ctx = h_ctx(tfm);
1570         struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1571         struct sk_buff *skb = NULL;
1572         struct uld_ctx *u_ctx = ULD_CTX(ctx);
1573         struct chcr_wr *chcr_req;
1574         struct ulptx_sgl *ulptx;
1575         unsigned int nents = 0, transhdr_len;
1576         unsigned int temp = 0;
1577         gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1578                 GFP_ATOMIC;
1579         struct adapter *adap = padap(h_ctx(tfm)->dev);
1580         int error = 0;
1581         unsigned int rx_channel_id = req_ctx->rxqidx / ctx->rxq_perchan;
1582
1583         transhdr_len = HASH_TRANSHDR_SIZE(param->kctx_len);
1584         req_ctx->hctx_wr.imm = (transhdr_len + param->bfr_len +
1585                                 param->sg_len) <= SGE_MAX_WR_LEN;
1586         nents = sg_nents_xlen(req_ctx->hctx_wr.srcsg, param->sg_len,
1587                       CHCR_SRC_SG_SIZE, req_ctx->hctx_wr.src_ofst);
1588         nents += param->bfr_len ? 1 : 0;
1589         transhdr_len += req_ctx->hctx_wr.imm ? roundup(param->bfr_len +
1590                                 param->sg_len, 16) : (sgl_len(nents) * 8);
1591         transhdr_len = roundup(transhdr_len, 16);
1592
1593         skb = alloc_skb(transhdr_len, flags);
1594         if (!skb)
1595                 return ERR_PTR(-ENOMEM);
1596         chcr_req = __skb_put_zero(skb, transhdr_len);
1597
1598         chcr_req->sec_cpl.op_ivinsrtofst =
1599                 FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 0);
1600
1601         chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len);
1602
1603         chcr_req->sec_cpl.aadstart_cipherstop_hi =
1604                 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0);
1605         chcr_req->sec_cpl.cipherstop_lo_authinsert =
1606                 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0);
1607         chcr_req->sec_cpl.seqno_numivs =
1608                 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode,
1609                                          param->opad_needed, 0);
1610
1611         chcr_req->sec_cpl.ivgen_hdrlen =
1612                 FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0);
1613
1614         memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash,
1615                param->alg_prm.result_size);
1616
1617         if (param->opad_needed)
1618                 memcpy(chcr_req->key_ctx.key +
1619                        ((param->alg_prm.result_size <= 32) ? 32 :
1620                         CHCR_HASH_MAX_DIGEST_SIZE),
1621                        hmacctx->opad, param->alg_prm.result_size);
1622
1623         chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY,
1624                                             param->alg_prm.mk_size, 0,
1625                                             param->opad_needed,
1626                                             ((param->kctx_len +
1627                                              sizeof(chcr_req->key_ctx)) >> 4));
1628         chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1);
1629         ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + param->kctx_len +
1630                                      DUMMY_BYTES);
1631         if (param->bfr_len != 0) {
1632                 req_ctx->hctx_wr.dma_addr =
1633                         dma_map_single(&u_ctx->lldi.pdev->dev, req_ctx->reqbfr,
1634                                        param->bfr_len, DMA_TO_DEVICE);
1635                 if (dma_mapping_error(&u_ctx->lldi.pdev->dev,
1636                                        req_ctx->hctx_wr. dma_addr)) {
1637                         error = -ENOMEM;
1638                         goto err;
1639                 }
1640                 req_ctx->hctx_wr.dma_len = param->bfr_len;
1641         } else {
1642                 req_ctx->hctx_wr.dma_addr = 0;
1643         }
1644         chcr_add_hash_src_ent(req, ulptx, param);
1645         /* Request upto max wr size */
1646         temp = param->kctx_len + DUMMY_BYTES + (req_ctx->hctx_wr.imm ?
1647                                 (param->sg_len + param->bfr_len) : 0);
1648         atomic_inc(&adap->chcr_stats.digest_rqst);
1649         create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->hctx_wr.imm,
1650                     param->hash_size, transhdr_len,
1651                     temp,  0);
1652         req_ctx->hctx_wr.skb = skb;
1653         return skb;
1654 err:
1655         kfree_skb(skb);
1656         return  ERR_PTR(error);
1657 }
1658
1659 static int chcr_ahash_update(struct ahash_request *req)
1660 {
1661         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1662         struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1663         struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm));
1664         struct chcr_context *ctx = h_ctx(rtfm);
1665         struct chcr_dev *dev = h_ctx(rtfm)->dev;
1666         struct sk_buff *skb;
1667         u8 remainder = 0, bs;
1668         unsigned int nbytes = req->nbytes;
1669         struct hash_wr_param params;
1670         int error;
1671         unsigned int cpu;
1672
1673         cpu = get_cpu();
1674         req_ctx->txqidx = cpu % ctx->ntxq;
1675         req_ctx->rxqidx = cpu % ctx->nrxq;
1676         put_cpu();
1677
1678         bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1679
1680         if (nbytes + req_ctx->reqlen >= bs) {
1681                 remainder = (nbytes + req_ctx->reqlen) % bs;
1682                 nbytes = nbytes + req_ctx->reqlen - remainder;
1683         } else {
1684                 sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr
1685                                    + req_ctx->reqlen, nbytes, 0);
1686                 req_ctx->reqlen += nbytes;
1687                 return 0;
1688         }
1689         error = chcr_inc_wrcount(dev);
1690         if (error)
1691                 return -ENXIO;
1692         /* Detach state for CHCR means lldi or padap is freed. Increasing
1693          * inflight count for dev guarantees that lldi and padap is valid
1694          */
1695         if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1696                                                 req_ctx->txqidx) &&
1697                 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1698                         error = -ENOSPC;
1699                         goto err;
1700         }
1701
1702         chcr_init_hctx_per_wr(req_ctx);
1703         error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
1704         if (error) {
1705                 error = -ENOMEM;
1706                 goto err;
1707         }
1708         get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1709         params.kctx_len = roundup(params.alg_prm.result_size, 16);
1710         params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
1711                                      HASH_SPACE_LEFT(params.kctx_len), 0);
1712         if (params.sg_len > req->nbytes)
1713                 params.sg_len = req->nbytes;
1714         params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs) -
1715                         req_ctx->reqlen;
1716         params.opad_needed = 0;
1717         params.more = 1;
1718         params.last = 0;
1719         params.bfr_len = req_ctx->reqlen;
1720         params.scmd1 = 0;
1721         req_ctx->hctx_wr.srcsg = req->src;
1722
1723         params.hash_size = params.alg_prm.result_size;
1724         req_ctx->data_len += params.sg_len + params.bfr_len;
1725         skb = create_hash_wr(req, &params);
1726         if (IS_ERR(skb)) {
1727                 error = PTR_ERR(skb);
1728                 goto unmap;
1729         }
1730
1731         req_ctx->hctx_wr.processed += params.sg_len;
1732         if (remainder) {
1733                 /* Swap buffers */
1734                 swap(req_ctx->reqbfr, req_ctx->skbfr);
1735                 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
1736                                    req_ctx->reqbfr, remainder, req->nbytes -
1737                                    remainder);
1738         }
1739         req_ctx->reqlen = remainder;
1740         skb->dev = u_ctx->lldi.ports[0];
1741         set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
1742         chcr_send_wr(skb);
1743         return -EINPROGRESS;
1744 unmap:
1745         chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
1746 err:
1747         chcr_dec_wrcount(dev);
1748         return error;
1749 }
1750
1751 static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1)
1752 {
1753         memset(bfr_ptr, 0, bs);
1754         *bfr_ptr = 0x80;
1755         if (bs == 64)
1756                 *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1  << 3);
1757         else
1758                 *(__be64 *)(bfr_ptr + 120) =  cpu_to_be64(scmd1  << 3);
1759 }
1760
1761 static int chcr_ahash_final(struct ahash_request *req)
1762 {
1763         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1764         struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1765         struct chcr_dev *dev = h_ctx(rtfm)->dev;
1766         struct hash_wr_param params;
1767         struct sk_buff *skb;
1768         struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm));
1769         struct chcr_context *ctx = h_ctx(rtfm);
1770         u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1771         int error;
1772         unsigned int cpu;
1773
1774         cpu = get_cpu();
1775         req_ctx->txqidx = cpu % ctx->ntxq;
1776         req_ctx->rxqidx = cpu % ctx->nrxq;
1777         put_cpu();
1778
1779         error = chcr_inc_wrcount(dev);
1780         if (error)
1781                 return -ENXIO;
1782
1783         chcr_init_hctx_per_wr(req_ctx);
1784         if (is_hmac(crypto_ahash_tfm(rtfm)))
1785                 params.opad_needed = 1;
1786         else
1787                 params.opad_needed = 0;
1788         params.sg_len = 0;
1789         req_ctx->hctx_wr.isfinal = 1;
1790         get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1791         params.kctx_len = roundup(params.alg_prm.result_size, 16);
1792         if (is_hmac(crypto_ahash_tfm(rtfm))) {
1793                 params.opad_needed = 1;
1794                 params.kctx_len *= 2;
1795         } else {
1796                 params.opad_needed = 0;
1797         }
1798
1799         req_ctx->hctx_wr.result = 1;
1800         params.bfr_len = req_ctx->reqlen;
1801         req_ctx->data_len += params.bfr_len + params.sg_len;
1802         req_ctx->hctx_wr.srcsg = req->src;
1803         if (req_ctx->reqlen == 0) {
1804                 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1805                 params.last = 0;
1806                 params.more = 1;
1807                 params.scmd1 = 0;
1808                 params.bfr_len = bs;
1809
1810         } else {
1811                 params.scmd1 = req_ctx->data_len;
1812                 params.last = 1;
1813                 params.more = 0;
1814         }
1815         params.hash_size = crypto_ahash_digestsize(rtfm);
1816         skb = create_hash_wr(req, &params);
1817         if (IS_ERR(skb)) {
1818                 error = PTR_ERR(skb);
1819                 goto err;
1820         }
1821         req_ctx->reqlen = 0;
1822         skb->dev = u_ctx->lldi.ports[0];
1823         set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
1824         chcr_send_wr(skb);
1825         return -EINPROGRESS;
1826 err:
1827         chcr_dec_wrcount(dev);
1828         return error;
1829 }
1830
1831 static int chcr_ahash_finup(struct ahash_request *req)
1832 {
1833         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1834         struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1835         struct chcr_dev *dev = h_ctx(rtfm)->dev;
1836         struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm));
1837         struct chcr_context *ctx = h_ctx(rtfm);
1838         struct sk_buff *skb;
1839         struct hash_wr_param params;
1840         u8  bs;
1841         int error;
1842         unsigned int cpu;
1843
1844         cpu = get_cpu();
1845         req_ctx->txqidx = cpu % ctx->ntxq;
1846         req_ctx->rxqidx = cpu % ctx->nrxq;
1847         put_cpu();
1848
1849         bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1850         error = chcr_inc_wrcount(dev);
1851         if (error)
1852                 return -ENXIO;
1853
1854         if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1855                                                 req_ctx->txqidx) &&
1856                 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1857                         error = -ENOSPC;
1858                         goto err;
1859         }
1860         chcr_init_hctx_per_wr(req_ctx);
1861         error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
1862         if (error) {
1863                 error = -ENOMEM;
1864                 goto err;
1865         }
1866
1867         get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1868         params.kctx_len = roundup(params.alg_prm.result_size, 16);
1869         if (is_hmac(crypto_ahash_tfm(rtfm))) {
1870                 params.kctx_len *= 2;
1871                 params.opad_needed = 1;
1872         } else {
1873                 params.opad_needed = 0;
1874         }
1875
1876         params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
1877                                     HASH_SPACE_LEFT(params.kctx_len), 0);
1878         if (params.sg_len < req->nbytes) {
1879                 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1880                         params.kctx_len /= 2;
1881                         params.opad_needed = 0;
1882                 }
1883                 params.last = 0;
1884                 params.more = 1;
1885                 params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs)
1886                                         - req_ctx->reqlen;
1887                 params.hash_size = params.alg_prm.result_size;
1888                 params.scmd1 = 0;
1889         } else {
1890                 params.last = 1;
1891                 params.more = 0;
1892                 params.sg_len = req->nbytes;
1893                 params.hash_size = crypto_ahash_digestsize(rtfm);
1894                 params.scmd1 = req_ctx->data_len + req_ctx->reqlen +
1895                                 params.sg_len;
1896         }
1897         params.bfr_len = req_ctx->reqlen;
1898         req_ctx->data_len += params.bfr_len + params.sg_len;
1899         req_ctx->hctx_wr.result = 1;
1900         req_ctx->hctx_wr.srcsg = req->src;
1901         if ((req_ctx->reqlen + req->nbytes) == 0) {
1902                 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1903                 params.last = 0;
1904                 params.more = 1;
1905                 params.scmd1 = 0;
1906                 params.bfr_len = bs;
1907         }
1908         skb = create_hash_wr(req, &params);
1909         if (IS_ERR(skb)) {
1910                 error = PTR_ERR(skb);
1911                 goto unmap;
1912         }
1913         req_ctx->reqlen = 0;
1914         req_ctx->hctx_wr.processed += params.sg_len;
1915         skb->dev = u_ctx->lldi.ports[0];
1916         set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
1917         chcr_send_wr(skb);
1918         return -EINPROGRESS;
1919 unmap:
1920         chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
1921 err:
1922         chcr_dec_wrcount(dev);
1923         return error;
1924 }
1925
1926 static int chcr_ahash_digest(struct ahash_request *req)
1927 {
1928         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1929         struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1930         struct chcr_dev *dev = h_ctx(rtfm)->dev;
1931         struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm));
1932         struct chcr_context *ctx = h_ctx(rtfm);
1933         struct sk_buff *skb;
1934         struct hash_wr_param params;
1935         u8  bs;
1936         int error;
1937         unsigned int cpu;
1938
1939         cpu = get_cpu();
1940         req_ctx->txqidx = cpu % ctx->ntxq;
1941         req_ctx->rxqidx = cpu % ctx->nrxq;
1942         put_cpu();
1943
1944         rtfm->init(req);
1945         bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1946         error = chcr_inc_wrcount(dev);
1947         if (error)
1948                 return -ENXIO;
1949
1950         if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1951                                                 req_ctx->txqidx) &&
1952                 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1953                         error = -ENOSPC;
1954                         goto err;
1955         }
1956
1957         chcr_init_hctx_per_wr(req_ctx);
1958         error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
1959         if (error) {
1960                 error = -ENOMEM;
1961                 goto err;
1962         }
1963
1964         get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1965         params.kctx_len = roundup(params.alg_prm.result_size, 16);
1966         if (is_hmac(crypto_ahash_tfm(rtfm))) {
1967                 params.kctx_len *= 2;
1968                 params.opad_needed = 1;
1969         } else {
1970                 params.opad_needed = 0;
1971         }
1972         params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
1973                                 HASH_SPACE_LEFT(params.kctx_len), 0);
1974         if (params.sg_len < req->nbytes) {
1975                 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1976                         params.kctx_len /= 2;
1977                         params.opad_needed = 0;
1978                 }
1979                 params.last = 0;
1980                 params.more = 1;
1981                 params.scmd1 = 0;
1982                 params.sg_len = rounddown(params.sg_len, bs);
1983                 params.hash_size = params.alg_prm.result_size;
1984         } else {
1985                 params.sg_len = req->nbytes;
1986                 params.hash_size = crypto_ahash_digestsize(rtfm);
1987                 params.last = 1;
1988                 params.more = 0;
1989                 params.scmd1 = req->nbytes + req_ctx->data_len;
1990
1991         }
1992         params.bfr_len = 0;
1993         req_ctx->hctx_wr.result = 1;
1994         req_ctx->hctx_wr.srcsg = req->src;
1995         req_ctx->data_len += params.bfr_len + params.sg_len;
1996
1997         if (req->nbytes == 0) {
1998                 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1999                 params.more = 1;
2000                 params.bfr_len = bs;
2001         }
2002
2003         skb = create_hash_wr(req, &params);
2004         if (IS_ERR(skb)) {
2005                 error = PTR_ERR(skb);
2006                 goto unmap;
2007         }
2008         req_ctx->hctx_wr.processed += params.sg_len;
2009         skb->dev = u_ctx->lldi.ports[0];
2010         set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
2011         chcr_send_wr(skb);
2012         return -EINPROGRESS;
2013 unmap:
2014         chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
2015 err:
2016         chcr_dec_wrcount(dev);
2017         return error;
2018 }
2019
2020 static int chcr_ahash_continue(struct ahash_request *req)
2021 {
2022         struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
2023         struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr;
2024         struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
2025         struct chcr_context *ctx = h_ctx(rtfm);
2026         struct uld_ctx *u_ctx = ULD_CTX(ctx);
2027         struct sk_buff *skb;
2028         struct hash_wr_param params;
2029         u8  bs;
2030         int error;
2031         unsigned int cpu;
2032
2033         cpu = get_cpu();
2034         reqctx->txqidx = cpu % ctx->ntxq;
2035         reqctx->rxqidx = cpu % ctx->nrxq;
2036         put_cpu();
2037
2038         bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
2039         get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
2040         params.kctx_len = roundup(params.alg_prm.result_size, 16);
2041         if (is_hmac(crypto_ahash_tfm(rtfm))) {
2042                 params.kctx_len *= 2;
2043                 params.opad_needed = 1;
2044         } else {
2045                 params.opad_needed = 0;
2046         }
2047         params.sg_len = chcr_hash_ent_in_wr(hctx_wr->srcsg, 0,
2048                                             HASH_SPACE_LEFT(params.kctx_len),
2049                                             hctx_wr->src_ofst);
2050         if ((params.sg_len + hctx_wr->processed) > req->nbytes)
2051                 params.sg_len = req->nbytes - hctx_wr->processed;
2052         if (!hctx_wr->result ||
2053             ((params.sg_len + hctx_wr->processed) < req->nbytes)) {
2054                 if (is_hmac(crypto_ahash_tfm(rtfm))) {
2055                         params.kctx_len /= 2;
2056                         params.opad_needed = 0;
2057                 }
2058                 params.last = 0;
2059                 params.more = 1;
2060                 params.sg_len = rounddown(params.sg_len, bs);
2061                 params.hash_size = params.alg_prm.result_size;
2062                 params.scmd1 = 0;
2063         } else {
2064                 params.last = 1;
2065                 params.more = 0;
2066                 params.hash_size = crypto_ahash_digestsize(rtfm);
2067                 params.scmd1 = reqctx->data_len + params.sg_len;
2068         }
2069         params.bfr_len = 0;
2070         reqctx->data_len += params.sg_len;
2071         skb = create_hash_wr(req, &params);
2072         if (IS_ERR(skb)) {
2073                 error = PTR_ERR(skb);
2074                 goto err;
2075         }
2076         hctx_wr->processed += params.sg_len;
2077         skb->dev = u_ctx->lldi.ports[0];
2078         set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
2079         chcr_send_wr(skb);
2080         return 0;
2081 err:
2082         return error;
2083 }
2084
2085 static inline void chcr_handle_ahash_resp(struct ahash_request *req,
2086                                           unsigned char *input,
2087                                           int err)
2088 {
2089         struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
2090         struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr;
2091         int digestsize, updated_digestsize;
2092         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2093         struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm));
2094         struct chcr_dev *dev = h_ctx(tfm)->dev;
2095
2096         if (input == NULL)
2097                 goto out;
2098         digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));
2099         updated_digestsize = digestsize;
2100         if (digestsize == SHA224_DIGEST_SIZE)
2101                 updated_digestsize = SHA256_DIGEST_SIZE;
2102         else if (digestsize == SHA384_DIGEST_SIZE)
2103                 updated_digestsize = SHA512_DIGEST_SIZE;
2104
2105         if (hctx_wr->dma_addr) {
2106                 dma_unmap_single(&u_ctx->lldi.pdev->dev, hctx_wr->dma_addr,
2107                                  hctx_wr->dma_len, DMA_TO_DEVICE);
2108                 hctx_wr->dma_addr = 0;
2109         }
2110         if (hctx_wr->isfinal || ((hctx_wr->processed + reqctx->reqlen) ==
2111                                  req->nbytes)) {
2112                 if (hctx_wr->result == 1) {
2113                         hctx_wr->result = 0;
2114                         memcpy(req->result, input + sizeof(struct cpl_fw6_pld),
2115                                digestsize);
2116                 } else {
2117                         memcpy(reqctx->partial_hash,
2118                                input + sizeof(struct cpl_fw6_pld),
2119                                updated_digestsize);
2120
2121                 }
2122                 goto unmap;
2123         }
2124         memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld),
2125                updated_digestsize);
2126
2127         err = chcr_ahash_continue(req);
2128         if (err)
2129                 goto unmap;
2130         return;
2131 unmap:
2132         if (hctx_wr->is_sg_map)
2133                 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
2134
2135
2136 out:
2137         chcr_dec_wrcount(dev);
2138         req->base.complete(&req->base, err);
2139 }
2140
2141 /*
2142  *      chcr_handle_resp - Unmap the DMA buffers associated with the request
2143  *      @req: crypto request
2144  */
2145 int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
2146                          int err)
2147 {
2148         struct crypto_tfm *tfm = req->tfm;
2149         struct chcr_context *ctx = crypto_tfm_ctx(tfm);
2150         struct adapter *adap = padap(ctx->dev);
2151
2152         switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
2153         case CRYPTO_ALG_TYPE_AEAD:
2154                 err = chcr_handle_aead_resp(aead_request_cast(req), input, err);
2155                 break;
2156
2157         case CRYPTO_ALG_TYPE_SKCIPHER:
2158                  chcr_handle_cipher_resp(skcipher_request_cast(req),
2159                                                input, err);
2160                 break;
2161         case CRYPTO_ALG_TYPE_AHASH:
2162                 chcr_handle_ahash_resp(ahash_request_cast(req), input, err);
2163                 }
2164         atomic_inc(&adap->chcr_stats.complete);
2165         return err;
2166 }
2167 static int chcr_ahash_export(struct ahash_request *areq, void *out)
2168 {
2169         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2170         struct chcr_ahash_req_ctx *state = out;
2171
2172         state->reqlen = req_ctx->reqlen;
2173         state->data_len = req_ctx->data_len;
2174         memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen);
2175         memcpy(state->partial_hash, req_ctx->partial_hash,
2176                CHCR_HASH_MAX_DIGEST_SIZE);
2177         chcr_init_hctx_per_wr(state);
2178         return 0;
2179 }
2180
2181 static int chcr_ahash_import(struct ahash_request *areq, const void *in)
2182 {
2183         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2184         struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in;
2185
2186         req_ctx->reqlen = state->reqlen;
2187         req_ctx->data_len = state->data_len;
2188         req_ctx->reqbfr = req_ctx->bfr1;
2189         req_ctx->skbfr = req_ctx->bfr2;
2190         memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128);
2191         memcpy(req_ctx->partial_hash, state->partial_hash,
2192                CHCR_HASH_MAX_DIGEST_SIZE);
2193         chcr_init_hctx_per_wr(req_ctx);
2194         return 0;
2195 }
2196
2197 static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2198                              unsigned int keylen)
2199 {
2200         struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm));
2201         unsigned int digestsize = crypto_ahash_digestsize(tfm);
2202         unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2203         unsigned int i, err = 0, updated_digestsize;
2204
2205         SHASH_DESC_ON_STACK(shash, hmacctx->base_hash);
2206
2207         /* use the key to calculate the ipad and opad. ipad will sent with the
2208          * first request's data. opad will be sent with the final hash result
2209          * ipad in hmacctx->ipad and opad in hmacctx->opad location
2210          */
2211         shash->tfm = hmacctx->base_hash;
2212         if (keylen > bs) {
2213                 err = crypto_shash_digest(shash, key, keylen,
2214                                           hmacctx->ipad);
2215                 if (err)
2216                         goto out;
2217                 keylen = digestsize;
2218         } else {
2219                 memcpy(hmacctx->ipad, key, keylen);
2220         }
2221         memset(hmacctx->ipad + keylen, 0, bs - keylen);
2222         memcpy(hmacctx->opad, hmacctx->ipad, bs);
2223
2224         for (i = 0; i < bs / sizeof(int); i++) {
2225                 *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA;
2226                 *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA;
2227         }
2228
2229         updated_digestsize = digestsize;
2230         if (digestsize == SHA224_DIGEST_SIZE)
2231                 updated_digestsize = SHA256_DIGEST_SIZE;
2232         else if (digestsize == SHA384_DIGEST_SIZE)
2233                 updated_digestsize = SHA512_DIGEST_SIZE;
2234         err = chcr_compute_partial_hash(shash, hmacctx->ipad,
2235                                         hmacctx->ipad, digestsize);
2236         if (err)
2237                 goto out;
2238         chcr_change_order(hmacctx->ipad, updated_digestsize);
2239
2240         err = chcr_compute_partial_hash(shash, hmacctx->opad,
2241                                         hmacctx->opad, digestsize);
2242         if (err)
2243                 goto out;
2244         chcr_change_order(hmacctx->opad, updated_digestsize);
2245 out:
2246         return err;
2247 }
2248
2249 static int chcr_aes_xts_setkey(struct crypto_skcipher *cipher, const u8 *key,
2250                                unsigned int key_len)
2251 {
2252         struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
2253         unsigned short context_size = 0;
2254         int err;
2255
2256         err = chcr_cipher_fallback_setkey(cipher, key, key_len);
2257         if (err)
2258                 goto badkey_err;
2259
2260         memcpy(ablkctx->key, key, key_len);
2261         ablkctx->enckey_len = key_len;
2262         get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2);
2263         context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4;
2264         /* Both keys for xts must be aligned to 16 byte boundary
2265          * by padding with zeros. So for 24 byte keys padding 8 zeroes.
2266          */
2267         if (key_len == 48) {
2268                 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len
2269                                 + 16) >> 4;
2270                 memmove(ablkctx->key + 32, ablkctx->key + 24, 24);
2271                 memset(ablkctx->key + 24, 0, 8);
2272                 memset(ablkctx->key + 56, 0, 8);
2273                 ablkctx->enckey_len = 64;
2274                 ablkctx->key_ctx_hdr =
2275                         FILL_KEY_CTX_HDR(CHCR_KEYCTX_CIPHER_KEY_SIZE_192,
2276                                          CHCR_KEYCTX_NO_KEY, 1,
2277                                          0, context_size);
2278         } else {
2279                 ablkctx->key_ctx_hdr =
2280                 FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ?
2281                                  CHCR_KEYCTX_CIPHER_KEY_SIZE_128 :
2282                                  CHCR_KEYCTX_CIPHER_KEY_SIZE_256,
2283                                  CHCR_KEYCTX_NO_KEY, 1,
2284                                  0, context_size);
2285         }
2286         ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
2287         return 0;
2288 badkey_err:
2289         ablkctx->enckey_len = 0;
2290
2291         return err;
2292 }
2293
2294 static int chcr_sha_init(struct ahash_request *areq)
2295 {
2296         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2297         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2298         int digestsize =  crypto_ahash_digestsize(tfm);
2299
2300         req_ctx->data_len = 0;
2301         req_ctx->reqlen = 0;
2302         req_ctx->reqbfr = req_ctx->bfr1;
2303         req_ctx->skbfr = req_ctx->bfr2;
2304         copy_hash_init_values(req_ctx->partial_hash, digestsize);
2305
2306         return 0;
2307 }
2308
2309 static int chcr_sha_cra_init(struct crypto_tfm *tfm)
2310 {
2311         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2312                                  sizeof(struct chcr_ahash_req_ctx));
2313         return chcr_device_init(crypto_tfm_ctx(tfm));
2314 }
2315
2316 static int chcr_hmac_init(struct ahash_request *areq)
2317 {
2318         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2319         struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq);
2320         struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(rtfm));
2321         unsigned int digestsize = crypto_ahash_digestsize(rtfm);
2322         unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
2323
2324         chcr_sha_init(areq);
2325         req_ctx->data_len = bs;
2326         if (is_hmac(crypto_ahash_tfm(rtfm))) {
2327                 if (digestsize == SHA224_DIGEST_SIZE)
2328                         memcpy(req_ctx->partial_hash, hmacctx->ipad,
2329                                SHA256_DIGEST_SIZE);
2330                 else if (digestsize == SHA384_DIGEST_SIZE)
2331                         memcpy(req_ctx->partial_hash, hmacctx->ipad,
2332                                SHA512_DIGEST_SIZE);
2333                 else
2334                         memcpy(req_ctx->partial_hash, hmacctx->ipad,
2335                                digestsize);
2336         }
2337         return 0;
2338 }
2339
2340 static int chcr_hmac_cra_init(struct crypto_tfm *tfm)
2341 {
2342         struct chcr_context *ctx = crypto_tfm_ctx(tfm);
2343         struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
2344         unsigned int digestsize =
2345                 crypto_ahash_digestsize(__crypto_ahash_cast(tfm));
2346
2347         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2348                                  sizeof(struct chcr_ahash_req_ctx));
2349         hmacctx->base_hash = chcr_alloc_shash(digestsize);
2350         if (IS_ERR(hmacctx->base_hash))
2351                 return PTR_ERR(hmacctx->base_hash);
2352         return chcr_device_init(crypto_tfm_ctx(tfm));
2353 }
2354
2355 static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)
2356 {
2357         struct chcr_context *ctx = crypto_tfm_ctx(tfm);
2358         struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
2359
2360         if (hmacctx->base_hash) {
2361                 chcr_free_shash(hmacctx->base_hash);
2362                 hmacctx->base_hash = NULL;
2363         }
2364 }
2365
2366 inline void chcr_aead_common_exit(struct aead_request *req)
2367 {
2368         struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
2369         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2370         struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm));
2371
2372         chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op);
2373 }
2374
2375 static int chcr_aead_common_init(struct aead_request *req)
2376 {
2377         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2378         struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2379         struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
2380         unsigned int authsize = crypto_aead_authsize(tfm);
2381         int error = -EINVAL;
2382
2383         /* validate key size */
2384         if (aeadctx->enckey_len == 0)
2385                 goto err;
2386         if (reqctx->op && req->cryptlen < authsize)
2387                 goto err;
2388         if (reqctx->b0_len)
2389                 reqctx->scratch_pad = reqctx->iv + IV;
2390         else
2391                 reqctx->scratch_pad = NULL;
2392
2393         error = chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
2394                                   reqctx->op);
2395         if (error) {
2396                 error = -ENOMEM;
2397                 goto err;
2398         }
2399
2400         return 0;
2401 err:
2402         return error;
2403 }
2404
2405 static int chcr_aead_need_fallback(struct aead_request *req, int dst_nents,
2406                                    int aadmax, int wrlen,
2407                                    unsigned short op_type)
2408 {
2409         unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
2410
2411         if (((req->cryptlen - (op_type ? authsize : 0)) == 0) ||
2412             dst_nents > MAX_DSGL_ENT ||
2413             (req->assoclen > aadmax) ||
2414             (wrlen > SGE_MAX_WR_LEN))
2415                 return 1;
2416         return 0;
2417 }
2418
2419 static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type)
2420 {
2421         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2422         struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2423         struct aead_request *subreq = aead_request_ctx(req);
2424
2425         aead_request_set_tfm(subreq, aeadctx->sw_cipher);
2426         aead_request_set_callback(subreq, req->base.flags,
2427                                   req->base.complete, req->base.data);
2428         aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
2429                                  req->iv);
2430         aead_request_set_ad(subreq, req->assoclen);
2431         return op_type ? crypto_aead_decrypt(subreq) :
2432                 crypto_aead_encrypt(subreq);
2433 }
2434
2435 static struct sk_buff *create_authenc_wr(struct aead_request *req,
2436                                          unsigned short qid,
2437                                          int size)
2438 {
2439         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2440         struct chcr_context *ctx = a_ctx(tfm);
2441         struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2442         struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
2443         struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2444         struct sk_buff *skb = NULL;
2445         struct chcr_wr *chcr_req;
2446         struct cpl_rx_phys_dsgl *phys_cpl;
2447         struct ulptx_sgl *ulptx;
2448         unsigned int transhdr_len;
2449         unsigned int dst_size = 0, temp, subtype = get_aead_subtype(tfm);
2450         unsigned int   kctx_len = 0, dnents, snents;
2451         unsigned int  authsize = crypto_aead_authsize(tfm);
2452         int error = -EINVAL;
2453         u8 *ivptr;
2454         int null = 0;
2455         gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2456                 GFP_ATOMIC;
2457         struct adapter *adap = padap(ctx->dev);
2458         unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2459
2460         if (req->cryptlen == 0)
2461                 return NULL;
2462
2463         reqctx->b0_len = 0;
2464         error = chcr_aead_common_init(req);
2465         if (error)
2466                 return ERR_PTR(error);
2467
2468         if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL ||
2469                 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
2470                 null = 1;
2471         }
2472         dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen +
2473                 (reqctx->op ? -authsize : authsize), CHCR_DST_SG_SIZE, 0);
2474         dnents += MIN_AUTH_SG; // For IV
2475         snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen,
2476                                CHCR_SRC_SG_SIZE, 0);
2477         dst_size = get_space_for_phys_dsgl(dnents);
2478         kctx_len = (KEY_CONTEXT_CTX_LEN_G(ntohl(aeadctx->key_ctx_hdr)) << 4)
2479                 - sizeof(chcr_req->key_ctx);
2480         transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2481         reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen) <
2482                         SGE_MAX_WR_LEN;
2483         temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen, 16)
2484                         : (sgl_len(snents) * 8);
2485         transhdr_len += temp;
2486         transhdr_len = roundup(transhdr_len, 16);
2487
2488         if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE,
2489                                     transhdr_len, reqctx->op)) {
2490                 atomic_inc(&adap->chcr_stats.fallback);
2491                 chcr_aead_common_exit(req);
2492                 return ERR_PTR(chcr_aead_fallback(req, reqctx->op));
2493         }
2494         skb = alloc_skb(transhdr_len, flags);
2495         if (!skb) {
2496                 error = -ENOMEM;
2497                 goto err;
2498         }
2499
2500         chcr_req = __skb_put_zero(skb, transhdr_len);
2501
2502         temp  = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize;
2503
2504         /*
2505          * Input order  is AAD,IV and Payload. where IV should be included as
2506          * the part of authdata. All other fields should be filled according
2507          * to the hardware spec
2508          */
2509         chcr_req->sec_cpl.op_ivinsrtofst =
2510                                 FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1);
2511         chcr_req->sec_cpl.pldlen = htonl(req->assoclen + IV + req->cryptlen);
2512         chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2513                                         null ? 0 : 1 + IV,
2514                                         null ? 0 : IV + req->assoclen,
2515                                         req->assoclen + IV + 1,
2516                                         (temp & 0x1F0) >> 4);
2517         chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(
2518                                         temp & 0xF,
2519                                         null ? 0 : req->assoclen + IV + 1,
2520                                         temp, temp);
2521         if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL ||
2522             subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA)
2523                 temp = CHCR_SCMD_CIPHER_MODE_AES_CTR;
2524         else
2525                 temp = CHCR_SCMD_CIPHER_MODE_AES_CBC;
2526         chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op,
2527                                         (reqctx->op == CHCR_ENCRYPT_OP) ? 1 : 0,
2528                                         temp,
2529                                         actx->auth_mode, aeadctx->hmac_ctrl,
2530                                         IV >> 1);
2531         chcr_req->sec_cpl.ivgen_hdrlen =  FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
2532                                          0, 0, dst_size);
2533
2534         chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2535         if (reqctx->op == CHCR_ENCRYPT_OP ||
2536                 subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
2537                 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL)
2538                 memcpy(chcr_req->key_ctx.key, aeadctx->key,
2539                        aeadctx->enckey_len);
2540         else
2541                 memcpy(chcr_req->key_ctx.key, actx->dec_rrkey,
2542                        aeadctx->enckey_len);
2543
2544         memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16),
2545                actx->h_iopad, kctx_len - roundup(aeadctx->enckey_len, 16));
2546         phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2547         ivptr = (u8 *)(phys_cpl + 1) + dst_size;
2548         ulptx = (struct ulptx_sgl *)(ivptr + IV);
2549         if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
2550             subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
2551                 memcpy(ivptr, aeadctx->nonce, CTR_RFC3686_NONCE_SIZE);
2552                 memcpy(ivptr + CTR_RFC3686_NONCE_SIZE, req->iv,
2553                                 CTR_RFC3686_IV_SIZE);
2554                 *(__be32 *)(ivptr + CTR_RFC3686_NONCE_SIZE +
2555                         CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
2556         } else {
2557                 memcpy(ivptr, req->iv, IV);
2558         }
2559         chcr_add_aead_dst_ent(req, phys_cpl, qid);
2560         chcr_add_aead_src_ent(req, ulptx);
2561         atomic_inc(&adap->chcr_stats.cipher_rqst);
2562         temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV +
2563                 kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen) : 0);
2564         create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size,
2565                    transhdr_len, temp, 0);
2566         reqctx->skb = skb;
2567
2568         return skb;
2569 err:
2570         chcr_aead_common_exit(req);
2571
2572         return ERR_PTR(error);
2573 }
2574
2575 int chcr_aead_dma_map(struct device *dev,
2576                       struct aead_request *req,
2577                       unsigned short op_type)
2578 {
2579         int error;
2580         struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
2581         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2582         unsigned int authsize = crypto_aead_authsize(tfm);
2583         int src_len, dst_len;
2584
2585         /* calculate and handle src and dst sg length separately
2586          * for inplace and out-of place operations
2587          */
2588         if (req->src == req->dst) {
2589                 src_len = req->assoclen + req->cryptlen + (op_type ?
2590                                                         0 : authsize);
2591                 dst_len = src_len;
2592         } else {
2593                 src_len = req->assoclen + req->cryptlen;
2594                 dst_len = req->assoclen + req->cryptlen + (op_type ?
2595                                                         -authsize : authsize);
2596         }
2597
2598         if (!req->cryptlen || !src_len || !dst_len)
2599                 return 0;
2600         reqctx->iv_dma = dma_map_single(dev, reqctx->iv, (IV + reqctx->b0_len),
2601                                         DMA_BIDIRECTIONAL);
2602         if (dma_mapping_error(dev, reqctx->iv_dma))
2603                 return -ENOMEM;
2604         if (reqctx->b0_len)
2605                 reqctx->b0_dma = reqctx->iv_dma + IV;
2606         else
2607                 reqctx->b0_dma = 0;
2608         if (req->src == req->dst) {
2609                 error = dma_map_sg(dev, req->src,
2610                                 sg_nents_for_len(req->src, src_len),
2611                                         DMA_BIDIRECTIONAL);
2612                 if (!error)
2613                         goto err;
2614         } else {
2615                 error = dma_map_sg(dev, req->src,
2616                                    sg_nents_for_len(req->src, src_len),
2617                                    DMA_TO_DEVICE);
2618                 if (!error)
2619                         goto err;
2620                 error = dma_map_sg(dev, req->dst,
2621                                    sg_nents_for_len(req->dst, dst_len),
2622                                    DMA_FROM_DEVICE);
2623                 if (!error) {
2624                         dma_unmap_sg(dev, req->src,
2625                                      sg_nents_for_len(req->src, src_len),
2626                                      DMA_TO_DEVICE);
2627                         goto err;
2628                 }
2629         }
2630
2631         return 0;
2632 err:
2633         dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
2634         return -ENOMEM;
2635 }
2636
2637 void chcr_aead_dma_unmap(struct device *dev,
2638                          struct aead_request *req,
2639                          unsigned short op_type)
2640 {
2641         struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
2642         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2643         unsigned int authsize = crypto_aead_authsize(tfm);
2644         int src_len, dst_len;
2645
2646         /* calculate and handle src and dst sg length separately
2647          * for inplace and out-of place operations
2648          */
2649         if (req->src == req->dst) {
2650                 src_len = req->assoclen + req->cryptlen + (op_type ?
2651                                                         0 : authsize);
2652                 dst_len = src_len;
2653         } else {
2654                 src_len = req->assoclen + req->cryptlen;
2655                 dst_len = req->assoclen + req->cryptlen + (op_type ?
2656                                                 -authsize : authsize);
2657         }
2658
2659         if (!req->cryptlen || !src_len || !dst_len)
2660                 return;
2661
2662         dma_unmap_single(dev, reqctx->iv_dma, (IV + reqctx->b0_len),
2663                                         DMA_BIDIRECTIONAL);
2664         if (req->src == req->dst) {
2665                 dma_unmap_sg(dev, req->src,
2666                              sg_nents_for_len(req->src, src_len),
2667                              DMA_BIDIRECTIONAL);
2668         } else {
2669                 dma_unmap_sg(dev, req->src,
2670                              sg_nents_for_len(req->src, src_len),
2671                              DMA_TO_DEVICE);
2672                 dma_unmap_sg(dev, req->dst,
2673                              sg_nents_for_len(req->dst, dst_len),
2674                              DMA_FROM_DEVICE);
2675         }
2676 }
2677
2678 void chcr_add_aead_src_ent(struct aead_request *req,
2679                            struct ulptx_sgl *ulptx)
2680 {
2681         struct ulptx_walk ulp_walk;
2682         struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
2683
2684         if (reqctx->imm) {
2685                 u8 *buf = (u8 *)ulptx;
2686
2687                 if (reqctx->b0_len) {
2688                         memcpy(buf, reqctx->scratch_pad, reqctx->b0_len);
2689                         buf += reqctx->b0_len;
2690                 }
2691                 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
2692                                    buf, req->cryptlen + req->assoclen, 0);
2693         } else {
2694                 ulptx_walk_init(&ulp_walk, ulptx);
2695                 if (reqctx->b0_len)
2696                         ulptx_walk_add_page(&ulp_walk, reqctx->b0_len,
2697                                             reqctx->b0_dma);
2698                 ulptx_walk_add_sg(&ulp_walk, req->src, req->cryptlen +
2699                                   req->assoclen,  0);
2700                 ulptx_walk_end(&ulp_walk);
2701         }
2702 }
2703
2704 void chcr_add_aead_dst_ent(struct aead_request *req,
2705                            struct cpl_rx_phys_dsgl *phys_cpl,
2706                            unsigned short qid)
2707 {
2708         struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
2709         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2710         struct dsgl_walk dsgl_walk;
2711         unsigned int authsize = crypto_aead_authsize(tfm);
2712         struct chcr_context *ctx = a_ctx(tfm);
2713         u32 temp;
2714         unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2715
2716         dsgl_walk_init(&dsgl_walk, phys_cpl);
2717         dsgl_walk_add_page(&dsgl_walk, IV + reqctx->b0_len, reqctx->iv_dma);
2718         temp = req->assoclen + req->cryptlen +
2719                 (reqctx->op ? -authsize : authsize);
2720         dsgl_walk_add_sg(&dsgl_walk, req->dst, temp, 0);
2721         dsgl_walk_end(&dsgl_walk, qid, rx_channel_id);
2722 }
2723
2724 void chcr_add_cipher_src_ent(struct skcipher_request *req,
2725                              void *ulptx,
2726                              struct  cipher_wr_param *wrparam)
2727 {
2728         struct ulptx_walk ulp_walk;
2729         struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
2730         u8 *buf = ulptx;
2731
2732         memcpy(buf, reqctx->iv, IV);
2733         buf += IV;
2734         if (reqctx->imm) {
2735                 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
2736                                    buf, wrparam->bytes, reqctx->processed);
2737         } else {
2738                 ulptx_walk_init(&ulp_walk, (struct ulptx_sgl *)buf);
2739                 ulptx_walk_add_sg(&ulp_walk, reqctx->srcsg, wrparam->bytes,
2740                                   reqctx->src_ofst);
2741                 reqctx->srcsg = ulp_walk.last_sg;
2742                 reqctx->src_ofst = ulp_walk.last_sg_len;
2743                 ulptx_walk_end(&ulp_walk);
2744         }
2745 }
2746
2747 void chcr_add_cipher_dst_ent(struct skcipher_request *req,
2748                              struct cpl_rx_phys_dsgl *phys_cpl,
2749                              struct  cipher_wr_param *wrparam,
2750                              unsigned short qid)
2751 {
2752         struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
2753         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(wrparam->req);
2754         struct chcr_context *ctx = c_ctx(tfm);
2755         struct dsgl_walk dsgl_walk;
2756         unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2757
2758         dsgl_walk_init(&dsgl_walk, phys_cpl);
2759         dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes,
2760                          reqctx->dst_ofst);
2761         reqctx->dstsg = dsgl_walk.last_sg;
2762         reqctx->dst_ofst = dsgl_walk.last_sg_len;
2763         dsgl_walk_end(&dsgl_walk, qid, rx_channel_id);
2764 }
2765
2766 void chcr_add_hash_src_ent(struct ahash_request *req,
2767                            struct ulptx_sgl *ulptx,
2768                            struct hash_wr_param *param)
2769 {
2770         struct ulptx_walk ulp_walk;
2771         struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
2772
2773         if (reqctx->hctx_wr.imm) {
2774                 u8 *buf = (u8 *)ulptx;
2775
2776                 if (param->bfr_len) {
2777                         memcpy(buf, reqctx->reqbfr, param->bfr_len);
2778                         buf += param->bfr_len;
2779                 }
2780
2781                 sg_pcopy_to_buffer(reqctx->hctx_wr.srcsg,
2782                                    sg_nents(reqctx->hctx_wr.srcsg), buf,
2783                                    param->sg_len, 0);
2784         } else {
2785                 ulptx_walk_init(&ulp_walk, ulptx);
2786                 if (param->bfr_len)
2787                         ulptx_walk_add_page(&ulp_walk, param->bfr_len,
2788                                             reqctx->hctx_wr.dma_addr);
2789                 ulptx_walk_add_sg(&ulp_walk, reqctx->hctx_wr.srcsg,
2790                                   param->sg_len, reqctx->hctx_wr.src_ofst);
2791                 reqctx->hctx_wr.srcsg = ulp_walk.last_sg;
2792                 reqctx->hctx_wr.src_ofst = ulp_walk.last_sg_len;
2793                 ulptx_walk_end(&ulp_walk);
2794         }
2795 }
2796
2797 int chcr_hash_dma_map(struct device *dev,
2798                       struct ahash_request *req)
2799 {
2800         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
2801         int error = 0;
2802
2803         if (!req->nbytes)
2804                 return 0;
2805         error = dma_map_sg(dev, req->src, sg_nents(req->src),
2806                            DMA_TO_DEVICE);
2807         if (!error)
2808                 return -ENOMEM;
2809         req_ctx->hctx_wr.is_sg_map = 1;
2810         return 0;
2811 }
2812
2813 void chcr_hash_dma_unmap(struct device *dev,
2814                          struct ahash_request *req)
2815 {
2816         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
2817
2818         if (!req->nbytes)
2819                 return;
2820
2821         dma_unmap_sg(dev, req->src, sg_nents(req->src),
2822                            DMA_TO_DEVICE);
2823         req_ctx->hctx_wr.is_sg_map = 0;
2824
2825 }
2826
2827 int chcr_cipher_dma_map(struct device *dev,
2828                         struct skcipher_request *req)
2829 {
2830         int error;
2831
2832         if (req->src == req->dst) {
2833                 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2834                                    DMA_BIDIRECTIONAL);
2835                 if (!error)
2836                         goto err;
2837         } else {
2838                 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2839                                    DMA_TO_DEVICE);
2840                 if (!error)
2841                         goto err;
2842                 error = dma_map_sg(dev, req->dst, sg_nents(req->dst),
2843                                    DMA_FROM_DEVICE);
2844                 if (!error) {
2845                         dma_unmap_sg(dev, req->src, sg_nents(req->src),
2846                                    DMA_TO_DEVICE);
2847                         goto err;
2848                 }
2849         }
2850
2851         return 0;
2852 err:
2853         return -ENOMEM;
2854 }
2855
2856 void chcr_cipher_dma_unmap(struct device *dev,
2857                            struct skcipher_request *req)
2858 {
2859         if (req->src == req->dst) {
2860                 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2861                                    DMA_BIDIRECTIONAL);
2862         } else {
2863                 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2864                                    DMA_TO_DEVICE);
2865                 dma_unmap_sg(dev, req->dst, sg_nents(req->dst),
2866                                    DMA_FROM_DEVICE);
2867         }
2868 }
2869
2870 static int set_msg_len(u8 *block, unsigned int msglen, int csize)
2871 {
2872         __be32 data;
2873
2874         memset(block, 0, csize);
2875         block += csize;
2876
2877         if (csize >= 4)
2878                 csize = 4;
2879         else if (msglen > (unsigned int)(1 << (8 * csize)))
2880                 return -EOVERFLOW;
2881
2882         data = cpu_to_be32(msglen);
2883         memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
2884
2885         return 0;
2886 }
2887
2888 static int generate_b0(struct aead_request *req, u8 *ivptr,
2889                         unsigned short op_type)
2890 {
2891         unsigned int l, lp, m;
2892         int rc;
2893         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2894         struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2895         u8 *b0 = reqctx->scratch_pad;
2896
2897         m = crypto_aead_authsize(aead);
2898
2899         memcpy(b0, ivptr, 16);
2900
2901         lp = b0[0];
2902         l = lp + 1;
2903
2904         /* set m, bits 3-5 */
2905         *b0 |= (8 * ((m - 2) / 2));
2906
2907         /* set adata, bit 6, if associated data is used */
2908         if (req->assoclen)
2909                 *b0 |= 64;
2910         rc = set_msg_len(b0 + 16 - l,
2911                          (op_type == CHCR_DECRYPT_OP) ?
2912                          req->cryptlen - m : req->cryptlen, l);
2913
2914         return rc;
2915 }
2916
2917 static inline int crypto_ccm_check_iv(const u8 *iv)
2918 {
2919         /* 2 <= L <= 8, so 1 <= L' <= 7. */
2920         if (iv[0] < 1 || iv[0] > 7)
2921                 return -EINVAL;
2922
2923         return 0;
2924 }
2925
2926 static int ccm_format_packet(struct aead_request *req,
2927                              u8 *ivptr,
2928                              unsigned int sub_type,
2929                              unsigned short op_type,
2930                              unsigned int assoclen)
2931 {
2932         struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2933         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2934         struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2935         int rc = 0;
2936
2937         if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2938                 ivptr[0] = 3;
2939                 memcpy(ivptr + 1, &aeadctx->salt[0], 3);
2940                 memcpy(ivptr + 4, req->iv, 8);
2941                 memset(ivptr + 12, 0, 4);
2942         } else {
2943                 memcpy(ivptr, req->iv, 16);
2944         }
2945         if (assoclen)
2946                 put_unaligned_be16(assoclen, &reqctx->scratch_pad[16]);
2947
2948         rc = generate_b0(req, ivptr, op_type);
2949         /* zero the ctr value */
2950         memset(ivptr + 15 - ivptr[0], 0, ivptr[0] + 1);
2951         return rc;
2952 }
2953
2954 static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
2955                                   unsigned int dst_size,
2956                                   struct aead_request *req,
2957                                   unsigned short op_type)
2958 {
2959         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2960         struct chcr_context *ctx = a_ctx(tfm);
2961         struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2962         struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2963         unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM;
2964         unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC;
2965         unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2966         unsigned int ccm_xtra;
2967         unsigned int tag_offset = 0, auth_offset = 0;
2968         unsigned int assoclen;
2969
2970         if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2971                 assoclen = req->assoclen - 8;
2972         else
2973                 assoclen = req->assoclen;
2974         ccm_xtra = CCM_B0_SIZE +
2975                 ((assoclen) ? CCM_AAD_FIELD_SIZE : 0);
2976
2977         auth_offset = req->cryptlen ?
2978                 (req->assoclen + IV + 1 + ccm_xtra) : 0;
2979         if (op_type == CHCR_DECRYPT_OP) {
2980                 if (crypto_aead_authsize(tfm) != req->cryptlen)
2981                         tag_offset = crypto_aead_authsize(tfm);
2982                 else
2983                         auth_offset = 0;
2984         }
2985
2986         sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1);
2987         sec_cpl->pldlen =
2988                 htonl(req->assoclen + IV + req->cryptlen + ccm_xtra);
2989         /* For CCM there wil be b0 always. So AAD start will be 1 always */
2990         sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2991                                 1 + IV, IV + assoclen + ccm_xtra,
2992                                 req->assoclen + IV + 1 + ccm_xtra, 0);
2993
2994         sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0,
2995                                         auth_offset, tag_offset,
2996                                         (op_type == CHCR_ENCRYPT_OP) ? 0 :
2997                                         crypto_aead_authsize(tfm));
2998         sec_cpl->seqno_numivs =  FILL_SEC_CPL_SCMD0_SEQNO(op_type,
2999                                         (op_type == CHCR_ENCRYPT_OP) ? 0 : 1,
3000                                         cipher_mode, mac_mode,
3001                                         aeadctx->hmac_ctrl, IV >> 1);
3002
3003         sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0,
3004                                         0, dst_size);
3005 }
3006
3007 static int aead_ccm_validate_input(unsigned short op_type,
3008                                    struct aead_request *req,
3009                                    struct chcr_aead_ctx *aeadctx,
3010                                    unsigned int sub_type)
3011 {
3012         if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
3013                 if (crypto_ccm_check_iv(req->iv)) {
3014                         pr_err("CCM: IV check fails\n");
3015                         return -EINVAL;
3016                 }
3017         } else {
3018                 if (req->assoclen != 16 && req->assoclen != 20) {
3019                         pr_err("RFC4309: Invalid AAD length %d\n",
3020                                req->assoclen);
3021                         return -EINVAL;
3022                 }
3023         }
3024         return 0;
3025 }
3026
3027 static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
3028                                           unsigned short qid,
3029                                           int size)
3030 {
3031         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3032         struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3033         struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3034         struct sk_buff *skb = NULL;
3035         struct chcr_wr *chcr_req;
3036         struct cpl_rx_phys_dsgl *phys_cpl;
3037         struct ulptx_sgl *ulptx;
3038         unsigned int transhdr_len;
3039         unsigned int dst_size = 0, kctx_len, dnents, temp, snents;
3040         unsigned int sub_type, assoclen = req->assoclen;
3041         unsigned int authsize = crypto_aead_authsize(tfm);
3042         int error = -EINVAL;
3043         u8 *ivptr;
3044         gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
3045                 GFP_ATOMIC;
3046         struct adapter *adap = padap(a_ctx(tfm)->dev);
3047
3048         sub_type = get_aead_subtype(tfm);
3049         if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
3050                 assoclen -= 8;
3051         reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0);
3052         error = chcr_aead_common_init(req);
3053         if (error)
3054                 return ERR_PTR(error);
3055
3056         error = aead_ccm_validate_input(reqctx->op, req, aeadctx, sub_type);
3057         if (error)
3058                 goto err;
3059         dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen
3060                         + (reqctx->op ? -authsize : authsize),
3061                         CHCR_DST_SG_SIZE, 0);
3062         dnents += MIN_CCM_SG; // For IV and B0
3063         dst_size = get_space_for_phys_dsgl(dnents);
3064         snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen,
3065                                CHCR_SRC_SG_SIZE, 0);
3066         snents += MIN_CCM_SG; //For B0
3067         kctx_len = roundup(aeadctx->enckey_len, 16) * 2;
3068         transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
3069         reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen +
3070                        reqctx->b0_len) <= SGE_MAX_WR_LEN;
3071         temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen +
3072                                      reqctx->b0_len, 16) :
3073                 (sgl_len(snents) *  8);
3074         transhdr_len += temp;
3075         transhdr_len = roundup(transhdr_len, 16);
3076
3077         if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE -
3078                                 reqctx->b0_len, transhdr_len, reqctx->op)) {
3079                 atomic_inc(&adap->chcr_stats.fallback);
3080                 chcr_aead_common_exit(req);
3081                 return ERR_PTR(chcr_aead_fallback(req, reqctx->op));
3082         }
3083         skb = alloc_skb(transhdr_len,  flags);
3084
3085         if (!skb) {
3086                 error = -ENOMEM;
3087                 goto err;
3088         }
3089
3090         chcr_req = __skb_put_zero(skb, transhdr_len);
3091
3092         fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, reqctx->op);
3093
3094         chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
3095         memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
3096         memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16),
3097                         aeadctx->key, aeadctx->enckey_len);
3098
3099         phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
3100         ivptr = (u8 *)(phys_cpl + 1) + dst_size;
3101         ulptx = (struct ulptx_sgl *)(ivptr + IV);
3102         error = ccm_format_packet(req, ivptr, sub_type, reqctx->op, assoclen);
3103         if (error)
3104                 goto dstmap_fail;
3105         chcr_add_aead_dst_ent(req, phys_cpl, qid);
3106         chcr_add_aead_src_ent(req, ulptx);
3107
3108         atomic_inc(&adap->chcr_stats.aead_rqst);
3109         temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV +
3110                 kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen +
3111                 reqctx->b0_len) : 0);
3112         create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, 0,
3113                     transhdr_len, temp, 0);
3114         reqctx->skb = skb;
3115
3116         return skb;
3117 dstmap_fail:
3118         kfree_skb(skb);
3119 err:
3120         chcr_aead_common_exit(req);
3121         return ERR_PTR(error);
3122 }
3123
3124 static struct sk_buff *create_gcm_wr(struct aead_request *req,
3125                                      unsigned short qid,
3126                                      int size)
3127 {
3128         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3129         struct chcr_context *ctx = a_ctx(tfm);
3130         struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
3131         struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
3132         struct sk_buff *skb = NULL;
3133         struct chcr_wr *chcr_req;
3134         struct cpl_rx_phys_dsgl *phys_cpl;
3135         struct ulptx_sgl *ulptx;
3136         unsigned int transhdr_len, dnents = 0, snents;
3137         unsigned int dst_size = 0, temp = 0, kctx_len, assoclen = req->assoclen;
3138         unsigned int authsize = crypto_aead_authsize(tfm);
3139         int error = -EINVAL;
3140         u8 *ivptr;
3141         gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
3142                 GFP_ATOMIC;
3143         struct adapter *adap = padap(ctx->dev);
3144         unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
3145
3146         if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106)
3147                 assoclen = req->assoclen - 8;
3148
3149         reqctx->b0_len = 0;
3150         error = chcr_aead_common_init(req);
3151         if (error)
3152                 return ERR_PTR(error);
3153         dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen +
3154                                 (reqctx->op ? -authsize : authsize),
3155                                 CHCR_DST_SG_SIZE, 0);
3156         snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen,
3157                                CHCR_SRC_SG_SIZE, 0);
3158         dnents += MIN_GCM_SG; // For IV
3159         dst_size = get_space_for_phys_dsgl(dnents);
3160         kctx_len = roundup(aeadctx->enckey_len, 16) + AEAD_H_SIZE;
3161         transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
3162         reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen) <=
3163                         SGE_MAX_WR_LEN;
3164         temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen, 16) :
3165                 (sgl_len(snents) * 8);
3166         transhdr_len += temp;
3167         transhdr_len = roundup(transhdr_len, 16);
3168         if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE,
3169                             transhdr_len, reqctx->op)) {
3170
3171                 atomic_inc(&adap->chcr_stats.fallback);
3172                 chcr_aead_common_exit(req);
3173                 return ERR_PTR(chcr_aead_fallback(req, reqctx->op));
3174         }
3175         skb = alloc_skb(transhdr_len, flags);
3176         if (!skb) {
3177                 error = -ENOMEM;
3178                 goto err;
3179         }
3180
3181         chcr_req = __skb_put_zero(skb, transhdr_len);
3182
3183         //Offset of tag from end
3184         temp = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize;
3185         chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(
3186                                                 rx_channel_id, 2, 1);
3187         chcr_req->sec_cpl.pldlen =
3188                 htonl(req->assoclen + IV + req->cryptlen);
3189         chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
3190                                         assoclen ? 1 + IV : 0,
3191                                         assoclen ? IV + assoclen : 0,
3192                                         req->assoclen + IV + 1, 0);
3193         chcr_req->sec_cpl.cipherstop_lo_authinsert =
3194                         FILL_SEC_CPL_AUTHINSERT(0, req->assoclen + IV + 1,
3195                                                 temp, temp);
3196         chcr_req->sec_cpl.seqno_numivs =
3197                         FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, (reqctx->op ==
3198                                         CHCR_ENCRYPT_OP) ? 1 : 0,
3199                                         CHCR_SCMD_CIPHER_MODE_AES_GCM,
3200                                         CHCR_SCMD_AUTH_MODE_GHASH,
3201                                         aeadctx->hmac_ctrl, IV >> 1);
3202         chcr_req->sec_cpl.ivgen_hdrlen =  FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
3203                                         0, 0, dst_size);
3204         chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
3205         memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
3206         memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16),
3207                GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE);
3208
3209         phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
3210         ivptr = (u8 *)(phys_cpl + 1) + dst_size;
3211         /* prepare a 16 byte iv */
3212         /* S   A   L  T |  IV | 0x00000001 */
3213         if (get_aead_subtype(tfm) ==
3214             CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) {
3215                 memcpy(ivptr, aeadctx->salt, 4);
3216                 memcpy(ivptr + 4, req->iv, GCM_RFC4106_IV_SIZE);
3217         } else {
3218                 memcpy(ivptr, req->iv, GCM_AES_IV_SIZE);
3219         }
3220         put_unaligned_be32(0x01, &ivptr[12]);
3221         ulptx = (struct ulptx_sgl *)(ivptr + 16);
3222
3223         chcr_add_aead_dst_ent(req, phys_cpl, qid);
3224         chcr_add_aead_src_ent(req, ulptx);
3225         atomic_inc(&adap->chcr_stats.aead_rqst);
3226         temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV +
3227                 kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen) : 0);
3228         create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size,
3229                     transhdr_len, temp, reqctx->verify);
3230         reqctx->skb = skb;
3231         return skb;
3232
3233 err:
3234         chcr_aead_common_exit(req);
3235         return ERR_PTR(error);
3236 }
3237
3238
3239
3240 static int chcr_aead_cra_init(struct crypto_aead *tfm)
3241 {
3242         struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3243         struct aead_alg *alg = crypto_aead_alg(tfm);
3244
3245         aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0,
3246                                                CRYPTO_ALG_NEED_FALLBACK |
3247                                                CRYPTO_ALG_ASYNC);
3248         if  (IS_ERR(aeadctx->sw_cipher))
3249                 return PTR_ERR(aeadctx->sw_cipher);
3250         crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx),
3251                                  sizeof(struct aead_request) +
3252                                  crypto_aead_reqsize(aeadctx->sw_cipher)));
3253         return chcr_device_init(a_ctx(tfm));
3254 }
3255
3256 static void chcr_aead_cra_exit(struct crypto_aead *tfm)
3257 {
3258         struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3259
3260         crypto_free_aead(aeadctx->sw_cipher);
3261 }
3262
3263 static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm,
3264                                         unsigned int authsize)
3265 {
3266         struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3267
3268         aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP;
3269         aeadctx->mayverify = VERIFY_HW;
3270         return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
3271 }
3272 static int chcr_authenc_setauthsize(struct crypto_aead *tfm,
3273                                     unsigned int authsize)
3274 {
3275         struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3276         u32 maxauth = crypto_aead_maxauthsize(tfm);
3277
3278         /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not
3279          * true for sha1. authsize == 12 condition should be before
3280          * authsize == (maxauth >> 1)
3281          */
3282         if (authsize == ICV_4) {
3283                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3284                 aeadctx->mayverify = VERIFY_HW;
3285         } else if (authsize == ICV_6) {
3286                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
3287                 aeadctx->mayverify = VERIFY_HW;
3288         } else if (authsize == ICV_10) {
3289                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
3290                 aeadctx->mayverify = VERIFY_HW;
3291         } else if (authsize == ICV_12) {
3292                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3293                 aeadctx->mayverify = VERIFY_HW;
3294         } else if (authsize == ICV_14) {
3295                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3296                 aeadctx->mayverify = VERIFY_HW;
3297         } else if (authsize == (maxauth >> 1)) {
3298                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3299                 aeadctx->mayverify = VERIFY_HW;
3300         } else if (authsize == maxauth) {
3301                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3302                 aeadctx->mayverify = VERIFY_HW;
3303         } else {
3304                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3305                 aeadctx->mayverify = VERIFY_SW;
3306         }
3307         return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
3308 }
3309
3310
3311 static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
3312 {
3313         struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3314
3315         switch (authsize) {
3316         case ICV_4:
3317                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3318                 aeadctx->mayverify = VERIFY_HW;
3319                 break;
3320         case ICV_8:
3321                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3322                 aeadctx->mayverify = VERIFY_HW;
3323                 break;
3324         case ICV_12:
3325                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3326                 aeadctx->mayverify = VERIFY_HW;
3327                 break;
3328         case ICV_14:
3329                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3330                 aeadctx->mayverify = VERIFY_HW;
3331                 break;
3332         case ICV_16:
3333                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3334                 aeadctx->mayverify = VERIFY_HW;
3335                 break;
3336         case ICV_13:
3337         case ICV_15:
3338                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3339                 aeadctx->mayverify = VERIFY_SW;
3340                 break;
3341         default:
3342                 return -EINVAL;
3343         }
3344         return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
3345 }
3346
3347 static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm,
3348                                           unsigned int authsize)
3349 {
3350         struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3351
3352         switch (authsize) {
3353         case ICV_8:
3354                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3355                 aeadctx->mayverify = VERIFY_HW;
3356                 break;
3357         case ICV_12:
3358                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3359                 aeadctx->mayverify = VERIFY_HW;
3360                 break;
3361         case ICV_16:
3362                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3363                 aeadctx->mayverify = VERIFY_HW;
3364                 break;
3365         default:
3366                 return -EINVAL;
3367         }
3368         return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
3369 }
3370
3371 static int chcr_ccm_setauthsize(struct crypto_aead *tfm,
3372                                 unsigned int authsize)
3373 {
3374         struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3375
3376         switch (authsize) {
3377         case ICV_4:
3378                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3379                 aeadctx->mayverify = VERIFY_HW;
3380                 break;
3381         case ICV_6:
3382                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
3383                 aeadctx->mayverify = VERIFY_HW;
3384                 break;
3385         case ICV_8:
3386                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3387                 aeadctx->mayverify = VERIFY_HW;
3388                 break;
3389         case ICV_10:
3390                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
3391                 aeadctx->mayverify = VERIFY_HW;
3392                 break;
3393         case ICV_12:
3394                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3395                 aeadctx->mayverify = VERIFY_HW;
3396                 break;
3397         case ICV_14:
3398                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3399                 aeadctx->mayverify = VERIFY_HW;
3400                 break;
3401         case ICV_16:
3402                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3403                 aeadctx->mayverify = VERIFY_HW;
3404                 break;
3405         default:
3406                 return -EINVAL;
3407         }
3408         return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
3409 }
3410
3411 static int chcr_ccm_common_setkey(struct crypto_aead *aead,
3412                                 const u8 *key,
3413                                 unsigned int keylen)
3414 {
3415         struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3416         unsigned char ck_size, mk_size;
3417         int key_ctx_size = 0;
3418
3419         key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) * 2;
3420         if (keylen == AES_KEYSIZE_128) {
3421                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3422                 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
3423         } else if (keylen == AES_KEYSIZE_192) {
3424                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3425                 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192;
3426         } else if (keylen == AES_KEYSIZE_256) {
3427                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3428                 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
3429         } else {
3430                 aeadctx->enckey_len = 0;
3431                 return  -EINVAL;
3432         }
3433         aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0,
3434                                                 key_ctx_size >> 4);
3435         memcpy(aeadctx->key, key, keylen);
3436         aeadctx->enckey_len = keylen;
3437
3438         return 0;
3439 }
3440
3441 static int chcr_aead_ccm_setkey(struct crypto_aead *aead,
3442                                 const u8 *key,
3443                                 unsigned int keylen)
3444 {
3445         struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3446         int error;
3447
3448         crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3449         crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
3450                               CRYPTO_TFM_REQ_MASK);
3451         error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3452         if (error)
3453                 return error;
3454         return chcr_ccm_common_setkey(aead, key, keylen);
3455 }
3456
3457 static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key,
3458                                     unsigned int keylen)
3459 {
3460         struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3461         int error;
3462
3463         if (keylen < 3) {
3464                 aeadctx->enckey_len = 0;
3465                 return  -EINVAL;
3466         }
3467         crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3468         crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
3469                               CRYPTO_TFM_REQ_MASK);
3470         error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3471         if (error)
3472                 return error;
3473         keylen -= 3;
3474         memcpy(aeadctx->salt, key + keylen, 3);
3475         return chcr_ccm_common_setkey(aead, key, keylen);
3476 }
3477
3478 static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
3479                            unsigned int keylen)
3480 {
3481         struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3482         struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx);
3483         unsigned int ck_size;
3484         int ret = 0, key_ctx_size = 0;
3485         struct crypto_aes_ctx aes;
3486
3487         aeadctx->enckey_len = 0;
3488         crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3489         crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead)
3490                               & CRYPTO_TFM_REQ_MASK);
3491         ret = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3492         if (ret)
3493                 goto out;
3494
3495         if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 &&
3496             keylen > 3) {
3497                 keylen -= 4;  /* nonce/salt is present in the last 4 bytes */
3498                 memcpy(aeadctx->salt, key + keylen, 4);
3499         }
3500         if (keylen == AES_KEYSIZE_128) {
3501                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3502         } else if (keylen == AES_KEYSIZE_192) {
3503                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3504         } else if (keylen == AES_KEYSIZE_256) {
3505                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3506         } else {
3507                 pr_err("GCM: Invalid key length %d\n", keylen);
3508                 ret = -EINVAL;
3509                 goto out;
3510         }
3511
3512         memcpy(aeadctx->key, key, keylen);
3513         aeadctx->enckey_len = keylen;
3514         key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) +
3515                 AEAD_H_SIZE;
3516         aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
3517                                                 CHCR_KEYCTX_MAC_KEY_SIZE_128,
3518                                                 0, 0,
3519                                                 key_ctx_size >> 4);
3520         /* Calculate the H = CIPH(K, 0 repeated 16 times).
3521          * It will go in key context
3522          */
3523         ret = aes_expandkey(&aes, key, keylen);
3524         if (ret) {
3525                 aeadctx->enckey_len = 0;
3526                 goto out;
3527         }
3528         memset(gctx->ghash_h, 0, AEAD_H_SIZE);
3529         aes_encrypt(&aes, gctx->ghash_h, gctx->ghash_h);
3530         memzero_explicit(&aes, sizeof(aes));
3531
3532 out:
3533         return ret;
3534 }
3535
3536 static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
3537                                    unsigned int keylen)
3538 {
3539         struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc));
3540         struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
3541         /* it contains auth and cipher key both*/
3542         struct crypto_authenc_keys keys;
3543         unsigned int bs, subtype;
3544         unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize;
3545         int err = 0, i, key_ctx_len = 0;
3546         unsigned char ck_size = 0;
3547         unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 };
3548         struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
3549         struct algo_param param;
3550         int align;
3551         u8 *o_ptr = NULL;
3552
3553         crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3554         crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
3555                               & CRYPTO_TFM_REQ_MASK);
3556         err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3557         if (err)
3558                 goto out;
3559
3560         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
3561                 goto out;
3562
3563         if (get_alg_config(&param, max_authsize)) {
3564                 pr_err("Unsupported digest size\n");
3565                 goto out;
3566         }
3567         subtype = get_aead_subtype(authenc);
3568         if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
3569                 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
3570                 if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE)
3571                         goto out;
3572                 memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen
3573                 - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE);
3574                 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE;
3575         }
3576         if (keys.enckeylen == AES_KEYSIZE_128) {
3577                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3578         } else if (keys.enckeylen == AES_KEYSIZE_192) {
3579                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3580         } else if (keys.enckeylen == AES_KEYSIZE_256) {
3581                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3582         } else {
3583                 pr_err("Unsupported cipher key\n");
3584                 goto out;
3585         }
3586
3587         /* Copy only encryption key. We use authkey to generate h(ipad) and
3588          * h(opad) so authkey is not needed again. authkeylen size have the
3589          * size of the hash digest size.
3590          */
3591         memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
3592         aeadctx->enckey_len = keys.enckeylen;
3593         if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA ||
3594                 subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) {
3595
3596                 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
3597                             aeadctx->enckey_len << 3);
3598         }
3599         base_hash  = chcr_alloc_shash(max_authsize);
3600         if (IS_ERR(base_hash)) {
3601                 pr_err("Base driver cannot be loaded\n");
3602                 goto out;
3603         }
3604         {
3605                 SHASH_DESC_ON_STACK(shash, base_hash);
3606
3607                 shash->tfm = base_hash;
3608                 bs = crypto_shash_blocksize(base_hash);
3609                 align = KEYCTX_ALIGN_PAD(max_authsize);
3610                 o_ptr =  actx->h_iopad + param.result_size + align;
3611
3612                 if (keys.authkeylen > bs) {
3613                         err = crypto_shash_digest(shash, keys.authkey,
3614                                                   keys.authkeylen,
3615                                                   o_ptr);
3616                         if (err) {
3617                                 pr_err("Base driver cannot be loaded\n");
3618                                 goto out;
3619                         }
3620                         keys.authkeylen = max_authsize;
3621                 } else
3622                         memcpy(o_ptr, keys.authkey, keys.authkeylen);
3623
3624                 /* Compute the ipad-digest*/
3625                 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
3626                 memcpy(pad, o_ptr, keys.authkeylen);
3627                 for (i = 0; i < bs >> 2; i++)
3628                         *((unsigned int *)pad + i) ^= IPAD_DATA;
3629
3630                 if (chcr_compute_partial_hash(shash, pad, actx->h_iopad,
3631                                               max_authsize))
3632                         goto out;
3633                 /* Compute the opad-digest */
3634                 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
3635                 memcpy(pad, o_ptr, keys.authkeylen);
3636                 for (i = 0; i < bs >> 2; i++)
3637                         *((unsigned int *)pad + i) ^= OPAD_DATA;
3638
3639                 if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize))
3640                         goto out;
3641
3642                 /* convert the ipad and opad digest to network order */
3643                 chcr_change_order(actx->h_iopad, param.result_size);
3644                 chcr_change_order(o_ptr, param.result_size);
3645                 key_ctx_len = sizeof(struct _key_ctx) +
3646                         roundup(keys.enckeylen, 16) +
3647                         (param.result_size + align) * 2;
3648                 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size,
3649                                                 0, 1, key_ctx_len >> 4);
3650                 actx->auth_mode = param.auth_mode;
3651                 chcr_free_shash(base_hash);
3652
3653                 memzero_explicit(&keys, sizeof(keys));
3654                 return 0;
3655         }
3656 out:
3657         aeadctx->enckey_len = 0;
3658         memzero_explicit(&keys, sizeof(keys));
3659         if (!IS_ERR(base_hash))
3660                 chcr_free_shash(base_hash);
3661         return -EINVAL;
3662 }
3663
3664 static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
3665                                         const u8 *key, unsigned int keylen)
3666 {
3667         struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc));
3668         struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
3669         struct crypto_authenc_keys keys;
3670         int err;
3671         /* it contains auth and cipher key both*/
3672         unsigned int subtype;
3673         int key_ctx_len = 0;
3674         unsigned char ck_size = 0;
3675
3676         crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3677         crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
3678                               & CRYPTO_TFM_REQ_MASK);
3679         err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3680         if (err)
3681                 goto out;
3682
3683         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
3684                 goto out;
3685
3686         subtype = get_aead_subtype(authenc);
3687         if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
3688             subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
3689                 if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE)
3690                         goto out;
3691                 memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen
3692                         - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE);
3693                 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE;
3694         }
3695         if (keys.enckeylen == AES_KEYSIZE_128) {
3696                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3697         } else if (keys.enckeylen == AES_KEYSIZE_192) {
3698                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3699         } else if (keys.enckeylen == AES_KEYSIZE_256) {
3700                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3701         } else {
3702                 pr_err("Unsupported cipher key %d\n", keys.enckeylen);
3703                 goto out;
3704         }
3705         memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
3706         aeadctx->enckey_len = keys.enckeylen;
3707         if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA ||
3708             subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) {
3709                 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
3710                                 aeadctx->enckey_len << 3);
3711         }
3712         key_ctx_len =  sizeof(struct _key_ctx) + roundup(keys.enckeylen, 16);
3713
3714         aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0,
3715                                                 0, key_ctx_len >> 4);
3716         actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP;
3717         memzero_explicit(&keys, sizeof(keys));
3718         return 0;
3719 out:
3720         aeadctx->enckey_len = 0;
3721         memzero_explicit(&keys, sizeof(keys));
3722         return -EINVAL;
3723 }
3724
3725 static int chcr_aead_op(struct aead_request *req,
3726                         int size,
3727                         create_wr_t create_wr_fn)
3728 {
3729         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3730         struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
3731         struct chcr_context *ctx = a_ctx(tfm);
3732         struct uld_ctx *u_ctx = ULD_CTX(ctx);
3733         struct sk_buff *skb;
3734         struct chcr_dev *cdev;
3735
3736         cdev = a_ctx(tfm)->dev;
3737         if (!cdev) {
3738                 pr_err("%s : No crypto device.\n", __func__);
3739                 return -ENXIO;
3740         }
3741
3742         if (chcr_inc_wrcount(cdev)) {
3743         /* Detach state for CHCR means lldi or padap is freed.
3744          * We cannot increment fallback here.
3745          */
3746                 return chcr_aead_fallback(req, reqctx->op);
3747         }
3748
3749         if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
3750                                         reqctx->txqidx) &&
3751                 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))) {
3752                         chcr_dec_wrcount(cdev);
3753                         return -ENOSPC;
3754         }
3755
3756         if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 &&
3757             crypto_ipsec_check_assoclen(req->assoclen) != 0) {
3758                 pr_err("RFC4106: Invalid value of assoclen %d\n",
3759                        req->assoclen);
3760                 return -EINVAL;
3761         }
3762
3763         /* Form a WR from req */
3764         skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx], size);
3765
3766         if (IS_ERR_OR_NULL(skb)) {
3767                 chcr_dec_wrcount(cdev);
3768                 return PTR_ERR_OR_ZERO(skb);
3769         }
3770
3771         skb->dev = u_ctx->lldi.ports[0];
3772         set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
3773         chcr_send_wr(skb);
3774         return -EINPROGRESS;
3775 }
3776
3777 static int chcr_aead_encrypt(struct aead_request *req)
3778 {
3779         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3780         struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3781         struct chcr_context *ctx = a_ctx(tfm);
3782         unsigned int cpu;
3783
3784         cpu = get_cpu();
3785         reqctx->txqidx = cpu % ctx->ntxq;
3786         reqctx->rxqidx = cpu % ctx->nrxq;
3787         put_cpu();
3788
3789         reqctx->verify = VERIFY_HW;
3790         reqctx->op = CHCR_ENCRYPT_OP;
3791
3792         switch (get_aead_subtype(tfm)) {
3793         case CRYPTO_ALG_SUB_TYPE_CTR_SHA:
3794         case CRYPTO_ALG_SUB_TYPE_CBC_SHA:
3795         case CRYPTO_ALG_SUB_TYPE_CBC_NULL:
3796         case CRYPTO_ALG_SUB_TYPE_CTR_NULL:
3797                 return chcr_aead_op(req, 0, create_authenc_wr);
3798         case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3799         case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
3800                 return chcr_aead_op(req, 0, create_aead_ccm_wr);
3801         default:
3802                 return chcr_aead_op(req, 0, create_gcm_wr);
3803         }
3804 }
3805
3806 static int chcr_aead_decrypt(struct aead_request *req)
3807 {
3808         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3809         struct chcr_context *ctx = a_ctx(tfm);
3810         struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
3811         struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3812         int size;
3813         unsigned int cpu;
3814
3815         cpu = get_cpu();
3816         reqctx->txqidx = cpu % ctx->ntxq;
3817         reqctx->rxqidx = cpu % ctx->nrxq;
3818         put_cpu();
3819
3820         if (aeadctx->mayverify == VERIFY_SW) {
3821                 size = crypto_aead_maxauthsize(tfm);
3822                 reqctx->verify = VERIFY_SW;
3823         } else {
3824                 size = 0;
3825                 reqctx->verify = VERIFY_HW;
3826         }
3827         reqctx->op = CHCR_DECRYPT_OP;
3828         switch (get_aead_subtype(tfm)) {
3829         case CRYPTO_ALG_SUB_TYPE_CBC_SHA:
3830         case CRYPTO_ALG_SUB_TYPE_CTR_SHA:
3831         case CRYPTO_ALG_SUB_TYPE_CBC_NULL:
3832         case CRYPTO_ALG_SUB_TYPE_CTR_NULL:
3833                 return chcr_aead_op(req, size, create_authenc_wr);
3834         case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3835         case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
3836                 return chcr_aead_op(req, size, create_aead_ccm_wr);
3837         default:
3838                 return chcr_aead_op(req, size, create_gcm_wr);
3839         }
3840 }
3841
3842 static struct chcr_alg_template driver_algs[] = {
3843         /* AES-CBC */
3844         {
3845                 .type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC,
3846                 .is_registered = 0,
3847                 .alg.skcipher = {
3848                         .base.cra_name          = "cbc(aes)",
3849                         .base.cra_driver_name   = "cbc-aes-chcr",
3850                         .base.cra_blocksize     = AES_BLOCK_SIZE,
3851
3852                         .init                   = chcr_init_tfm,
3853                         .exit                   = chcr_exit_tfm,
3854                         .min_keysize            = AES_MIN_KEY_SIZE,
3855                         .max_keysize            = AES_MAX_KEY_SIZE,
3856                         .ivsize                 = AES_BLOCK_SIZE,
3857                         .setkey                 = chcr_aes_cbc_setkey,
3858                         .encrypt                = chcr_aes_encrypt,
3859                         .decrypt                = chcr_aes_decrypt,
3860                         }
3861         },
3862         {
3863                 .type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS,
3864                 .is_registered = 0,
3865                 .alg.skcipher = {
3866                         .base.cra_name          = "xts(aes)",
3867                         .base.cra_driver_name   = "xts-aes-chcr",
3868                         .base.cra_blocksize     = AES_BLOCK_SIZE,
3869
3870                         .init                   = chcr_init_tfm,
3871                         .exit                   = chcr_exit_tfm,
3872                         .min_keysize            = 2 * AES_MIN_KEY_SIZE,
3873                         .max_keysize            = 2 * AES_MAX_KEY_SIZE,
3874                         .ivsize                 = AES_BLOCK_SIZE,
3875                         .setkey                 = chcr_aes_xts_setkey,
3876                         .encrypt                = chcr_aes_encrypt,
3877                         .decrypt                = chcr_aes_decrypt,
3878                         }
3879         },
3880         {
3881                 .type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR,
3882                 .is_registered = 0,
3883                 .alg.skcipher = {
3884                         .base.cra_name          = "ctr(aes)",
3885                         .base.cra_driver_name   = "ctr-aes-chcr",
3886                         .base.cra_blocksize     = 1,
3887
3888                         .init                   = chcr_init_tfm,
3889                         .exit                   = chcr_exit_tfm,
3890                         .min_keysize            = AES_MIN_KEY_SIZE,
3891                         .max_keysize            = AES_MAX_KEY_SIZE,
3892                         .ivsize                 = AES_BLOCK_SIZE,
3893                         .setkey                 = chcr_aes_ctr_setkey,
3894                         .encrypt                = chcr_aes_encrypt,
3895                         .decrypt                = chcr_aes_decrypt,
3896                 }
3897         },
3898         {
3899                 .type = CRYPTO_ALG_TYPE_SKCIPHER |
3900                         CRYPTO_ALG_SUB_TYPE_CTR_RFC3686,
3901                 .is_registered = 0,
3902                 .alg.skcipher = {
3903                         .base.cra_name          = "rfc3686(ctr(aes))",
3904                         .base.cra_driver_name   = "rfc3686-ctr-aes-chcr",
3905                         .base.cra_blocksize     = 1,
3906
3907                         .init                   = chcr_rfc3686_init,
3908                         .exit                   = chcr_exit_tfm,
3909                         .min_keysize            = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
3910                         .max_keysize            = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
3911                         .ivsize                 = CTR_RFC3686_IV_SIZE,
3912                         .setkey                 = chcr_aes_rfc3686_setkey,
3913                         .encrypt                = chcr_aes_encrypt,
3914                         .decrypt                = chcr_aes_decrypt,
3915                 }
3916         },
3917         /* SHA */
3918         {
3919                 .type = CRYPTO_ALG_TYPE_AHASH,
3920                 .is_registered = 0,
3921                 .alg.hash = {
3922                         .halg.digestsize = SHA1_DIGEST_SIZE,
3923                         .halg.base = {
3924                                 .cra_name = "sha1",
3925                                 .cra_driver_name = "sha1-chcr",
3926                                 .cra_blocksize = SHA1_BLOCK_SIZE,
3927                         }
3928                 }
3929         },
3930         {
3931                 .type = CRYPTO_ALG_TYPE_AHASH,
3932                 .is_registered = 0,
3933                 .alg.hash = {
3934                         .halg.digestsize = SHA256_DIGEST_SIZE,
3935                         .halg.base = {
3936                                 .cra_name = "sha256",
3937                                 .cra_driver_name = "sha256-chcr",
3938                                 .cra_blocksize = SHA256_BLOCK_SIZE,
3939                         }
3940                 }
3941         },
3942         {
3943                 .type = CRYPTO_ALG_TYPE_AHASH,
3944                 .is_registered = 0,
3945                 .alg.hash = {
3946                         .halg.digestsize = SHA224_DIGEST_SIZE,
3947                         .halg.base = {
3948                                 .cra_name = "sha224",
3949                                 .cra_driver_name = "sha224-chcr",
3950                                 .cra_blocksize = SHA224_BLOCK_SIZE,
3951                         }
3952                 }
3953         },
3954         {
3955                 .type = CRYPTO_ALG_TYPE_AHASH,
3956                 .is_registered = 0,
3957                 .alg.hash = {
3958                         .halg.digestsize = SHA384_DIGEST_SIZE,
3959                         .halg.base = {
3960                                 .cra_name = "sha384",
3961                                 .cra_driver_name = "sha384-chcr",
3962                                 .cra_blocksize = SHA384_BLOCK_SIZE,
3963                         }
3964                 }
3965         },
3966         {
3967                 .type = CRYPTO_ALG_TYPE_AHASH,
3968                 .is_registered = 0,
3969                 .alg.hash = {
3970                         .halg.digestsize = SHA512_DIGEST_SIZE,
3971                         .halg.base = {
3972                                 .cra_name = "sha512",
3973                                 .cra_driver_name = "sha512-chcr",
3974                                 .cra_blocksize = SHA512_BLOCK_SIZE,
3975                         }
3976                 }
3977         },
3978         /* HMAC */
3979         {
3980                 .type = CRYPTO_ALG_TYPE_HMAC,
3981                 .is_registered = 0,
3982                 .alg.hash = {
3983                         .halg.digestsize = SHA1_DIGEST_SIZE,
3984                         .halg.base = {
3985                                 .cra_name = "hmac(sha1)",
3986                                 .cra_driver_name = "hmac-sha1-chcr",
3987                                 .cra_blocksize = SHA1_BLOCK_SIZE,
3988                         }
3989                 }
3990         },
3991         {
3992                 .type = CRYPTO_ALG_TYPE_HMAC,
3993                 .is_registered = 0,
3994                 .alg.hash = {
3995                         .halg.digestsize = SHA224_DIGEST_SIZE,
3996                         .halg.base = {
3997                                 .cra_name = "hmac(sha224)",
3998                                 .cra_driver_name = "hmac-sha224-chcr",
3999                                 .cra_blocksize = SHA224_BLOCK_SIZE,
4000                         }
4001                 }
4002         },
4003         {
4004                 .type = CRYPTO_ALG_TYPE_HMAC,
4005                 .is_registered = 0,
4006                 .alg.hash = {
4007                         .halg.digestsize = SHA256_DIGEST_SIZE,
4008                         .halg.base = {
4009                                 .cra_name = "hmac(sha256)",
4010                                 .cra_driver_name = "hmac-sha256-chcr",
4011                                 .cra_blocksize = SHA256_BLOCK_SIZE,
4012                         }
4013                 }
4014         },
4015         {
4016                 .type = CRYPTO_ALG_TYPE_HMAC,
4017                 .is_registered = 0,
4018                 .alg.hash = {
4019                         .halg.digestsize = SHA384_DIGEST_SIZE,
4020                         .halg.base = {
4021                                 .cra_name = "hmac(sha384)",
4022                                 .cra_driver_name = "hmac-sha384-chcr",
4023                                 .cra_blocksize = SHA384_BLOCK_SIZE,
4024                         }
4025                 }
4026         },
4027         {
4028                 .type = CRYPTO_ALG_TYPE_HMAC,
4029                 .is_registered = 0,
4030                 .alg.hash = {
4031                         .halg.digestsize = SHA512_DIGEST_SIZE,
4032                         .halg.base = {
4033                                 .cra_name = "hmac(sha512)",
4034                                 .cra_driver_name = "hmac-sha512-chcr",
4035                                 .cra_blocksize = SHA512_BLOCK_SIZE,
4036                         }
4037                 }
4038         },
4039         /* Add AEAD Algorithms */
4040         {
4041                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM,
4042                 .is_registered = 0,
4043                 .alg.aead = {
4044                         .base = {
4045                                 .cra_name = "gcm(aes)",
4046                                 .cra_driver_name = "gcm-aes-chcr",
4047                                 .cra_blocksize  = 1,
4048                                 .cra_priority = CHCR_AEAD_PRIORITY,
4049                                 .cra_ctxsize =  sizeof(struct chcr_context) +
4050                                                 sizeof(struct chcr_aead_ctx) +
4051                                                 sizeof(struct chcr_gcm_ctx),
4052                         },
4053                         .ivsize = GCM_AES_IV_SIZE,
4054                         .maxauthsize = GHASH_DIGEST_SIZE,
4055                         .setkey = chcr_gcm_setkey,
4056                         .setauthsize = chcr_gcm_setauthsize,
4057                 }
4058         },
4059         {
4060                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106,
4061                 .is_registered = 0,
4062                 .alg.aead = {
4063                         .base = {
4064                                 .cra_name = "rfc4106(gcm(aes))",
4065                                 .cra_driver_name = "rfc4106-gcm-aes-chcr",
4066                                 .cra_blocksize   = 1,
4067                                 .cra_priority = CHCR_AEAD_PRIORITY + 1,
4068                                 .cra_ctxsize =  sizeof(struct chcr_context) +
4069                                                 sizeof(struct chcr_aead_ctx) +
4070                                                 sizeof(struct chcr_gcm_ctx),
4071
4072                         },
4073                         .ivsize = GCM_RFC4106_IV_SIZE,
4074                         .maxauthsize    = GHASH_DIGEST_SIZE,
4075                         .setkey = chcr_gcm_setkey,
4076                         .setauthsize    = chcr_4106_4309_setauthsize,
4077                 }
4078         },
4079         {
4080                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM,
4081                 .is_registered = 0,
4082                 .alg.aead = {
4083                         .base = {
4084                                 .cra_name = "ccm(aes)",
4085                                 .cra_driver_name = "ccm-aes-chcr",
4086                                 .cra_blocksize   = 1,
4087                                 .cra_priority = CHCR_AEAD_PRIORITY,
4088                                 .cra_ctxsize =  sizeof(struct chcr_context) +
4089                                                 sizeof(struct chcr_aead_ctx),
4090
4091                         },
4092                         .ivsize = AES_BLOCK_SIZE,
4093                         .maxauthsize    = GHASH_DIGEST_SIZE,
4094                         .setkey = chcr_aead_ccm_setkey,
4095                         .setauthsize    = chcr_ccm_setauthsize,
4096                 }
4097         },
4098         {
4099                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309,
4100                 .is_registered = 0,
4101                 .alg.aead = {
4102                         .base = {
4103                                 .cra_name = "rfc4309(ccm(aes))",
4104                                 .cra_driver_name = "rfc4309-ccm-aes-chcr",
4105                                 .cra_blocksize   = 1,
4106                                 .cra_priority = CHCR_AEAD_PRIORITY + 1,
4107                                 .cra_ctxsize =  sizeof(struct chcr_context) +
4108                                                 sizeof(struct chcr_aead_ctx),
4109
4110                         },
4111                         .ivsize = 8,
4112                         .maxauthsize    = GHASH_DIGEST_SIZE,
4113                         .setkey = chcr_aead_rfc4309_setkey,
4114                         .setauthsize = chcr_4106_4309_setauthsize,
4115                 }
4116         },
4117         {
4118                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
4119                 .is_registered = 0,
4120                 .alg.aead = {
4121                         .base = {
4122                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
4123                                 .cra_driver_name =
4124                                         "authenc-hmac-sha1-cbc-aes-chcr",
4125                                 .cra_blocksize   = AES_BLOCK_SIZE,
4126                                 .cra_priority = CHCR_AEAD_PRIORITY,
4127                                 .cra_ctxsize =  sizeof(struct chcr_context) +
4128                                                 sizeof(struct chcr_aead_ctx) +
4129                                                 sizeof(struct chcr_authenc_ctx),
4130
4131                         },
4132                         .ivsize = AES_BLOCK_SIZE,
4133                         .maxauthsize = SHA1_DIGEST_SIZE,
4134                         .setkey = chcr_authenc_setkey,
4135                         .setauthsize = chcr_authenc_setauthsize,
4136                 }
4137         },
4138         {
4139                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
4140                 .is_registered = 0,
4141                 .alg.aead = {
4142                         .base = {
4143
4144                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
4145                                 .cra_driver_name =
4146                                         "authenc-hmac-sha256-cbc-aes-chcr",
4147                                 .cra_blocksize   = AES_BLOCK_SIZE,
4148                                 .cra_priority = CHCR_AEAD_PRIORITY,
4149                                 .cra_ctxsize =  sizeof(struct chcr_context) +
4150                                                 sizeof(struct chcr_aead_ctx) +
4151                                                 sizeof(struct chcr_authenc_ctx),
4152
4153                         },
4154                         .ivsize = AES_BLOCK_SIZE,
4155                         .maxauthsize    = SHA256_DIGEST_SIZE,
4156                         .setkey = chcr_authenc_setkey,
4157                         .setauthsize = chcr_authenc_setauthsize,
4158                 }
4159         },
4160         {
4161                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
4162                 .is_registered = 0,
4163                 .alg.aead = {
4164                         .base = {
4165                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
4166                                 .cra_driver_name =
4167                                         "authenc-hmac-sha224-cbc-aes-chcr",
4168                                 .cra_blocksize   = AES_BLOCK_SIZE,
4169                                 .cra_priority = CHCR_AEAD_PRIORITY,
4170                                 .cra_ctxsize =  sizeof(struct chcr_context) +
4171                                                 sizeof(struct chcr_aead_ctx) +
4172                                                 sizeof(struct chcr_authenc_ctx),
4173                         },
4174                         .ivsize = AES_BLOCK_SIZE,
4175                         .maxauthsize = SHA224_DIGEST_SIZE,
4176                         .setkey = chcr_authenc_setkey,
4177                         .setauthsize = chcr_authenc_setauthsize,
4178                 }
4179         },
4180         {
4181                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
4182                 .is_registered = 0,
4183                 .alg.aead = {
4184                         .base = {
4185                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
4186                                 .cra_driver_name =
4187                                         "authenc-hmac-sha384-cbc-aes-chcr",
4188                                 .cra_blocksize   = AES_BLOCK_SIZE,
4189                                 .cra_priority = CHCR_AEAD_PRIORITY,
4190                                 .cra_ctxsize =  sizeof(struct chcr_context) +
4191                                                 sizeof(struct chcr_aead_ctx) +
4192                                                 sizeof(struct chcr_authenc_ctx),
4193
4194                         },
4195                         .ivsize = AES_BLOCK_SIZE,
4196                         .maxauthsize = SHA384_DIGEST_SIZE,
4197                         .setkey = chcr_authenc_setkey,
4198                         .setauthsize = chcr_authenc_setauthsize,
4199                 }
4200         },
4201         {
4202                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
4203                 .is_registered = 0,
4204                 .alg.aead = {
4205                         .base = {
4206                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
4207                                 .cra_driver_name =
4208                                         "authenc-hmac-sha512-cbc-aes-chcr",
4209                                 .cra_blocksize   = AES_BLOCK_SIZE,
4210                                 .cra_priority = CHCR_AEAD_PRIORITY,
4211                                 .cra_ctxsize =  sizeof(struct chcr_context) +
4212                                                 sizeof(struct chcr_aead_ctx) +
4213                                                 sizeof(struct chcr_authenc_ctx),
4214
4215                         },
4216                         .ivsize = AES_BLOCK_SIZE,
4217                         .maxauthsize = SHA512_DIGEST_SIZE,
4218                         .setkey = chcr_authenc_setkey,
4219                         .setauthsize = chcr_authenc_setauthsize,
4220                 }
4221         },
4222         {
4223                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_NULL,
4224                 .is_registered = 0,
4225                 .alg.aead = {
4226                         .base = {
4227                                 .cra_name = "authenc(digest_null,cbc(aes))",
4228                                 .cra_driver_name =
4229                                         "authenc-digest_null-cbc-aes-chcr",
4230                                 .cra_blocksize   = AES_BLOCK_SIZE,
4231                                 .cra_priority = CHCR_AEAD_PRIORITY,
4232                                 .cra_ctxsize =  sizeof(struct chcr_context) +
4233                                                 sizeof(struct chcr_aead_ctx) +
4234                                                 sizeof(struct chcr_authenc_ctx),
4235
4236                         },
4237                         .ivsize  = AES_BLOCK_SIZE,
4238                         .maxauthsize = 0,
4239                         .setkey  = chcr_aead_digest_null_setkey,
4240                         .setauthsize = chcr_authenc_null_setauthsize,
4241                 }
4242         },
4243         {
4244                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4245                 .is_registered = 0,
4246                 .alg.aead = {
4247                         .base = {
4248                                 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
4249                                 .cra_driver_name =
4250                                 "authenc-hmac-sha1-rfc3686-ctr-aes-chcr",
4251                                 .cra_blocksize   = 1,
4252                                 .cra_priority = CHCR_AEAD_PRIORITY,
4253                                 .cra_ctxsize =  sizeof(struct chcr_context) +
4254                                                 sizeof(struct chcr_aead_ctx) +
4255                                                 sizeof(struct chcr_authenc_ctx),
4256
4257                         },
4258                         .ivsize = CTR_RFC3686_IV_SIZE,
4259                         .maxauthsize = SHA1_DIGEST_SIZE,
4260                         .setkey = chcr_authenc_setkey,
4261                         .setauthsize = chcr_authenc_setauthsize,
4262                 }
4263         },
4264         {
4265                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4266                 .is_registered = 0,
4267                 .alg.aead = {
4268                         .base = {
4269
4270                                 .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
4271                                 .cra_driver_name =
4272                                 "authenc-hmac-sha256-rfc3686-ctr-aes-chcr",
4273                                 .cra_blocksize   = 1,
4274                                 .cra_priority = CHCR_AEAD_PRIORITY,
4275                                 .cra_ctxsize =  sizeof(struct chcr_context) +
4276                                                 sizeof(struct chcr_aead_ctx) +
4277                                                 sizeof(struct chcr_authenc_ctx),
4278
4279                         },
4280                         .ivsize = CTR_RFC3686_IV_SIZE,
4281                         .maxauthsize    = SHA256_DIGEST_SIZE,
4282                         .setkey = chcr_authenc_setkey,
4283                         .setauthsize = chcr_authenc_setauthsize,
4284                 }
4285         },
4286         {
4287                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4288                 .is_registered = 0,
4289                 .alg.aead = {
4290                         .base = {
4291                                 .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))",
4292                                 .cra_driver_name =
4293                                 "authenc-hmac-sha224-rfc3686-ctr-aes-chcr",
4294                                 .cra_blocksize   = 1,
4295                                 .cra_priority = CHCR_AEAD_PRIORITY,
4296                                 .cra_ctxsize =  sizeof(struct chcr_context) +
4297                                                 sizeof(struct chcr_aead_ctx) +
4298                                                 sizeof(struct chcr_authenc_ctx),
4299                         },
4300                         .ivsize = CTR_RFC3686_IV_SIZE,
4301                         .maxauthsize = SHA224_DIGEST_SIZE,
4302                         .setkey = chcr_authenc_setkey,
4303                         .setauthsize = chcr_authenc_setauthsize,
4304                 }
4305         },
4306         {
4307                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4308                 .is_registered = 0,
4309                 .alg.aead = {
4310                         .base = {
4311                                 .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
4312                                 .cra_driver_name =
4313                                 "authenc-hmac-sha384-rfc3686-ctr-aes-chcr",
4314                                 .cra_blocksize   = 1,
4315                                 .cra_priority = CHCR_AEAD_PRIORITY,
4316                                 .cra_ctxsize =  sizeof(struct chcr_context) +
4317                                                 sizeof(struct chcr_aead_ctx) +
4318                                                 sizeof(struct chcr_authenc_ctx),
4319
4320                         },
4321                         .ivsize = CTR_RFC3686_IV_SIZE,
4322                         .maxauthsize = SHA384_DIGEST_SIZE,
4323                         .setkey = chcr_authenc_setkey,
4324                         .setauthsize = chcr_authenc_setauthsize,
4325                 }
4326         },
4327         {
4328                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4329                 .is_registered = 0,
4330                 .alg.aead = {
4331                         .base = {
4332                                 .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
4333                                 .cra_driver_name =
4334                                 "authenc-hmac-sha512-rfc3686-ctr-aes-chcr",
4335                                 .cra_blocksize   = 1,
4336                                 .cra_priority = CHCR_AEAD_PRIORITY,
4337                                 .cra_ctxsize =  sizeof(struct chcr_context) +
4338                                                 sizeof(struct chcr_aead_ctx) +
4339                                                 sizeof(struct chcr_authenc_ctx),
4340
4341                         },
4342                         .ivsize = CTR_RFC3686_IV_SIZE,
4343                         .maxauthsize = SHA512_DIGEST_SIZE,
4344                         .setkey = chcr_authenc_setkey,
4345                         .setauthsize = chcr_authenc_setauthsize,
4346                 }
4347         },
4348         {
4349                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_NULL,
4350                 .is_registered = 0,
4351                 .alg.aead = {
4352                         .base = {
4353                                 .cra_name = "authenc(digest_null,rfc3686(ctr(aes)))",
4354                                 .cra_driver_name =
4355                                 "authenc-digest_null-rfc3686-ctr-aes-chcr",
4356                                 .cra_blocksize   = 1,
4357                                 .cra_priority = CHCR_AEAD_PRIORITY,
4358                                 .cra_ctxsize =  sizeof(struct chcr_context) +
4359                                                 sizeof(struct chcr_aead_ctx) +
4360                                                 sizeof(struct chcr_authenc_ctx),
4361
4362                         },
4363                         .ivsize  = CTR_RFC3686_IV_SIZE,
4364                         .maxauthsize = 0,
4365                         .setkey  = chcr_aead_digest_null_setkey,
4366                         .setauthsize = chcr_authenc_null_setauthsize,
4367                 }
4368         },
4369 };
4370
4371 /*
4372  *      chcr_unregister_alg - Deregister crypto algorithms with
4373  *      kernel framework.
4374  */
4375 static int chcr_unregister_alg(void)
4376 {
4377         int i;
4378
4379         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4380                 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
4381                 case CRYPTO_ALG_TYPE_SKCIPHER:
4382                         if (driver_algs[i].is_registered && refcount_read(
4383                             &driver_algs[i].alg.skcipher.base.cra_refcnt)
4384                             == 1) {
4385                                 crypto_unregister_skcipher(
4386                                                 &driver_algs[i].alg.skcipher);
4387                                 driver_algs[i].is_registered = 0;
4388                         }
4389                         break;
4390                 case CRYPTO_ALG_TYPE_AEAD:
4391                         if (driver_algs[i].is_registered && refcount_read(
4392                             &driver_algs[i].alg.aead.base.cra_refcnt) == 1) {
4393                                 crypto_unregister_aead(
4394                                                 &driver_algs[i].alg.aead);
4395                                 driver_algs[i].is_registered = 0;
4396                         }
4397                         break;
4398                 case CRYPTO_ALG_TYPE_AHASH:
4399                         if (driver_algs[i].is_registered && refcount_read(
4400                             &driver_algs[i].alg.hash.halg.base.cra_refcnt)
4401                             == 1) {
4402                                 crypto_unregister_ahash(
4403                                                 &driver_algs[i].alg.hash);
4404                                 driver_algs[i].is_registered = 0;
4405                         }
4406                         break;
4407                 }
4408         }
4409         return 0;
4410 }
4411
4412 #define SZ_AHASH_CTX sizeof(struct chcr_context)
4413 #define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx))
4414 #define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx)
4415
4416 /*
4417  *      chcr_register_alg - Register crypto algorithms with kernel framework.
4418  */
4419 static int chcr_register_alg(void)
4420 {
4421         struct crypto_alg ai;
4422         struct ahash_alg *a_hash;
4423         int err = 0, i;
4424         char *name = NULL;
4425
4426         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4427                 if (driver_algs[i].is_registered)
4428                         continue;
4429                 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
4430                 case CRYPTO_ALG_TYPE_SKCIPHER:
4431                         driver_algs[i].alg.skcipher.base.cra_priority =
4432                                 CHCR_CRA_PRIORITY;
4433                         driver_algs[i].alg.skcipher.base.cra_module = THIS_MODULE;
4434                         driver_algs[i].alg.skcipher.base.cra_flags =
4435                                 CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
4436                                 CRYPTO_ALG_ALLOCATES_MEMORY |
4437                                 CRYPTO_ALG_NEED_FALLBACK;
4438                         driver_algs[i].alg.skcipher.base.cra_ctxsize =
4439                                 sizeof(struct chcr_context) +
4440                                 sizeof(struct ablk_ctx);
4441                         driver_algs[i].alg.skcipher.base.cra_alignmask = 0;
4442
4443                         err = crypto_register_skcipher(&driver_algs[i].alg.skcipher);
4444                         name = driver_algs[i].alg.skcipher.base.cra_driver_name;
4445                         break;
4446                 case CRYPTO_ALG_TYPE_AEAD:
4447                         driver_algs[i].alg.aead.base.cra_flags =
4448                                 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK |
4449                                 CRYPTO_ALG_ALLOCATES_MEMORY;
4450                         driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt;
4451                         driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt;
4452                         driver_algs[i].alg.aead.init = chcr_aead_cra_init;
4453                         driver_algs[i].alg.aead.exit = chcr_aead_cra_exit;
4454                         driver_algs[i].alg.aead.base.cra_module = THIS_MODULE;
4455                         err = crypto_register_aead(&driver_algs[i].alg.aead);
4456                         name = driver_algs[i].alg.aead.base.cra_driver_name;
4457                         break;
4458                 case CRYPTO_ALG_TYPE_AHASH:
4459                         a_hash = &driver_algs[i].alg.hash;
4460                         a_hash->update = chcr_ahash_update;
4461                         a_hash->final = chcr_ahash_final;
4462                         a_hash->finup = chcr_ahash_finup;
4463                         a_hash->digest = chcr_ahash_digest;
4464                         a_hash->export = chcr_ahash_export;
4465                         a_hash->import = chcr_ahash_import;
4466                         a_hash->halg.statesize = SZ_AHASH_REQ_CTX;
4467                         a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY;
4468                         a_hash->halg.base.cra_module = THIS_MODULE;
4469                         a_hash->halg.base.cra_flags =
4470                                 CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY;
4471                         a_hash->halg.base.cra_alignmask = 0;
4472                         a_hash->halg.base.cra_exit = NULL;
4473
4474                         if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) {
4475                                 a_hash->halg.base.cra_init = chcr_hmac_cra_init;
4476                                 a_hash->halg.base.cra_exit = chcr_hmac_cra_exit;
4477                                 a_hash->init = chcr_hmac_init;
4478                                 a_hash->setkey = chcr_ahash_setkey;
4479                                 a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX;
4480                         } else {
4481                                 a_hash->init = chcr_sha_init;
4482                                 a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX;
4483                                 a_hash->halg.base.cra_init = chcr_sha_cra_init;
4484                         }
4485                         err = crypto_register_ahash(&driver_algs[i].alg.hash);
4486                         ai = driver_algs[i].alg.hash.halg.base;
4487                         name = ai.cra_driver_name;
4488                         break;
4489                 }
4490                 if (err) {
4491                         pr_err("%s : Algorithm registration failed\n", name);
4492                         goto register_err;
4493                 } else {
4494                         driver_algs[i].is_registered = 1;
4495                 }
4496         }
4497         return 0;
4498
4499 register_err:
4500         chcr_unregister_alg();
4501         return err;
4502 }
4503
4504 /*
4505  *      start_crypto - Register the crypto algorithms.
4506  *      This should called once when the first device comesup. After this
4507  *      kernel will start calling driver APIs for crypto operations.
4508  */
4509 int start_crypto(void)
4510 {
4511         return chcr_register_alg();
4512 }
4513
4514 /*
4515  *      stop_crypto - Deregister all the crypto algorithms with kernel.
4516  *      This should be called once when the last device goes down. After this
4517  *      kernel will not call the driver API for crypto operations.
4518  */
4519 int stop_crypto(void)
4520 {
4521         chcr_unregister_alg();
4522         return 0;
4523 }