drivers/crypto/caam/caamhash.c

   1 /*
   2  * caam - Freescale FSL CAAM support for ahash functions of crypto API
   3  *
   4  * Copyright 2011 Freescale Semiconductor, Inc.
   5  *
   6  * Based on caamalg.c crypto API driver.
   7  *
   8  * relationship of digest job descriptor or first job descriptor after init to
   9  * shared descriptors:
  10  *
  11  * ---------------                     ---------------
  12  * | JobDesc #1  |-------------------->|  ShareDesc  |
  13  * | *(packet 1) |                     |  (hashKey)  |
  14  * ---------------                     | (operation) |
  15  *                                     ---------------
  16  *
  17  * relationship of subsequent job descriptors to shared descriptors:
  18  *
  19  * ---------------                     ---------------
  20  * | JobDesc #2  |-------------------->|  ShareDesc  |
  21  * | *(packet 2) |      |------------->|  (hashKey)  |
  22  * ---------------      |    |-------->| (operation) |
  23  *       .              |    |         | (load ctx2) |
  24  *       .              |    |         ---------------
  25  * ---------------      |    |
  26  * | JobDesc #3  |------|    |
  27  * | *(packet 3) |           |
  28  * ---------------           |
  29  *       .                   |
  30  *       .                   |
  31  * ---------------           |
  32  * | JobDesc #4  |------------
  33  * | *(packet 4) |
  34  * ---------------
  35  *
  36  * The SharedDesc never changes for a connection unless rekeyed, but
  37  * each packet will likely be in a different place. So all we need
  38  * to know to process the packet is where the input is, where the
  39  * output goes, and what context we want to process with. Context is
  40  * in the SharedDesc, packet references in the JobDesc.
  41  *
  42  * So, a job desc looks like:
  43  *
  44  * ---------------------
  45  * | Header            |
  46  * | ShareDesc Pointer |
  47  * | SEQ_OUT_PTR       |
  48  * | (output buffer)   |
  49  * | (output length)   |
  50  * | SEQ_IN_PTR        |
  51  * | (input buffer)    |
  52  * | (input length)    |
  53  * ---------------------
  54  */
  55
  56 #include "compat.h"
  57
  58 #include "regs.h"
  59 #include "intern.h"
  60 #include "desc_constr.h"
  61 #include "jr.h"
  62 #include "error.h"
  63 #include "sg_sw_sec4.h"
  64 #include "key_gen.h"
  65
  66 #define CAAM_CRA_PRIORITY               3000
  67
  68 /* max hash key is max split key size */
  69 #define CAAM_MAX_HASH_KEY_SIZE          (SHA512_DIGEST_SIZE * 2)
  70
  71 #define CAAM_MAX_HASH_BLOCK_SIZE        SHA512_BLOCK_SIZE
  72 #define CAAM_MAX_HASH_DIGEST_SIZE       SHA512_DIGEST_SIZE
  73
  74 /* length of descriptors text */
  75 #define DESC_AHASH_BASE                 (3 * CAAM_CMD_SZ)
  76 #define DESC_AHASH_UPDATE_LEN           (6 * CAAM_CMD_SZ)
  77 #define DESC_AHASH_UPDATE_FIRST_LEN     (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
  78 #define DESC_AHASH_FINAL_LEN            (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
  79 #define DESC_AHASH_FINUP_LEN            (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
  80 #define DESC_AHASH_DIGEST_LEN           (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
  81
  82 #define DESC_HASH_MAX_USED_BYTES        (DESC_AHASH_FINAL_LEN + \
  83                                          CAAM_MAX_HASH_KEY_SIZE)
  84 #define DESC_HASH_MAX_USED_LEN          (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ)
  85
  86 /* caam context sizes for hashes: running digest + 8 */
  87 #define HASH_MSG_LEN                    8
  88 #define MAX_CTX_LEN                     (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
  89
  90 #ifdef DEBUG
  91 /* for print_hex_dumps with line references */
  92 #define debug(format, arg...) printk(format, arg)
  93 #else
  94 #define debug(format, arg...)
  95 #endif
  96
  97
  98 static struct list_head hash_list;
  99
 100 /* ahash per-session context */
 101 struct caam_hash_ctx {
 102         u32 sh_desc_update[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 103         u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 104         u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 105         u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 106         dma_addr_t sh_desc_update_dma ____cacheline_aligned;
 107         dma_addr_t sh_desc_update_first_dma;
 108         dma_addr_t sh_desc_fin_dma;
 109         dma_addr_t sh_desc_digest_dma;
 110         struct device *jrdev;
 111         u8 key[CAAM_MAX_HASH_KEY_SIZE];
 112         int ctx_len;
 113         struct alginfo adata;
 114 };
 115
 116 /* ahash state */
 117 struct caam_hash_state {
 118         dma_addr_t buf_dma;
 119         dma_addr_t ctx_dma;
 120         u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
 121         int buflen_0;
 122         u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
 123         int buflen_1;
 124         u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
 125         int (*update)(struct ahash_request *req);
 126         int (*final)(struct ahash_request *req);
 127         int (*finup)(struct ahash_request *req);
 128         int current_buf;
 129 };
 130
 131 struct caam_export_state {
 132         u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
 133         u8 caam_ctx[MAX_CTX_LEN];
 134         int buflen;
 135         int (*update)(struct ahash_request *req);
 136         int (*final)(struct ahash_request *req);
 137         int (*finup)(struct ahash_request *req);
 138 };
 139
 140 static inline void switch_buf(struct caam_hash_state *state)
 141 {
 142         state->current_buf ^= 1;
 143 }
 144
 145 static inline u8 *current_buf(struct caam_hash_state *state)
 146 {
 147         return state->current_buf ? state->buf_1 : state->buf_0;
 148 }
 149
 150 static inline u8 *alt_buf(struct caam_hash_state *state)
 151 {
 152         return state->current_buf ? state->buf_0 : state->buf_1;
 153 }
 154
 155 static inline int *current_buflen(struct caam_hash_state *state)
 156 {
 157         return state->current_buf ? &state->buflen_1 : &state->buflen_0;
 158 }
 159
 160 static inline int *alt_buflen(struct caam_hash_state *state)
 161 {
 162         return state->current_buf ? &state->buflen_0 : &state->buflen_1;
 163 }
 164
 165 /* Common job descriptor seq in/out ptr routines */
 166
 167 /* Map state->caam_ctx, and append seq_out_ptr command that points to it */
 168 static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev,
 169                                       struct caam_hash_state *state,
 170                                       int ctx_len)
 171 {
 172         state->ctx_dma = dma_map_single(jrdev, state->caam_ctx,
 173                                         ctx_len, DMA_FROM_DEVICE);
 174         if (dma_mapping_error(jrdev, state->ctx_dma)) {
 175                 dev_err(jrdev, "unable to map ctx\n");
 176                 state->ctx_dma = 0;
 177                 return -ENOMEM;
 178         }
 179
 180         append_seq_out_ptr(desc, state->ctx_dma, ctx_len, 0);
 181
 182         return 0;
 183 }
 184
 185 /* Map req->result, and append seq_out_ptr command that points to it */
 186 static inline dma_addr_t map_seq_out_ptr_result(u32 *desc, struct device *jrdev,
 187                                                 u8 *result, int digestsize)
 188 {
 189         dma_addr_t dst_dma;
 190
 191         dst_dma = dma_map_single(jrdev, result, digestsize, DMA_FROM_DEVICE);
 192         append_seq_out_ptr(desc, dst_dma, digestsize, 0);
 193
 194         return dst_dma;
 195 }
 196
 197 /* Map current buffer in state (if length > 0) and put it in link table */
 198 static inline int buf_map_to_sec4_sg(struct device *jrdev,
 199                                      struct sec4_sg_entry *sec4_sg,
 200                                      struct caam_hash_state *state)
 201 {
 202         int buflen = *current_buflen(state);
 203
 204         if (!buflen)
 205                 return 0;
 206
 207         state->buf_dma = dma_map_single(jrdev, current_buf(state), buflen,
 208                                         DMA_TO_DEVICE);
 209         if (dma_mapping_error(jrdev, state->buf_dma)) {
 210                 dev_err(jrdev, "unable to map buf\n");
 211                 state->buf_dma = 0;
 212                 return -ENOMEM;
 213         }
 214
 215         dma_to_sec4_sg_one(sec4_sg, state->buf_dma, buflen, 0);
 216
 217         return 0;
 218 }
 219
 220 /* Map state->caam_ctx, and add it to link table */
 221 static inline int ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev,
 222                                      struct caam_hash_state *state, int ctx_len,
 223                                      struct sec4_sg_entry *sec4_sg, u32 flag)
 224 {
 225         state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag);
 226         if (dma_mapping_error(jrdev, state->ctx_dma)) {
 227                 dev_err(jrdev, "unable to map ctx\n");
 228                 state->ctx_dma = 0;
 229                 return -ENOMEM;
 230         }
 231
 232         dma_to_sec4_sg_one(sec4_sg, state->ctx_dma, ctx_len, 0);
 233
 234         return 0;
 235 }
 236
 237 /*
 238  * For ahash update, final and finup (import_ctx = true)
 239  *     import context, read and write to seqout
 240  * For ahash firsts and digest (import_ctx = false)
 241  *     read and write to seqout
 242  */
 243 static inline void ahash_gen_sh_desc(u32 *desc, u32 state, int digestsize,
 244                                      struct caam_hash_ctx *ctx, bool import_ctx)
 245 {
 246         u32 op = ctx->adata.algtype;
 247         u32 *skip_key_load;
 248
 249         init_sh_desc(desc, HDR_SHARE_SERIAL);
 250
 251         /* Append key if it has been set; ahash update excluded */
 252         if ((state != OP_ALG_AS_UPDATE) && (ctx->adata.keylen)) {
 253                 /* Skip key loading if already shared */
 254                 skip_key_load = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 255                                             JUMP_COND_SHRD);
 256
 257                 append_key_as_imm(desc, ctx->key, ctx->adata.keylen_pad,
 258                                   ctx->adata.keylen, CLASS_2 |
 259                                   KEY_DEST_MDHA_SPLIT | KEY_ENC);
 260
 261                 set_jump_tgt_here(desc, skip_key_load);
 262
 263                 op |= OP_ALG_AAI_HMAC_PRECOMP;
 264         }
 265
 266         /* If needed, import context from software */
 267         if (import_ctx)
 268                 append_seq_load(desc, ctx->ctx_len, LDST_CLASS_2_CCB |
 269                                 LDST_SRCDST_BYTE_CONTEXT);
 270
 271         /* Class 2 operation */
 272         append_operation(desc, op | state | OP_ALG_ENCRYPT);
 273
 274         /*
 275          * Load from buf and/or src and write to req->result or state->context
 276          * Calculate remaining bytes to read
 277          */
 278         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 279         /* Read remaining bytes */
 280         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 |
 281                              FIFOLD_TYPE_MSG | KEY_VLF);
 282         /* Store class2 context bytes */
 283         append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
 284                          LDST_SRCDST_BYTE_CONTEXT);
 285 }
 286
 287 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
 288 {
 289         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 290         int digestsize = crypto_ahash_digestsize(ahash);
 291         struct device *jrdev = ctx->jrdev;
 292         u32 *desc;
 293
 294         /* ahash_update shared descriptor */
 295         desc = ctx->sh_desc_update;
 296         ahash_gen_sh_desc(desc, OP_ALG_AS_UPDATE, ctx->ctx_len, ctx, true);
 297         dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
 298                                    desc_bytes(desc), DMA_TO_DEVICE);
 299 #ifdef DEBUG
 300         print_hex_dump(KERN_ERR,
 301                        "ahash update shdesc@"__stringify(__LINE__)": ",
 302                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 303 #endif
 304
 305         /* ahash_update_first shared descriptor */
 306         desc = ctx->sh_desc_update_first;
 307         ahash_gen_sh_desc(desc, OP_ALG_AS_INIT, ctx->ctx_len, ctx, false);
 308         dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
 309                                    desc_bytes(desc), DMA_TO_DEVICE);
 310 #ifdef DEBUG
 311         print_hex_dump(KERN_ERR,
 312                        "ahash update first shdesc@"__stringify(__LINE__)": ",
 313                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 314 #endif
 315
 316         /* ahash_final shared descriptor */
 317         desc = ctx->sh_desc_fin;
 318         ahash_gen_sh_desc(desc, OP_ALG_AS_FINALIZE, digestsize, ctx, true);
 319         dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
 320                                    desc_bytes(desc), DMA_TO_DEVICE);
 321 #ifdef DEBUG
 322         print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ",
 323                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 324                        desc_bytes(desc), 1);
 325 #endif
 326
 327         /* ahash_digest shared descriptor */
 328         desc = ctx->sh_desc_digest;
 329         ahash_gen_sh_desc(desc, OP_ALG_AS_INITFINAL, digestsize, ctx, false);
 330         dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
 331                                    desc_bytes(desc), DMA_TO_DEVICE);
 332 #ifdef DEBUG
 333         print_hex_dump(KERN_ERR,
 334                        "ahash digest shdesc@"__stringify(__LINE__)": ",
 335                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 336                        desc_bytes(desc), 1);
 337 #endif
 338
 339         return 0;
 340 }
 341
 342 /* Digest hash size if it is too large */
 343 static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
 344                            u32 *keylen, u8 *key_out, u32 digestsize)
 345 {
 346         struct device *jrdev = ctx->jrdev;
 347         u32 *desc;
 348         struct split_key_result result;
 349         dma_addr_t src_dma, dst_dma;
 350         int ret;
 351
 352         desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
 353         if (!desc) {
 354                 dev_err(jrdev, "unable to allocate key input memory\n");
 355                 return -ENOMEM;
 356         }
 357
 358         init_job_desc(desc, 0);
 359
 360         src_dma = dma_map_single(jrdev, (void *)key_in, *keylen,
 361                                  DMA_TO_DEVICE);
 362         if (dma_mapping_error(jrdev, src_dma)) {
 363                 dev_err(jrdev, "unable to map key input memory\n");
 364                 kfree(desc);
 365                 return -ENOMEM;
 366         }
 367         dst_dma = dma_map_single(jrdev, (void *)key_out, digestsize,
 368                                  DMA_FROM_DEVICE);
 369         if (dma_mapping_error(jrdev, dst_dma)) {
 370                 dev_err(jrdev, "unable to map key output memory\n");
 371                 dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE);
 372                 kfree(desc);
 373                 return -ENOMEM;
 374         }
 375
 376         /* Job descriptor to perform unkeyed hash on key_in */
 377         append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
 378                          OP_ALG_AS_INITFINAL);
 379         append_seq_in_ptr(desc, src_dma, *keylen, 0);
 380         append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
 381                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
 382         append_seq_out_ptr(desc, dst_dma, digestsize, 0);
 383         append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
 384                          LDST_SRCDST_BYTE_CONTEXT);
 385
 386 #ifdef DEBUG
 387         print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ",
 388                        DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1);
 389         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
 390                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 391 #endif
 392
 393         result.err = 0;
 394         init_completion(&result.completion);
 395
 396         ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result);
 397         if (!ret) {
 398                 /* in progress */
 399                 wait_for_completion(&result.completion);
 400                 ret = result.err;
 401 #ifdef DEBUG
 402                 print_hex_dump(KERN_ERR,
 403                                "digested key@"__stringify(__LINE__)": ",
 404                                DUMP_PREFIX_ADDRESS, 16, 4, key_in,
 405                                digestsize, 1);
 406 #endif
 407         }
 408         dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE);
 409         dma_unmap_single(jrdev, dst_dma, digestsize, DMA_FROM_DEVICE);
 410
 411         *keylen = digestsize;
 412
 413         kfree(desc);
 414
 415         return ret;
 416 }
 417
 418 static int ahash_setkey(struct crypto_ahash *ahash,
 419                         const u8 *key, unsigned int keylen)
 420 {
 421         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 422         int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
 423         int digestsize = crypto_ahash_digestsize(ahash);
 424         int ret;
 425         u8 *hashed_key = NULL;
 426
 427 #ifdef DEBUG
 428         printk(KERN_ERR "keylen %d\n", keylen);
 429 #endif
 430
 431         if (keylen > blocksize) {
 432                 hashed_key = kmalloc_array(digestsize,
 433                                            sizeof(*hashed_key),
 434                                            GFP_KERNEL | GFP_DMA);
 435                 if (!hashed_key)
 436                         return -ENOMEM;
 437                 ret = hash_digest_key(ctx, key, &keylen, hashed_key,
 438                                       digestsize);
 439                 if (ret)
 440                         goto bad_free_key;
 441                 key = hashed_key;
 442         }
 443
 444         ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key, keylen,
 445                             CAAM_MAX_HASH_KEY_SIZE);
 446         if (ret)
 447                 goto bad_free_key;
 448
 449 #ifdef DEBUG
 450         print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
 451                        DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 452                        ctx->adata.keylen_pad, 1);
 453 #endif
 454
 455         kfree(hashed_key);
 456         return ahash_set_sh_desc(ahash);
 457  bad_free_key:
 458         kfree(hashed_key);
 459         crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
 460         return -EINVAL;
 461 }
 462
 463 /*
 464  * ahash_edesc - s/w-extended ahash descriptor
 465  * @dst_dma: physical mapped address of req->result
 466  * @sec4_sg_dma: physical mapped address of h/w link table
 467  * @src_nents: number of segments in input scatterlist
 468  * @sec4_sg_bytes: length of dma mapped sec4_sg space
 469  * @hw_desc: the h/w job descriptor followed by any referenced link tables
 470  * @sec4_sg: h/w link table
 471  */
 472 struct ahash_edesc {
 473         dma_addr_t dst_dma;
 474         dma_addr_t sec4_sg_dma;
 475         int src_nents;
 476         int sec4_sg_bytes;
 477         u32 hw_desc[DESC_JOB_IO_LEN / sizeof(u32)] ____cacheline_aligned;
 478         struct sec4_sg_entry sec4_sg[0];
 479 };
 480
 481 static inline void ahash_unmap(struct device *dev,
 482                         struct ahash_edesc *edesc,
 483                         struct ahash_request *req, int dst_len)
 484 {
 485         struct caam_hash_state *state = ahash_request_ctx(req);
 486
 487         if (edesc->src_nents)
 488                 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
 489         if (edesc->dst_dma)
 490                 dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE);
 491
 492         if (edesc->sec4_sg_bytes)
 493                 dma_unmap_single(dev, edesc->sec4_sg_dma,
 494                                  edesc->sec4_sg_bytes, DMA_TO_DEVICE);
 495
 496         if (state->buf_dma) {
 497                 dma_unmap_single(dev, state->buf_dma, *current_buflen(state),
 498                                  DMA_TO_DEVICE);
 499                 state->buf_dma = 0;
 500         }
 501 }
 502
 503 static inline void ahash_unmap_ctx(struct device *dev,
 504                         struct ahash_edesc *edesc,
 505                         struct ahash_request *req, int dst_len, u32 flag)
 506 {
 507         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 508         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 509         struct caam_hash_state *state = ahash_request_ctx(req);
 510
 511         if (state->ctx_dma) {
 512                 dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag);
 513                 state->ctx_dma = 0;
 514         }
 515         ahash_unmap(dev, edesc, req, dst_len);
 516 }
 517
 518 static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
 519                        void *context)
 520 {
 521         struct ahash_request *req = context;
 522         struct ahash_edesc *edesc;
 523         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 524         int digestsize = crypto_ahash_digestsize(ahash);
 525 #ifdef DEBUG
 526         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 527         struct caam_hash_state *state = ahash_request_ctx(req);
 528
 529         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 530 #endif
 531
 532         edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
 533         if (err)
 534                 caam_jr_strstatus(jrdev, err);
 535
 536         ahash_unmap(jrdev, edesc, req, digestsize);
 537         kfree(edesc);
 538
 539 #ifdef DEBUG
 540         print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
 541                        DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
 542                        ctx->ctx_len, 1);
 543         if (req->result)
 544                 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
 545                                DUMP_PREFIX_ADDRESS, 16, 4, req->result,
 546                                digestsize, 1);
 547 #endif
 548
 549         req->base.complete(&req->base, err);
 550 }
 551
 552 static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
 553                             void *context)
 554 {
 555         struct ahash_request *req = context;
 556         struct ahash_edesc *edesc;
 557         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 558         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 559         struct caam_hash_state *state = ahash_request_ctx(req);
 560 #ifdef DEBUG
 561         int digestsize = crypto_ahash_digestsize(ahash);
 562
 563         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 564 #endif
 565
 566         edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
 567         if (err)
 568                 caam_jr_strstatus(jrdev, err);
 569
 570         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
 571         switch_buf(state);
 572         kfree(edesc);
 573
 574 #ifdef DEBUG
 575         print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
 576                        DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
 577                        ctx->ctx_len, 1);
 578         if (req->result)
 579                 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
 580                                DUMP_PREFIX_ADDRESS, 16, 4, req->result,
 581                                digestsize, 1);
 582 #endif
 583
 584         req->base.complete(&req->base, err);
 585 }
 586
 587 static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
 588                                void *context)
 589 {
 590         struct ahash_request *req = context;
 591         struct ahash_edesc *edesc;
 592         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 593         int digestsize = crypto_ahash_digestsize(ahash);
 594 #ifdef DEBUG
 595         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 596         struct caam_hash_state *state = ahash_request_ctx(req);
 597
 598         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 599 #endif
 600
 601         edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
 602         if (err)
 603                 caam_jr_strstatus(jrdev, err);
 604
 605         ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_TO_DEVICE);
 606         kfree(edesc);
 607
 608 #ifdef DEBUG
 609         print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
 610                        DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
 611                        ctx->ctx_len, 1);
 612         if (req->result)
 613                 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
 614                                DUMP_PREFIX_ADDRESS, 16, 4, req->result,
 615                                digestsize, 1);
 616 #endif
 617
 618         req->base.complete(&req->base, err);
 619 }
 620
 621 static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
 622                                void *context)
 623 {
 624         struct ahash_request *req = context;
 625         struct ahash_edesc *edesc;
 626         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 627         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 628         struct caam_hash_state *state = ahash_request_ctx(req);
 629 #ifdef DEBUG
 630         int digestsize = crypto_ahash_digestsize(ahash);
 631
 632         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 633 #endif
 634
 635         edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
 636         if (err)
 637                 caam_jr_strstatus(jrdev, err);
 638
 639         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE);
 640         switch_buf(state);
 641         kfree(edesc);
 642
 643 #ifdef DEBUG
 644         print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
 645                        DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
 646                        ctx->ctx_len, 1);
 647         if (req->result)
 648                 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
 649                                DUMP_PREFIX_ADDRESS, 16, 4, req->result,
 650                                digestsize, 1);
 651 #endif
 652
 653         req->base.complete(&req->base, err);
 654 }
 655
 656 /*
 657  * Allocate an enhanced descriptor, which contains the hardware descriptor
 658  * and space for hardware scatter table containing sg_num entries.
 659  */
 660 static struct ahash_edesc *ahash_edesc_alloc(struct caam_hash_ctx *ctx,
 661                                              int sg_num, u32 *sh_desc,
 662                                              dma_addr_t sh_desc_dma,
 663                                              gfp_t flags)
 664 {
 665         struct ahash_edesc *edesc;
 666         unsigned int sg_size = sg_num * sizeof(struct sec4_sg_entry);
 667
 668         edesc = kzalloc(sizeof(*edesc) + sg_size, GFP_DMA | flags);
 669         if (!edesc) {
 670                 dev_err(ctx->jrdev, "could not allocate extended descriptor\n");
 671                 return NULL;
 672         }
 673
 674         init_job_desc_shared(edesc->hw_desc, sh_desc_dma, desc_len(sh_desc),
 675                              HDR_SHARE_DEFER | HDR_REVERSE);
 676
 677         return edesc;
 678 }
 679
 680 static int ahash_edesc_add_src(struct caam_hash_ctx *ctx,
 681                                struct ahash_edesc *edesc,
 682                                struct ahash_request *req, int nents,
 683                                unsigned int first_sg,
 684                                unsigned int first_bytes, size_t to_hash)
 685 {
 686         dma_addr_t src_dma;
 687         u32 options;
 688
 689         if (nents > 1 || first_sg) {
 690                 struct sec4_sg_entry *sg = edesc->sec4_sg;
 691                 unsigned int sgsize = sizeof(*sg) * (first_sg + nents);
 692
 693                 sg_to_sec4_sg_last(req->src, nents, sg + first_sg, 0);
 694
 695                 src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE);
 696                 if (dma_mapping_error(ctx->jrdev, src_dma)) {
 697                         dev_err(ctx->jrdev, "unable to map S/G table\n");
 698                         return -ENOMEM;
 699                 }
 700
 701                 edesc->sec4_sg_bytes = sgsize;
 702                 edesc->sec4_sg_dma = src_dma;
 703                 options = LDST_SGF;
 704         } else {
 705                 src_dma = sg_dma_address(req->src);
 706                 options = 0;
 707         }
 708
 709         append_seq_in_ptr(edesc->hw_desc, src_dma, first_bytes + to_hash,
 710                           options);
 711
 712         return 0;
 713 }
 714
 715 /* submit update job descriptor */
 716 static int ahash_update_ctx(struct ahash_request *req)
 717 {
 718         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 719         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 720         struct caam_hash_state *state = ahash_request_ctx(req);
 721         struct device *jrdev = ctx->jrdev;
 722         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 723                        GFP_KERNEL : GFP_ATOMIC;
 724         u8 *buf = current_buf(state);
 725         int *buflen = current_buflen(state);
 726         u8 *next_buf = alt_buf(state);
 727         int *next_buflen = alt_buflen(state), last_buflen;
 728         int in_len = *buflen + req->nbytes, to_hash;
 729         u32 *desc;
 730         int src_nents, mapped_nents, sec4_sg_bytes, sec4_sg_src_index;
 731         struct ahash_edesc *edesc;
 732         int ret = 0;
 733
 734         last_buflen = *next_buflen;
 735         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
 736         to_hash = in_len - *next_buflen;
 737
 738         if (to_hash) {
 739                 src_nents = sg_nents_for_len(req->src,
 740                                              req->nbytes - (*next_buflen));
 741                 if (src_nents < 0) {
 742                         dev_err(jrdev, "Invalid number of src SG.\n");
 743                         return src_nents;
 744                 }
 745
 746                 if (src_nents) {
 747                         mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
 748                                                   DMA_TO_DEVICE);
 749                         if (!mapped_nents) {
 750                                 dev_err(jrdev, "unable to DMA map source\n");
 751                                 return -ENOMEM;
 752                         }
 753                 } else {
 754                         mapped_nents = 0;
 755                 }
 756
 757                 sec4_sg_src_index = 1 + (*buflen ? 1 : 0);
 758                 sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
 759                                  sizeof(struct sec4_sg_entry);
 760
 761                 /*
 762                  * allocate space for base edesc and hw desc commands,
 763                  * link tables
 764                  */
 765                 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
 766                                           ctx->sh_desc_update,
 767                                           ctx->sh_desc_update_dma, flags);
 768                 if (!edesc) {
 769                         dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
 770                         return -ENOMEM;
 771                 }
 772
 773                 edesc->src_nents = src_nents;
 774                 edesc->sec4_sg_bytes = sec4_sg_bytes;
 775
 776                 ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
 777                                          edesc->sec4_sg, DMA_BIDIRECTIONAL);
 778                 if (ret)
 779                         goto unmap_ctx;
 780
 781                 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
 782                 if (ret)
 783                         goto unmap_ctx;
 784
 785                 if (mapped_nents) {
 786                         sg_to_sec4_sg_last(req->src, mapped_nents,
 787                                            edesc->sec4_sg + sec4_sg_src_index,
 788                                            0);
 789                         if (*next_buflen)
 790                                 scatterwalk_map_and_copy(next_buf, req->src,
 791                                                          to_hash - *buflen,
 792                                                          *next_buflen, 0);
 793                 } else {
 794                         sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index -
 795                                             1);
 796                 }
 797
 798                 desc = edesc->hw_desc;
 799
 800                 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
 801                                                      sec4_sg_bytes,
 802                                                      DMA_TO_DEVICE);
 803                 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
 804                         dev_err(jrdev, "unable to map S/G table\n");
 805                         ret = -ENOMEM;
 806                         goto unmap_ctx;
 807                 }
 808
 809                 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len +
 810                                        to_hash, LDST_SGF);
 811
 812                 append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0);
 813
 814 #ifdef DEBUG
 815                 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
 816                                DUMP_PREFIX_ADDRESS, 16, 4, desc,
 817                                desc_bytes(desc), 1);
 818 #endif
 819
 820                 ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req);
 821                 if (ret)
 822                         goto unmap_ctx;
 823
 824                 ret = -EINPROGRESS;
 825         } else if (*next_buflen) {
 826                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
 827                                          req->nbytes, 0);
 828                 *buflen = *next_buflen;
 829                 *next_buflen = last_buflen;
 830         }
 831 #ifdef DEBUG
 832         print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
 833                        DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
 834         print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
 835                        DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
 836                        *next_buflen, 1);
 837 #endif
 838
 839         return ret;
 840  unmap_ctx:
 841         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
 842         kfree(edesc);
 843         return ret;
 844 }
 845
 846 static int ahash_final_ctx(struct ahash_request *req)
 847 {
 848         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 849         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 850         struct caam_hash_state *state = ahash_request_ctx(req);
 851         struct device *jrdev = ctx->jrdev;
 852         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 853                        GFP_KERNEL : GFP_ATOMIC;
 854         int buflen = *current_buflen(state);
 855         u32 *desc;
 856         int sec4_sg_bytes, sec4_sg_src_index;
 857         int digestsize = crypto_ahash_digestsize(ahash);
 858         struct ahash_edesc *edesc;
 859         int ret;
 860
 861         sec4_sg_src_index = 1 + (buflen ? 1 : 0);
 862         sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry);
 863
 864         /* allocate space for base edesc and hw desc commands, link tables */
 865         edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index,
 866                                   ctx->sh_desc_fin, ctx->sh_desc_fin_dma,
 867                                   flags);
 868         if (!edesc)
 869                 return -ENOMEM;
 870
 871         desc = edesc->hw_desc;
 872
 873         edesc->sec4_sg_bytes = sec4_sg_bytes;
 874         edesc->src_nents = 0;
 875
 876         ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
 877                                  edesc->sec4_sg, DMA_TO_DEVICE);
 878         if (ret)
 879                 goto unmap_ctx;
 880
 881         ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
 882         if (ret)
 883                 goto unmap_ctx;
 884
 885         sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 1);
 886
 887         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
 888                                             sec4_sg_bytes, DMA_TO_DEVICE);
 889         if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
 890                 dev_err(jrdev, "unable to map S/G table\n");
 891                 ret = -ENOMEM;
 892                 goto unmap_ctx;
 893         }
 894
 895         append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen,
 896                           LDST_SGF);
 897
 898         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
 899                                                 digestsize);
 900         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
 901                 dev_err(jrdev, "unable to map dst\n");
 902                 ret = -ENOMEM;
 903                 goto unmap_ctx;
 904         }
 905
 906 #ifdef DEBUG
 907         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
 908                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 909 #endif
 910
 911         ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
 912         if (ret)
 913                 goto unmap_ctx;
 914
 915         return -EINPROGRESS;
 916  unmap_ctx:
 917         ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
 918         kfree(edesc);
 919         return ret;
 920 }
 921
 922 static int ahash_finup_ctx(struct ahash_request *req)
 923 {
 924         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 925         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 926         struct caam_hash_state *state = ahash_request_ctx(req);
 927         struct device *jrdev = ctx->jrdev;
 928         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 929                        GFP_KERNEL : GFP_ATOMIC;
 930         int buflen = *current_buflen(state);
 931         u32 *desc;
 932         int sec4_sg_src_index;
 933         int src_nents, mapped_nents;
 934         int digestsize = crypto_ahash_digestsize(ahash);
 935         struct ahash_edesc *edesc;
 936         int ret;
 937
 938         src_nents = sg_nents_for_len(req->src, req->nbytes);
 939         if (src_nents < 0) {
 940                 dev_err(jrdev, "Invalid number of src SG.\n");
 941                 return src_nents;
 942         }
 943
 944         if (src_nents) {
 945                 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
 946                                           DMA_TO_DEVICE);
 947                 if (!mapped_nents) {
 948                         dev_err(jrdev, "unable to DMA map source\n");
 949                         return -ENOMEM;
 950                 }
 951         } else {
 952                 mapped_nents = 0;
 953         }
 954
 955         sec4_sg_src_index = 1 + (buflen ? 1 : 0);
 956
 957         /* allocate space for base edesc and hw desc commands, link tables */
 958         edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
 959                                   ctx->sh_desc_fin, ctx->sh_desc_fin_dma,
 960                                   flags);
 961         if (!edesc) {
 962                 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
 963                 return -ENOMEM;
 964         }
 965
 966         desc = edesc->hw_desc;
 967
 968         edesc->src_nents = src_nents;
 969
 970         ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
 971                                  edesc->sec4_sg, DMA_TO_DEVICE);
 972         if (ret)
 973                 goto unmap_ctx;
 974
 975         ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
 976         if (ret)
 977                 goto unmap_ctx;
 978
 979         ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents,
 980                                   sec4_sg_src_index, ctx->ctx_len + buflen,
 981                                   req->nbytes);
 982         if (ret)
 983                 goto unmap_ctx;
 984
 985         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
 986                                                 digestsize);
 987         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
 988                 dev_err(jrdev, "unable to map dst\n");
 989                 ret = -ENOMEM;
 990                 goto unmap_ctx;
 991         }
 992
 993 #ifdef DEBUG
 994         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
 995                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
 996 #endif
 997
 998         ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
 999         if (ret)
1000                 goto unmap_ctx;
1001
1002         return -EINPROGRESS;
1003  unmap_ctx:
1004         ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
1005         kfree(edesc);
1006         return ret;
1007 }
1008
1009 static int ahash_digest(struct ahash_request *req)
1010 {
1011         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1012         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1013         struct caam_hash_state *state = ahash_request_ctx(req);
1014         struct device *jrdev = ctx->jrdev;
1015         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1016                        GFP_KERNEL : GFP_ATOMIC;
1017         u32 *desc;
1018         int digestsize = crypto_ahash_digestsize(ahash);
1019         int src_nents, mapped_nents;
1020         struct ahash_edesc *edesc;
1021         int ret;
1022
1023         state->buf_dma = 0;
1024
1025         src_nents = sg_nents_for_len(req->src, req->nbytes);
1026         if (src_nents < 0) {
1027                 dev_err(jrdev, "Invalid number of src SG.\n");
1028                 return src_nents;
1029         }
1030
1031         if (src_nents) {
1032                 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1033                                           DMA_TO_DEVICE);
1034                 if (!mapped_nents) {
1035                         dev_err(jrdev, "unable to map source for DMA\n");
1036                         return -ENOMEM;
1037                 }
1038         } else {
1039                 mapped_nents = 0;
1040         }
1041
1042         /* allocate space for base edesc and hw desc commands, link tables */
1043         edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ? mapped_nents : 0,
1044                                   ctx->sh_desc_digest, ctx->sh_desc_digest_dma,
1045                                   flags);
1046         if (!edesc) {
1047                 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1048                 return -ENOMEM;
1049         }
1050
1051         edesc->src_nents = src_nents;
1052
1053         ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
1054                                   req->nbytes);
1055         if (ret) {
1056                 ahash_unmap(jrdev, edesc, req, digestsize);
1057                 kfree(edesc);
1058                 return ret;
1059         }
1060
1061         desc = edesc->hw_desc;
1062
1063         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1064                                                 digestsize);
1065         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1066                 dev_err(jrdev, "unable to map dst\n");
1067                 ahash_unmap(jrdev, edesc, req, digestsize);
1068                 kfree(edesc);
1069                 return -ENOMEM;
1070         }
1071
1072 #ifdef DEBUG
1073         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1074                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1075 #endif
1076
1077         ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1078         if (!ret) {
1079                 ret = -EINPROGRESS;
1080         } else {
1081                 ahash_unmap(jrdev, edesc, req, digestsize);
1082                 kfree(edesc);
1083         }
1084
1085         return ret;
1086 }
1087
1088 /* submit ahash final if it the first job descriptor */
1089 static int ahash_final_no_ctx(struct ahash_request *req)
1090 {
1091         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1092         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1093         struct caam_hash_state *state = ahash_request_ctx(req);
1094         struct device *jrdev = ctx->jrdev;
1095         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1096                        GFP_KERNEL : GFP_ATOMIC;
1097         u8 *buf = current_buf(state);
1098         int buflen = *current_buflen(state);
1099         u32 *desc;
1100         int digestsize = crypto_ahash_digestsize(ahash);
1101         struct ahash_edesc *edesc;
1102         int ret;
1103
1104         /* allocate space for base edesc and hw desc commands, link tables */
1105         edesc = ahash_edesc_alloc(ctx, 0, ctx->sh_desc_digest,
1106                                   ctx->sh_desc_digest_dma, flags);
1107         if (!edesc)
1108                 return -ENOMEM;
1109
1110         desc = edesc->hw_desc;
1111
1112         state->buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE);
1113         if (dma_mapping_error(jrdev, state->buf_dma)) {
1114                 dev_err(jrdev, "unable to map src\n");
1115                 goto unmap;
1116         }
1117
1118         append_seq_in_ptr(desc, state->buf_dma, buflen, 0);
1119
1120         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1121                                                 digestsize);
1122         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1123                 dev_err(jrdev, "unable to map dst\n");
1124                 goto unmap;
1125         }
1126         edesc->src_nents = 0;
1127
1128 #ifdef DEBUG
1129         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1130                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1131 #endif
1132
1133         ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1134         if (!ret) {
1135                 ret = -EINPROGRESS;
1136         } else {
1137                 ahash_unmap(jrdev, edesc, req, digestsize);
1138                 kfree(edesc);
1139         }
1140
1141         return ret;
1142  unmap:
1143         ahash_unmap(jrdev, edesc, req, digestsize);
1144         kfree(edesc);
1145         return -ENOMEM;
1146
1147 }
1148
1149 /* submit ahash update if it the first job descriptor after update */
1150 static int ahash_update_no_ctx(struct ahash_request *req)
1151 {
1152         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1153         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1154         struct caam_hash_state *state = ahash_request_ctx(req);
1155         struct device *jrdev = ctx->jrdev;
1156         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1157                        GFP_KERNEL : GFP_ATOMIC;
1158         u8 *buf = current_buf(state);
1159         int *buflen = current_buflen(state);
1160         u8 *next_buf = alt_buf(state);
1161         int *next_buflen = alt_buflen(state);
1162         int in_len = *buflen + req->nbytes, to_hash;
1163         int sec4_sg_bytes, src_nents, mapped_nents;
1164         struct ahash_edesc *edesc;
1165         u32 *desc;
1166         int ret = 0;
1167
1168         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
1169         to_hash = in_len - *next_buflen;
1170
1171         if (to_hash) {
1172                 src_nents = sg_nents_for_len(req->src,
1173                                              req->nbytes - *next_buflen);
1174                 if (src_nents < 0) {
1175                         dev_err(jrdev, "Invalid number of src SG.\n");
1176                         return src_nents;
1177                 }
1178
1179                 if (src_nents) {
1180                         mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1181                                                   DMA_TO_DEVICE);
1182                         if (!mapped_nents) {
1183                                 dev_err(jrdev, "unable to DMA map source\n");
1184                                 return -ENOMEM;
1185                         }
1186                 } else {
1187                         mapped_nents = 0;
1188                 }
1189
1190                 sec4_sg_bytes = (1 + mapped_nents) *
1191                                 sizeof(struct sec4_sg_entry);
1192
1193                 /*
1194                  * allocate space for base edesc and hw desc commands,
1195                  * link tables
1196                  */
1197                 edesc = ahash_edesc_alloc(ctx, 1 + mapped_nents,
1198                                           ctx->sh_desc_update_first,
1199                                           ctx->sh_desc_update_first_dma,
1200                                           flags);
1201                 if (!edesc) {
1202                         dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1203                         return -ENOMEM;
1204                 }
1205
1206                 edesc->src_nents = src_nents;
1207                 edesc->sec4_sg_bytes = sec4_sg_bytes;
1208                 edesc->dst_dma = 0;
1209
1210                 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);
1211                 if (ret)
1212                         goto unmap_ctx;
1213
1214                 sg_to_sec4_sg_last(req->src, mapped_nents,
1215                                    edesc->sec4_sg + 1, 0);
1216
1217                 if (*next_buflen) {
1218                         scatterwalk_map_and_copy(next_buf, req->src,
1219                                                  to_hash - *buflen,
1220                                                  *next_buflen, 0);
1221                 }
1222
1223                 desc = edesc->hw_desc;
1224
1225                 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1226                                                     sec4_sg_bytes,
1227                                                     DMA_TO_DEVICE);
1228                 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1229                         dev_err(jrdev, "unable to map S/G table\n");
1230                         ret = -ENOMEM;
1231                         goto unmap_ctx;
1232                 }
1233
1234                 append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF);
1235
1236                 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
1237                 if (ret)
1238                         goto unmap_ctx;
1239
1240 #ifdef DEBUG
1241                 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1242                                DUMP_PREFIX_ADDRESS, 16, 4, desc,
1243                                desc_bytes(desc), 1);
1244 #endif
1245
1246                 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
1247                 if (ret)
1248                         goto unmap_ctx;
1249
1250                 ret = -EINPROGRESS;
1251                 state->update = ahash_update_ctx;
1252                 state->finup = ahash_finup_ctx;
1253                 state->final = ahash_final_ctx;
1254         } else if (*next_buflen) {
1255                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
1256                                          req->nbytes, 0);
1257                 *buflen = *next_buflen;
1258                 *next_buflen = 0;
1259         }
1260 #ifdef DEBUG
1261         print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
1262                        DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
1263         print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
1264                        DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
1265                        *next_buflen, 1);
1266 #endif
1267
1268         return ret;
1269  unmap_ctx:
1270         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
1271         kfree(edesc);
1272         return ret;
1273 }
1274
1275 /* submit ahash finup if it the first job descriptor after update */
1276 static int ahash_finup_no_ctx(struct ahash_request *req)
1277 {
1278         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1279         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1280         struct caam_hash_state *state = ahash_request_ctx(req);
1281         struct device *jrdev = ctx->jrdev;
1282         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1283                        GFP_KERNEL : GFP_ATOMIC;
1284         int buflen = *current_buflen(state);
1285         u32 *desc;
1286         int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents;
1287         int digestsize = crypto_ahash_digestsize(ahash);
1288         struct ahash_edesc *edesc;
1289         int ret;
1290
1291         src_nents = sg_nents_for_len(req->src, req->nbytes);
1292         if (src_nents < 0) {
1293                 dev_err(jrdev, "Invalid number of src SG.\n");
1294                 return src_nents;
1295         }
1296
1297         if (src_nents) {
1298                 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1299                                           DMA_TO_DEVICE);
1300                 if (!mapped_nents) {
1301                         dev_err(jrdev, "unable to DMA map source\n");
1302                         return -ENOMEM;
1303                 }
1304         } else {
1305                 mapped_nents = 0;
1306         }
1307
1308         sec4_sg_src_index = 2;
1309         sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
1310                          sizeof(struct sec4_sg_entry);
1311
1312         /* allocate space for base edesc and hw desc commands, link tables */
1313         edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
1314                                   ctx->sh_desc_digest, ctx->sh_desc_digest_dma,
1315                                   flags);
1316         if (!edesc) {
1317                 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1318                 return -ENOMEM;
1319         }
1320
1321         desc = edesc->hw_desc;
1322
1323         edesc->src_nents = src_nents;
1324         edesc->sec4_sg_bytes = sec4_sg_bytes;
1325
1326         ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);
1327         if (ret)
1328                 goto unmap;
1329
1330         ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen,
1331                                   req->nbytes);
1332         if (ret) {
1333                 dev_err(jrdev, "unable to map S/G table\n");
1334                 goto unmap;
1335         }
1336
1337         edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1338                                                 digestsize);
1339         if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1340                 dev_err(jrdev, "unable to map dst\n");
1341                 goto unmap;
1342         }
1343
1344 #ifdef DEBUG
1345         print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1346                        DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1347 #endif
1348
1349         ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1350         if (!ret) {
1351                 ret = -EINPROGRESS;
1352         } else {
1353                 ahash_unmap(jrdev, edesc, req, digestsize);
1354                 kfree(edesc);
1355         }
1356
1357         return ret;
1358  unmap:
1359         ahash_unmap(jrdev, edesc, req, digestsize);
1360         kfree(edesc);
1361         return -ENOMEM;
1362
1363 }
1364
1365 /* submit first update job descriptor after init */
1366 static int ahash_update_first(struct ahash_request *req)
1367 {
1368         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1369         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1370         struct caam_hash_state *state = ahash_request_ctx(req);
1371         struct device *jrdev = ctx->jrdev;
1372         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1373                        GFP_KERNEL : GFP_ATOMIC;
1374         u8 *next_buf = alt_buf(state);
1375         int *next_buflen = alt_buflen(state);
1376         int to_hash;
1377         u32 *desc;
1378         int src_nents, mapped_nents;
1379         struct ahash_edesc *edesc;
1380         int ret = 0;
1381
1382         *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
1383                                       1);
1384         to_hash = req->nbytes - *next_buflen;
1385
1386         if (to_hash) {
1387                 src_nents = sg_nents_for_len(req->src,
1388                                              req->nbytes - *next_buflen);
1389                 if (src_nents < 0) {
1390                         dev_err(jrdev, "Invalid number of src SG.\n");
1391                         return src_nents;
1392                 }
1393
1394                 if (src_nents) {
1395                         mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1396                                                   DMA_TO_DEVICE);
1397                         if (!mapped_nents) {
1398                                 dev_err(jrdev, "unable to map source for DMA\n");
1399                                 return -ENOMEM;
1400                         }
1401                 } else {
1402                         mapped_nents = 0;
1403                 }
1404
1405                 /*
1406                  * allocate space for base edesc and hw desc commands,
1407                  * link tables
1408                  */
1409                 edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ?
1410                                           mapped_nents : 0,
1411                                           ctx->sh_desc_update_first,
1412                                           ctx->sh_desc_update_first_dma,
1413                                           flags);
1414                 if (!edesc) {
1415                         dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1416                         return -ENOMEM;
1417                 }
1418
1419                 edesc->src_nents = src_nents;
1420                 edesc->dst_dma = 0;
1421
1422                 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
1423                                           to_hash);
1424                 if (ret)
1425                         goto unmap_ctx;
1426
1427                 if (*next_buflen)
1428                         scatterwalk_map_and_copy(next_buf, req->src, to_hash,
1429                                                  *next_buflen, 0);
1430
1431                 desc = edesc->hw_desc;
1432
1433                 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
1434                 if (ret)
1435                         goto unmap_ctx;
1436
1437 #ifdef DEBUG
1438                 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1439                                DUMP_PREFIX_ADDRESS, 16, 4, desc,
1440                                desc_bytes(desc), 1);
1441 #endif
1442
1443                 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
1444                 if (ret)
1445                         goto unmap_ctx;
1446
1447                 ret = -EINPROGRESS;
1448                 state->update = ahash_update_ctx;
1449                 state->finup = ahash_finup_ctx;
1450                 state->final = ahash_final_ctx;
1451         } else if (*next_buflen) {
1452                 state->update = ahash_update_no_ctx;
1453                 state->finup = ahash_finup_no_ctx;
1454                 state->final = ahash_final_no_ctx;
1455                 scatterwalk_map_and_copy(next_buf, req->src, 0,
1456                                          req->nbytes, 0);
1457                 switch_buf(state);
1458         }
1459 #ifdef DEBUG
1460         print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
1461                        DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
1462                        *next_buflen, 1);
1463 #endif
1464
1465         return ret;
1466  unmap_ctx:
1467         ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
1468         kfree(edesc);
1469         return ret;
1470 }
1471
1472 static int ahash_finup_first(struct ahash_request *req)
1473 {
1474         return ahash_digest(req);
1475 }
1476
1477 static int ahash_init(struct ahash_request *req)
1478 {
1479         struct caam_hash_state *state = ahash_request_ctx(req);
1480
1481         state->update = ahash_update_first;
1482         state->finup = ahash_finup_first;
1483         state->final = ahash_final_no_ctx;
1484
1485         state->ctx_dma = 0;
1486         state->current_buf = 0;
1487         state->buf_dma = 0;
1488         state->buflen_0 = 0;
1489         state->buflen_1 = 0;
1490
1491         return 0;
1492 }
1493
1494 static int ahash_update(struct ahash_request *req)
1495 {
1496         struct caam_hash_state *state = ahash_request_ctx(req);
1497
1498         return state->update(req);
1499 }
1500
1501 static int ahash_finup(struct ahash_request *req)
1502 {
1503         struct caam_hash_state *state = ahash_request_ctx(req);
1504
1505         return state->finup(req);
1506 }
1507
1508 static int ahash_final(struct ahash_request *req)
1509 {
1510         struct caam_hash_state *state = ahash_request_ctx(req);
1511
1512         return state->final(req);
1513 }
1514
1515 static int ahash_export(struct ahash_request *req, void *out)
1516 {
1517         struct caam_hash_state *state = ahash_request_ctx(req);
1518         struct caam_export_state *export = out;
1519         int len;
1520         u8 *buf;
1521
1522         if (state->current_buf) {
1523                 buf = state->buf_1;
1524                 len = state->buflen_1;
1525         } else {
1526                 buf = state->buf_0;
1527                 len = state->buflen_0;
1528         }
1529
1530         memcpy(export->buf, buf, len);
1531         memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
1532         export->buflen = len;
1533         export->update = state->update;
1534         export->final = state->final;
1535         export->finup = state->finup;
1536
1537         return 0;
1538 }
1539
1540 static int ahash_import(struct ahash_request *req, const void *in)
1541 {
1542         struct caam_hash_state *state = ahash_request_ctx(req);
1543         const struct caam_export_state *export = in;
1544
1545         memset(state, 0, sizeof(*state));
1546         memcpy(state->buf_0, export->buf, export->buflen);
1547         memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
1548         state->buflen_0 = export->buflen;
1549         state->update = export->update;
1550         state->final = export->final;
1551         state->finup = export->finup;
1552
1553         return 0;
1554 }
1555
1556 struct caam_hash_template {
1557         char name[CRYPTO_MAX_ALG_NAME];
1558         char driver_name[CRYPTO_MAX_ALG_NAME];
1559         char hmac_name[CRYPTO_MAX_ALG_NAME];
1560         char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
1561         unsigned int blocksize;
1562         struct ahash_alg template_ahash;
1563         u32 alg_type;
1564 };
1565
1566 /* ahash descriptors */
1567 static struct caam_hash_template driver_hash[] = {
1568         {
1569                 .name = "sha1",
1570                 .driver_name = "sha1-caam",
1571                 .hmac_name = "hmac(sha1)",
1572                 .hmac_driver_name = "hmac-sha1-caam",
1573                 .blocksize = SHA1_BLOCK_SIZE,
1574                 .template_ahash = {
1575                         .init = ahash_init,
1576                         .update = ahash_update,
1577                         .final = ahash_final,
1578                         .finup = ahash_finup,
1579                         .digest = ahash_digest,
1580                         .export = ahash_export,
1581                         .import = ahash_import,
1582                         .setkey = ahash_setkey,
1583                         .halg = {
1584                                 .digestsize = SHA1_DIGEST_SIZE,
1585                                 .statesize = sizeof(struct caam_export_state),
1586                         },
1587                 },
1588                 .alg_type = OP_ALG_ALGSEL_SHA1,
1589         }, {
1590                 .name = "sha224",
1591                 .driver_name = "sha224-caam",
1592                 .hmac_name = "hmac(sha224)",
1593                 .hmac_driver_name = "hmac-sha224-caam",
1594                 .blocksize = SHA224_BLOCK_SIZE,
1595                 .template_ahash = {
1596                         .init = ahash_init,
1597                         .update = ahash_update,
1598                         .final = ahash_final,
1599                         .finup = ahash_finup,
1600                         .digest = ahash_digest,
1601                         .export = ahash_export,
1602                         .import = ahash_import,
1603                         .setkey = ahash_setkey,
1604                         .halg = {
1605                                 .digestsize = SHA224_DIGEST_SIZE,
1606                                 .statesize = sizeof(struct caam_export_state),
1607                         },
1608                 },
1609                 .alg_type = OP_ALG_ALGSEL_SHA224,
1610         }, {
1611                 .name = "sha256",
1612                 .driver_name = "sha256-caam",
1613                 .hmac_name = "hmac(sha256)",
1614                 .hmac_driver_name = "hmac-sha256-caam",
1615                 .blocksize = SHA256_BLOCK_SIZE,
1616                 .template_ahash = {
1617                         .init = ahash_init,
1618                         .update = ahash_update,
1619                         .final = ahash_final,
1620                         .finup = ahash_finup,
1621                         .digest = ahash_digest,
1622                         .export = ahash_export,
1623                         .import = ahash_import,
1624                         .setkey = ahash_setkey,
1625                         .halg = {
1626                                 .digestsize = SHA256_DIGEST_SIZE,
1627                                 .statesize = sizeof(struct caam_export_state),
1628                         },
1629                 },
1630                 .alg_type = OP_ALG_ALGSEL_SHA256,
1631         }, {
1632                 .name = "sha384",
1633                 .driver_name = "sha384-caam",
1634                 .hmac_name = "hmac(sha384)",
1635                 .hmac_driver_name = "hmac-sha384-caam",
1636                 .blocksize = SHA384_BLOCK_SIZE,
1637                 .template_ahash = {
1638                         .init = ahash_init,
1639                         .update = ahash_update,
1640                         .final = ahash_final,
1641                         .finup = ahash_finup,
1642                         .digest = ahash_digest,
1643                         .export = ahash_export,
1644                         .import = ahash_import,
1645                         .setkey = ahash_setkey,
1646                         .halg = {
1647                                 .digestsize = SHA384_DIGEST_SIZE,
1648                                 .statesize = sizeof(struct caam_export_state),
1649                         },
1650                 },
1651                 .alg_type = OP_ALG_ALGSEL_SHA384,
1652         }, {
1653                 .name = "sha512",
1654                 .driver_name = "sha512-caam",
1655                 .hmac_name = "hmac(sha512)",
1656                 .hmac_driver_name = "hmac-sha512-caam",
1657                 .blocksize = SHA512_BLOCK_SIZE,
1658                 .template_ahash = {
1659                         .init = ahash_init,
1660                         .update = ahash_update,
1661                         .final = ahash_final,
1662                         .finup = ahash_finup,
1663                         .digest = ahash_digest,
1664                         .export = ahash_export,
1665                         .import = ahash_import,
1666                         .setkey = ahash_setkey,
1667                         .halg = {
1668                                 .digestsize = SHA512_DIGEST_SIZE,
1669                                 .statesize = sizeof(struct caam_export_state),
1670                         },
1671                 },
1672                 .alg_type = OP_ALG_ALGSEL_SHA512,
1673         }, {
1674                 .name = "md5",
1675                 .driver_name = "md5-caam",
1676                 .hmac_name = "hmac(md5)",
1677                 .hmac_driver_name = "hmac-md5-caam",
1678                 .blocksize = MD5_BLOCK_WORDS * 4,
1679                 .template_ahash = {
1680                         .init = ahash_init,
1681                         .update = ahash_update,
1682                         .final = ahash_final,
1683                         .finup = ahash_finup,
1684                         .digest = ahash_digest,
1685                         .export = ahash_export,
1686                         .import = ahash_import,
1687                         .setkey = ahash_setkey,
1688                         .halg = {
1689                                 .digestsize = MD5_DIGEST_SIZE,
1690                                 .statesize = sizeof(struct caam_export_state),
1691                         },
1692                 },
1693                 .alg_type = OP_ALG_ALGSEL_MD5,
1694         },
1695 };
1696
1697 struct caam_hash_alg {
1698         struct list_head entry;
1699         int alg_type;
1700         struct ahash_alg ahash_alg;
1701 };
1702
1703 static int caam_hash_cra_init(struct crypto_tfm *tfm)
1704 {
1705         struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
1706         struct crypto_alg *base = tfm->__crt_alg;
1707         struct hash_alg_common *halg =
1708                  container_of(base, struct hash_alg_common, base);
1709         struct ahash_alg *alg =
1710                  container_of(halg, struct ahash_alg, halg);
1711         struct caam_hash_alg *caam_hash =
1712                  container_of(alg, struct caam_hash_alg, ahash_alg);
1713         struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1714         /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
1715         static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
1716                                          HASH_MSG_LEN + SHA1_DIGEST_SIZE,
1717                                          HASH_MSG_LEN + 32,
1718                                          HASH_MSG_LEN + SHA256_DIGEST_SIZE,
1719                                          HASH_MSG_LEN + 64,
1720                                          HASH_MSG_LEN + SHA512_DIGEST_SIZE };
1721         dma_addr_t dma_addr;
1722
1723         /*
1724          * Get a Job ring from Job Ring driver to ensure in-order
1725          * crypto request processing per tfm
1726          */
1727         ctx->jrdev = caam_jr_alloc();
1728         if (IS_ERR(ctx->jrdev)) {
1729                 pr_err("Job Ring Device allocation for transform failed\n");
1730                 return PTR_ERR(ctx->jrdev);
1731         }
1732
1733         dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update,
1734                                         offsetof(struct caam_hash_ctx,
1735                                                  sh_desc_update_dma),
1736                                         DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
1737         if (dma_mapping_error(ctx->jrdev, dma_addr)) {
1738                 dev_err(ctx->jrdev, "unable to map shared descriptors\n");
1739                 caam_jr_free(ctx->jrdev);
1740                 return -ENOMEM;
1741         }
1742
1743         ctx->sh_desc_update_dma = dma_addr;
1744         ctx->sh_desc_update_first_dma = dma_addr +
1745                                         offsetof(struct caam_hash_ctx,
1746                                                  sh_desc_update_first);
1747         ctx->sh_desc_fin_dma = dma_addr + offsetof(struct caam_hash_ctx,
1748                                                    sh_desc_fin);
1749         ctx->sh_desc_digest_dma = dma_addr + offsetof(struct caam_hash_ctx,
1750                                                       sh_desc_digest);
1751
1752         /* copy descriptor header template value */
1753         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
1754
1755         ctx->ctx_len = runninglen[(ctx->adata.algtype &
1756                                    OP_ALG_ALGSEL_SUBMASK) >>
1757                                   OP_ALG_ALGSEL_SHIFT];
1758
1759         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1760                                  sizeof(struct caam_hash_state));
1761         return ahash_set_sh_desc(ahash);
1762 }
1763
1764 static void caam_hash_cra_exit(struct crypto_tfm *tfm)
1765 {
1766         struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1767
1768         dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma,
1769                                offsetof(struct caam_hash_ctx,
1770                                         sh_desc_update_dma),
1771                                DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
1772         caam_jr_free(ctx->jrdev);
1773 }
1774
1775 static void __exit caam_algapi_hash_exit(void)
1776 {
1777         struct caam_hash_alg *t_alg, *n;
1778
1779         if (!hash_list.next)
1780                 return;
1781
1782         list_for_each_entry_safe(t_alg, n, &hash_list, entry) {
1783                 crypto_unregister_ahash(&t_alg->ahash_alg);
1784                 list_del(&t_alg->entry);
1785                 kfree(t_alg);
1786         }
1787 }
1788
1789 static struct caam_hash_alg *
1790 caam_hash_alloc(struct caam_hash_template *template,
1791                 bool keyed)
1792 {
1793         struct caam_hash_alg *t_alg;
1794         struct ahash_alg *halg;
1795         struct crypto_alg *alg;
1796
1797         t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
1798         if (!t_alg) {
1799                 pr_err("failed to allocate t_alg\n");
1800                 return ERR_PTR(-ENOMEM);
1801         }
1802
1803         t_alg->ahash_alg = template->template_ahash;
1804         halg = &t_alg->ahash_alg;
1805         alg = &halg->halg.base;
1806
1807         if (keyed) {
1808                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1809                          template->hmac_name);
1810                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1811                          template->hmac_driver_name);
1812         } else {
1813                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1814                          template->name);
1815                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1816                          template->driver_name);
1817                 t_alg->ahash_alg.setkey = NULL;
1818         }
1819         alg->cra_module = THIS_MODULE;
1820         alg->cra_init = caam_hash_cra_init;
1821         alg->cra_exit = caam_hash_cra_exit;
1822         alg->cra_ctxsize = sizeof(struct caam_hash_ctx);
1823         alg->cra_priority = CAAM_CRA_PRIORITY;
1824         alg->cra_blocksize = template->blocksize;
1825         alg->cra_alignmask = 0;
1826         alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_TYPE_AHASH;
1827         alg->cra_type = &crypto_ahash_type;
1828
1829         t_alg->alg_type = template->alg_type;
1830
1831         return t_alg;
1832 }
1833
1834 static int __init caam_algapi_hash_init(void)
1835 {
1836         struct device_node *dev_node;
1837         struct platform_device *pdev;
1838         struct device *ctrldev;
1839         int i = 0, err = 0;
1840         struct caam_drv_private *priv;
1841         unsigned int md_limit = SHA512_DIGEST_SIZE;
1842         u32 cha_inst, cha_vid;
1843
1844         dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
1845         if (!dev_node) {
1846                 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
1847                 if (!dev_node)
1848                         return -ENODEV;
1849         }
1850
1851         pdev = of_find_device_by_node(dev_node);
1852         if (!pdev) {
1853                 of_node_put(dev_node);
1854                 return -ENODEV;
1855         }
1856
1857         ctrldev = &pdev->dev;
1858         priv = dev_get_drvdata(ctrldev);
1859         of_node_put(dev_node);
1860
1861         /*
1862          * If priv is NULL, it's probably because the caam driver wasn't
1863          * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
1864          */
1865         if (!priv)
1866                 return -ENODEV;
1867
1868         /*
1869          * Register crypto algorithms the device supports.  First, identify
1870          * presence and attributes of MD block.
1871          */
1872         cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
1873         cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
1874
1875         /*
1876          * Skip registration of any hashing algorithms if MD block
1877          * is not present.
1878          */
1879         if (!((cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT))
1880                 return -ENODEV;
1881
1882         /* Limit digest size based on LP256 */
1883         if ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)
1884                 md_limit = SHA256_DIGEST_SIZE;
1885
1886         INIT_LIST_HEAD(&hash_list);
1887
1888         /* register crypto algorithms the device supports */
1889         for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
1890                 struct caam_hash_alg *t_alg;
1891                 struct caam_hash_template *alg = driver_hash + i;
1892
1893                 /* If MD size is not supported by device, skip registration */
1894                 if (alg->template_ahash.halg.digestsize > md_limit)
1895                         continue;
1896
1897                 /* register hmac version */
1898                 t_alg = caam_hash_alloc(alg, true);
1899                 if (IS_ERR(t_alg)) {
1900                         err = PTR_ERR(t_alg);
1901                         pr_warn("%s alg allocation failed\n", alg->driver_name);
1902                         continue;
1903                 }
1904
1905                 err = crypto_register_ahash(&t_alg->ahash_alg);
1906                 if (err) {
1907                         pr_warn("%s alg registration failed: %d\n",
1908                                 t_alg->ahash_alg.halg.base.cra_driver_name,
1909                                 err);
1910                         kfree(t_alg);
1911                 } else
1912                         list_add_tail(&t_alg->entry, &hash_list);
1913
1914                 /* register unkeyed version */
1915                 t_alg = caam_hash_alloc(alg, false);
1916                 if (IS_ERR(t_alg)) {
1917                         err = PTR_ERR(t_alg);
1918                         pr_warn("%s alg allocation failed\n", alg->driver_name);
1919                         continue;
1920                 }
1921
1922                 err = crypto_register_ahash(&t_alg->ahash_alg);
1923                 if (err) {
1924                         pr_warn("%s alg registration failed: %d\n",
1925                                 t_alg->ahash_alg.halg.base.cra_driver_name,
1926                                 err);
1927                         kfree(t_alg);
1928                 } else
1929                         list_add_tail(&t_alg->entry, &hash_list);
1930         }
1931
1932         return err;
1933 }
1934
1935 module_init(caam_algapi_hash_init);
1936 module_exit(caam_algapi_hash_exit);
1937
1938 MODULE_LICENSE("GPL");
1939 MODULE_DESCRIPTION("FSL CAAM support for ahash functions of crypto API");
1940 MODULE_AUTHOR("Freescale Semiconductor - NMG");