drivers/crypto/ccree/cc_hash.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (C) 2012-2018 ARM Limited or its affiliates. */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <crypto/algapi.h>
   7 #include <crypto/hash.h>
   8 #include <crypto/md5.h>
   9 #include <crypto/internal/hash.h>
  10
  11 #include "cc_driver.h"
  12 #include "cc_request_mgr.h"
  13 #include "cc_buffer_mgr.h"
  14 #include "cc_hash.h"
  15 #include "cc_sram_mgr.h"
  16
  17 #define CC_MAX_HASH_SEQ_LEN 12
  18 #define CC_MAX_OPAD_KEYS_SIZE CC_MAX_HASH_BLCK_SIZE
  19
  20 struct cc_hash_handle {
  21         cc_sram_addr_t digest_len_sram_addr; /* const value in SRAM*/
  22         cc_sram_addr_t larval_digest_sram_addr;   /* const value in SRAM */
  23         struct list_head hash_list;
  24 };
  25
  26 static const u32 digest_len_init[] = {
  27         0x00000040, 0x00000000, 0x00000000, 0x00000000 };
  28 static const u32 md5_init[] = {
  29         SHA1_H3, SHA1_H2, SHA1_H1, SHA1_H0 };
  30 static const u32 sha1_init[] = {
  31         SHA1_H4, SHA1_H3, SHA1_H2, SHA1_H1, SHA1_H0 };
  32 static const u32 sha224_init[] = {
  33         SHA224_H7, SHA224_H6, SHA224_H5, SHA224_H4,
  34         SHA224_H3, SHA224_H2, SHA224_H1, SHA224_H0 };
  35 static const u32 sha256_init[] = {
  36         SHA256_H7, SHA256_H6, SHA256_H5, SHA256_H4,
  37         SHA256_H3, SHA256_H2, SHA256_H1, SHA256_H0 };
  38 static const u32 digest_len_sha512_init[] = {
  39         0x00000080, 0x00000000, 0x00000000, 0x00000000 };
  40 static u64 sha384_init[] = {
  41         SHA384_H7, SHA384_H6, SHA384_H5, SHA384_H4,
  42         SHA384_H3, SHA384_H2, SHA384_H1, SHA384_H0 };
  43 static u64 sha512_init[] = {
  44         SHA512_H7, SHA512_H6, SHA512_H5, SHA512_H4,
  45         SHA512_H3, SHA512_H2, SHA512_H1, SHA512_H0 };
  46
  47 static void cc_setup_xcbc(struct ahash_request *areq, struct cc_hw_desc desc[],
  48                           unsigned int *seq_size);
  49
  50 static void cc_setup_cmac(struct ahash_request *areq, struct cc_hw_desc desc[],
  51                           unsigned int *seq_size);
  52
  53 static const void *cc_larval_digest(struct device *dev, u32 mode);
  54
  55 struct cc_hash_alg {
  56         struct list_head entry;
  57         int hash_mode;
  58         int hw_mode;
  59         int inter_digestsize;
  60         struct cc_drvdata *drvdata;
  61         struct ahash_alg ahash_alg;
  62 };
  63
  64 struct hash_key_req_ctx {
  65         u32 keylen;
  66         dma_addr_t key_dma_addr;
  67 };
  68
  69 /* hash per-session context */
  70 struct cc_hash_ctx {
  71         struct cc_drvdata *drvdata;
  72         /* holds the origin digest; the digest after "setkey" if HMAC,*
  73          * the initial digest if HASH.
  74          */
  75         u8 digest_buff[CC_MAX_HASH_DIGEST_SIZE]  ____cacheline_aligned;
  76         u8 opad_tmp_keys_buff[CC_MAX_OPAD_KEYS_SIZE]  ____cacheline_aligned;
  77
  78         dma_addr_t opad_tmp_keys_dma_addr  ____cacheline_aligned;
  79         dma_addr_t digest_buff_dma_addr;
  80         /* use for hmac with key large then mode block size */
  81         struct hash_key_req_ctx key_params;
  82         int hash_mode;
  83         int hw_mode;
  84         int inter_digestsize;
  85         struct completion setkey_comp;
  86         bool is_hmac;
  87 };
  88
  89 static void cc_set_desc(struct ahash_req_ctx *areq_ctx, struct cc_hash_ctx *ctx,
  90                         unsigned int flow_mode, struct cc_hw_desc desc[],
  91                         bool is_not_last_data, unsigned int *seq_size);
  92
  93 static void cc_set_endianity(u32 mode, struct cc_hw_desc *desc)
  94 {
  95         if (mode == DRV_HASH_MD5 || mode == DRV_HASH_SHA384 ||
  96             mode == DRV_HASH_SHA512) {
  97                 set_bytes_swap(desc, 1);
  98         } else {
  99                 set_cipher_config0(desc, HASH_DIGEST_RESULT_LITTLE_ENDIAN);
 100         }
 101 }
 102
 103 static int cc_map_result(struct device *dev, struct ahash_req_ctx *state,
 104                          unsigned int digestsize)
 105 {
 106         state->digest_result_dma_addr =
 107                 dma_map_single(dev, state->digest_result_buff,
 108                                digestsize, DMA_BIDIRECTIONAL);
 109         if (dma_mapping_error(dev, state->digest_result_dma_addr)) {
 110                 dev_err(dev, "Mapping digest result buffer %u B for DMA failed\n",
 111                         digestsize);
 112                 return -ENOMEM;
 113         }
 114         dev_dbg(dev, "Mapped digest result buffer %u B at va=%pK to dma=%pad\n",
 115                 digestsize, state->digest_result_buff,
 116                 &state->digest_result_dma_addr);
 117
 118         return 0;
 119 }
 120
 121 static void cc_init_req(struct device *dev, struct ahash_req_ctx *state,
 122                         struct cc_hash_ctx *ctx)
 123 {
 124         bool is_hmac = ctx->is_hmac;
 125
 126         memset(state, 0, sizeof(*state));
 127
 128         if (is_hmac) {
 129                 if (ctx->hw_mode != DRV_CIPHER_XCBC_MAC &&
 130                     ctx->hw_mode != DRV_CIPHER_CMAC) {
 131                         dma_sync_single_for_cpu(dev, ctx->digest_buff_dma_addr,
 132                                                 ctx->inter_digestsize,
 133                                                 DMA_BIDIRECTIONAL);
 134
 135                         memcpy(state->digest_buff, ctx->digest_buff,
 136                                ctx->inter_digestsize);
 137                         if (ctx->hash_mode == DRV_HASH_SHA512 ||
 138                             ctx->hash_mode == DRV_HASH_SHA384)
 139                                 memcpy(state->digest_bytes_len,
 140                                        digest_len_sha512_init,
 141                                        ctx->drvdata->hash_len_sz);
 142                         else
 143                                 memcpy(state->digest_bytes_len, digest_len_init,
 144                                        ctx->drvdata->hash_len_sz);
 145                 }
 146
 147                 if (ctx->hash_mode != DRV_HASH_NULL) {
 148                         dma_sync_single_for_cpu(dev,
 149                                                 ctx->opad_tmp_keys_dma_addr,
 150                                                 ctx->inter_digestsize,
 151                                                 DMA_BIDIRECTIONAL);
 152                         memcpy(state->opad_digest_buff,
 153                                ctx->opad_tmp_keys_buff, ctx->inter_digestsize);
 154                 }
 155         } else { /*hash*/
 156                 /* Copy the initial digests if hash flow. */
 157                 const void *larval = cc_larval_digest(dev, ctx->hash_mode);
 158
 159                 memcpy(state->digest_buff, larval, ctx->inter_digestsize);
 160         }
 161 }
 162
 163 static int cc_map_req(struct device *dev, struct ahash_req_ctx *state,
 164                       struct cc_hash_ctx *ctx)
 165 {
 166         bool is_hmac = ctx->is_hmac;
 167
 168         state->digest_buff_dma_addr =
 169                 dma_map_single(dev, state->digest_buff,
 170                                ctx->inter_digestsize, DMA_BIDIRECTIONAL);
 171         if (dma_mapping_error(dev, state->digest_buff_dma_addr)) {
 172                 dev_err(dev, "Mapping digest len %d B at va=%pK for DMA failed\n",
 173                         ctx->inter_digestsize, state->digest_buff);
 174                 return -EINVAL;
 175         }
 176         dev_dbg(dev, "Mapped digest %d B at va=%pK to dma=%pad\n",
 177                 ctx->inter_digestsize, state->digest_buff,
 178                 &state->digest_buff_dma_addr);
 179
 180         if (ctx->hw_mode != DRV_CIPHER_XCBC_MAC) {
 181                 state->digest_bytes_len_dma_addr =
 182                         dma_map_single(dev, state->digest_bytes_len,
 183                                        HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL);
 184                 if (dma_mapping_error(dev, state->digest_bytes_len_dma_addr)) {
 185                         dev_err(dev, "Mapping digest len %u B at va=%pK for DMA failed\n",
 186                                 HASH_MAX_LEN_SIZE, state->digest_bytes_len);
 187                         goto unmap_digest_buf;
 188                 }
 189                 dev_dbg(dev, "Mapped digest len %u B at va=%pK to dma=%pad\n",
 190                         HASH_MAX_LEN_SIZE, state->digest_bytes_len,
 191                         &state->digest_bytes_len_dma_addr);
 192         }
 193
 194         if (is_hmac && ctx->hash_mode != DRV_HASH_NULL) {
 195                 state->opad_digest_dma_addr =
 196                         dma_map_single(dev, state->opad_digest_buff,
 197                                        ctx->inter_digestsize,
 198                                        DMA_BIDIRECTIONAL);
 199                 if (dma_mapping_error(dev, state->opad_digest_dma_addr)) {
 200                         dev_err(dev, "Mapping opad digest %d B at va=%pK for DMA failed\n",
 201                                 ctx->inter_digestsize,
 202                                 state->opad_digest_buff);
 203                         goto unmap_digest_len;
 204                 }
 205                 dev_dbg(dev, "Mapped opad digest %d B at va=%pK to dma=%pad\n",
 206                         ctx->inter_digestsize, state->opad_digest_buff,
 207                         &state->opad_digest_dma_addr);
 208         }
 209
 210         return 0;
 211
 212 unmap_digest_len:
 213         if (state->digest_bytes_len_dma_addr) {
 214                 dma_unmap_single(dev, state->digest_bytes_len_dma_addr,
 215                                  HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL);
 216                 state->digest_bytes_len_dma_addr = 0;
 217         }
 218 unmap_digest_buf:
 219         if (state->digest_buff_dma_addr) {
 220                 dma_unmap_single(dev, state->digest_buff_dma_addr,
 221                                  ctx->inter_digestsize, DMA_BIDIRECTIONAL);
 222                 state->digest_buff_dma_addr = 0;
 223         }
 224
 225         return -EINVAL;
 226 }
 227
 228 static void cc_unmap_req(struct device *dev, struct ahash_req_ctx *state,
 229                          struct cc_hash_ctx *ctx)
 230 {
 231         if (state->digest_buff_dma_addr) {
 232                 dma_unmap_single(dev, state->digest_buff_dma_addr,
 233                                  ctx->inter_digestsize, DMA_BIDIRECTIONAL);
 234                 dev_dbg(dev, "Unmapped digest-buffer: digest_buff_dma_addr=%pad\n",
 235                         &state->digest_buff_dma_addr);
 236                 state->digest_buff_dma_addr = 0;
 237         }
 238         if (state->digest_bytes_len_dma_addr) {
 239                 dma_unmap_single(dev, state->digest_bytes_len_dma_addr,
 240                                  HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL);
 241                 dev_dbg(dev, "Unmapped digest-bytes-len buffer: digest_bytes_len_dma_addr=%pad\n",
 242                         &state->digest_bytes_len_dma_addr);
 243                 state->digest_bytes_len_dma_addr = 0;
 244         }
 245         if (state->opad_digest_dma_addr) {
 246                 dma_unmap_single(dev, state->opad_digest_dma_addr,
 247                                  ctx->inter_digestsize, DMA_BIDIRECTIONAL);
 248                 dev_dbg(dev, "Unmapped opad-digest: opad_digest_dma_addr=%pad\n",
 249                         &state->opad_digest_dma_addr);
 250                 state->opad_digest_dma_addr = 0;
 251         }
 252 }
 253
 254 static void cc_unmap_result(struct device *dev, struct ahash_req_ctx *state,
 255                             unsigned int digestsize, u8 *result)
 256 {
 257         if (state->digest_result_dma_addr) {
 258                 dma_unmap_single(dev, state->digest_result_dma_addr, digestsize,
 259                                  DMA_BIDIRECTIONAL);
 260                 dev_dbg(dev, "unmpa digest result buffer va (%pK) pa (%pad) len %u\n",
 261                         state->digest_result_buff,
 262                         &state->digest_result_dma_addr, digestsize);
 263                 memcpy(result, state->digest_result_buff, digestsize);
 264         }
 265         state->digest_result_dma_addr = 0;
 266 }
 267
 268 static void cc_update_complete(struct device *dev, void *cc_req, int err)
 269 {
 270         struct ahash_request *req = (struct ahash_request *)cc_req;
 271         struct ahash_req_ctx *state = ahash_request_ctx(req);
 272         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 273         struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 274
 275         dev_dbg(dev, "req=%pK\n", req);
 276
 277         cc_unmap_hash_request(dev, state, req->src, false);
 278         cc_unmap_req(dev, state, ctx);
 279         req->base.complete(&req->base, err);
 280 }
 281
 282 static void cc_digest_complete(struct device *dev, void *cc_req, int err)
 283 {
 284         struct ahash_request *req = (struct ahash_request *)cc_req;
 285         struct ahash_req_ctx *state = ahash_request_ctx(req);
 286         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 287         struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 288         u32 digestsize = crypto_ahash_digestsize(tfm);
 289
 290         dev_dbg(dev, "req=%pK\n", req);
 291
 292         cc_unmap_hash_request(dev, state, req->src, false);
 293         cc_unmap_result(dev, state, digestsize, req->result);
 294         cc_unmap_req(dev, state, ctx);
 295         req->base.complete(&req->base, err);
 296 }
 297
 298 static void cc_hash_complete(struct device *dev, void *cc_req, int err)
 299 {
 300         struct ahash_request *req = (struct ahash_request *)cc_req;
 301         struct ahash_req_ctx *state = ahash_request_ctx(req);
 302         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 303         struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 304         u32 digestsize = crypto_ahash_digestsize(tfm);
 305
 306         dev_dbg(dev, "req=%pK\n", req);
 307
 308         cc_unmap_hash_request(dev, state, req->src, false);
 309         cc_unmap_result(dev, state, digestsize, req->result);
 310         cc_unmap_req(dev, state, ctx);
 311         req->base.complete(&req->base, err);
 312 }
 313
 314 static int cc_fin_result(struct cc_hw_desc *desc, struct ahash_request *req,
 315                          int idx)
 316 {
 317         struct ahash_req_ctx *state = ahash_request_ctx(req);
 318         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 319         struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 320         u32 digestsize = crypto_ahash_digestsize(tfm);
 321
 322         /* Get final MAC result */
 323         hw_desc_init(&desc[idx]);
 324         set_cipher_mode(&desc[idx], ctx->hw_mode);
 325         /* TODO */
 326         set_dout_dlli(&desc[idx], state->digest_result_dma_addr, digestsize,
 327                       NS_BIT, 1);
 328         set_queue_last_ind(ctx->drvdata, &desc[idx]);
 329         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 330         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
 331         set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
 332         cc_set_endianity(ctx->hash_mode, &desc[idx]);
 333         idx++;
 334
 335         return idx;
 336 }
 337
 338 static int cc_fin_hmac(struct cc_hw_desc *desc, struct ahash_request *req,
 339                        int idx)
 340 {
 341         struct ahash_req_ctx *state = ahash_request_ctx(req);
 342         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 343         struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 344         u32 digestsize = crypto_ahash_digestsize(tfm);
 345
 346         /* store the hash digest result in the context */
 347         hw_desc_init(&desc[idx]);
 348         set_cipher_mode(&desc[idx], ctx->hw_mode);
 349         set_dout_dlli(&desc[idx], state->digest_buff_dma_addr, digestsize,
 350                       NS_BIT, 0);
 351         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 352         cc_set_endianity(ctx->hash_mode, &desc[idx]);
 353         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
 354         idx++;
 355
 356         /* Loading hash opad xor key state */
 357         hw_desc_init(&desc[idx]);
 358         set_cipher_mode(&desc[idx], ctx->hw_mode);
 359         set_din_type(&desc[idx], DMA_DLLI, state->opad_digest_dma_addr,
 360                      ctx->inter_digestsize, NS_BIT);
 361         set_flow_mode(&desc[idx], S_DIN_to_HASH);
 362         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
 363         idx++;
 364
 365         /* Load the hash current length */
 366         hw_desc_init(&desc[idx]);
 367         set_cipher_mode(&desc[idx], ctx->hw_mode);
 368         set_din_sram(&desc[idx],
 369                      cc_digest_len_addr(ctx->drvdata, ctx->hash_mode),
 370                      ctx->drvdata->hash_len_sz);
 371         set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
 372         set_flow_mode(&desc[idx], S_DIN_to_HASH);
 373         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
 374         idx++;
 375
 376         /* Memory Barrier: wait for IPAD/OPAD axi write to complete */
 377         hw_desc_init(&desc[idx]);
 378         set_din_no_dma(&desc[idx], 0, 0xfffff0);
 379         set_dout_no_dma(&desc[idx], 0, 0, 1);
 380         idx++;
 381
 382         /* Perform HASH update */
 383         hw_desc_init(&desc[idx]);
 384         set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,
 385                      digestsize, NS_BIT);
 386         set_flow_mode(&desc[idx], DIN_HASH);
 387         idx++;
 388
 389         return idx;
 390 }
 391
 392 static int cc_hash_digest(struct ahash_request *req)
 393 {
 394         struct ahash_req_ctx *state = ahash_request_ctx(req);
 395         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 396         struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 397         u32 digestsize = crypto_ahash_digestsize(tfm);
 398         struct scatterlist *src = req->src;
 399         unsigned int nbytes = req->nbytes;
 400         u8 *result = req->result;
 401         struct device *dev = drvdata_to_dev(ctx->drvdata);
 402         bool is_hmac = ctx->is_hmac;
 403         struct cc_crypto_req cc_req = {};
 404         struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
 405         cc_sram_addr_t larval_digest_addr =
 406                 cc_larval_digest_addr(ctx->drvdata, ctx->hash_mode);
 407         int idx = 0;
 408         int rc = 0;
 409         gfp_t flags = cc_gfp_flags(&req->base);
 410
 411         dev_dbg(dev, "===== %s-digest (%d) ====\n", is_hmac ? "hmac" : "hash",
 412                 nbytes);
 413
 414         cc_init_req(dev, state, ctx);
 415
 416         if (cc_map_req(dev, state, ctx)) {
 417                 dev_err(dev, "map_ahash_source() failed\n");
 418                 return -ENOMEM;
 419         }
 420
 421         if (cc_map_result(dev, state, digestsize)) {
 422                 dev_err(dev, "map_ahash_digest() failed\n");
 423                 cc_unmap_req(dev, state, ctx);
 424                 return -ENOMEM;
 425         }
 426
 427         if (cc_map_hash_request_final(ctx->drvdata, state, src, nbytes, 1,
 428                                       flags)) {
 429                 dev_err(dev, "map_ahash_request_final() failed\n");
 430                 cc_unmap_result(dev, state, digestsize, result);
 431                 cc_unmap_req(dev, state, ctx);
 432                 return -ENOMEM;
 433         }
 434
 435         /* Setup request structure */
 436         cc_req.user_cb = cc_digest_complete;
 437         cc_req.user_arg = req;
 438
 439         /* If HMAC then load hash IPAD xor key, if HASH then load initial
 440          * digest
 441          */
 442         hw_desc_init(&desc[idx]);
 443         set_cipher_mode(&desc[idx], ctx->hw_mode);
 444         if (is_hmac) {
 445                 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,
 446                              ctx->inter_digestsize, NS_BIT);
 447         } else {
 448                 set_din_sram(&desc[idx], larval_digest_addr,
 449                              ctx->inter_digestsize);
 450         }
 451         set_flow_mode(&desc[idx], S_DIN_to_HASH);
 452         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
 453         idx++;
 454
 455         /* Load the hash current length */
 456         hw_desc_init(&desc[idx]);
 457         set_cipher_mode(&desc[idx], ctx->hw_mode);
 458
 459         if (is_hmac) {
 460                 set_din_type(&desc[idx], DMA_DLLI,
 461                              state->digest_bytes_len_dma_addr,
 462                              ctx->drvdata->hash_len_sz, NS_BIT);
 463         } else {
 464                 set_din_const(&desc[idx], 0, ctx->drvdata->hash_len_sz);
 465                 if (nbytes)
 466                         set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
 467                 else
 468                         set_cipher_do(&desc[idx], DO_PAD);
 469         }
 470         set_flow_mode(&desc[idx], S_DIN_to_HASH);
 471         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
 472         idx++;
 473
 474         cc_set_desc(state, ctx, DIN_HASH, desc, false, &idx);
 475
 476         if (is_hmac) {
 477                 /* HW last hash block padding (aka. "DO_PAD") */
 478                 hw_desc_init(&desc[idx]);
 479                 set_cipher_mode(&desc[idx], ctx->hw_mode);
 480                 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr,
 481                               ctx->drvdata->hash_len_sz, NS_BIT, 0);
 482                 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 483                 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);
 484                 set_cipher_do(&desc[idx], DO_PAD);
 485                 idx++;
 486
 487                 idx = cc_fin_hmac(desc, req, idx);
 488         }
 489
 490         idx = cc_fin_result(desc, req, idx);
 491
 492         rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
 493         if (rc != -EINPROGRESS && rc != -EBUSY) {
 494                 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
 495                 cc_unmap_hash_request(dev, state, src, true);
 496                 cc_unmap_result(dev, state, digestsize, result);
 497                 cc_unmap_req(dev, state, ctx);
 498         }
 499         return rc;
 500 }
 501
 502 static int cc_restore_hash(struct cc_hw_desc *desc, struct cc_hash_ctx *ctx,
 503                            struct ahash_req_ctx *state, unsigned int idx)
 504 {
 505         /* Restore hash digest */
 506         hw_desc_init(&desc[idx]);
 507         set_cipher_mode(&desc[idx], ctx->hw_mode);
 508         set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,
 509                      ctx->inter_digestsize, NS_BIT);
 510         set_flow_mode(&desc[idx], S_DIN_to_HASH);
 511         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
 512         idx++;
 513
 514         /* Restore hash current length */
 515         hw_desc_init(&desc[idx]);
 516         set_cipher_mode(&desc[idx], ctx->hw_mode);
 517         set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
 518         set_din_type(&desc[idx], DMA_DLLI, state->digest_bytes_len_dma_addr,
 519                      ctx->drvdata->hash_len_sz, NS_BIT);
 520         set_flow_mode(&desc[idx], S_DIN_to_HASH);
 521         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
 522         idx++;
 523
 524         cc_set_desc(state, ctx, DIN_HASH, desc, false, &idx);
 525
 526         return idx;
 527 }
 528
 529 static int cc_hash_update(struct ahash_request *req)
 530 {
 531         struct ahash_req_ctx *state = ahash_request_ctx(req);
 532         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 533         struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 534         unsigned int block_size = crypto_tfm_alg_blocksize(&tfm->base);
 535         struct scatterlist *src = req->src;
 536         unsigned int nbytes = req->nbytes;
 537         struct device *dev = drvdata_to_dev(ctx->drvdata);
 538         struct cc_crypto_req cc_req = {};
 539         struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
 540         u32 idx = 0;
 541         int rc;
 542         gfp_t flags = cc_gfp_flags(&req->base);
 543
 544         dev_dbg(dev, "===== %s-update (%d) ====\n", ctx->is_hmac ?
 545                 "hmac" : "hash", nbytes);
 546
 547         if (nbytes == 0) {
 548                 /* no real updates required */
 549                 return 0;
 550         }
 551
 552         rc = cc_map_hash_request_update(ctx->drvdata, state, src, nbytes,
 553                                         block_size, flags);
 554         if (rc) {
 555                 if (rc == 1) {
 556                         dev_dbg(dev, " data size not require HW update %x\n",
 557                                 nbytes);
 558                         /* No hardware updates are required */
 559                         return 0;
 560                 }
 561                 dev_err(dev, "map_ahash_request_update() failed\n");
 562                 return -ENOMEM;
 563         }
 564
 565         if (cc_map_req(dev, state, ctx)) {
 566                 dev_err(dev, "map_ahash_source() failed\n");
 567                 cc_unmap_hash_request(dev, state, src, true);
 568                 return -EINVAL;
 569         }
 570
 571         /* Setup request structure */
 572         cc_req.user_cb = cc_update_complete;
 573         cc_req.user_arg = req;
 574
 575         idx = cc_restore_hash(desc, ctx, state, idx);
 576
 577         /* store the hash digest result in context */
 578         hw_desc_init(&desc[idx]);
 579         set_cipher_mode(&desc[idx], ctx->hw_mode);
 580         set_dout_dlli(&desc[idx], state->digest_buff_dma_addr,
 581                       ctx->inter_digestsize, NS_BIT, 0);
 582         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 583         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
 584         idx++;
 585
 586         /* store current hash length in context */
 587         hw_desc_init(&desc[idx]);
 588         set_cipher_mode(&desc[idx], ctx->hw_mode);
 589         set_dout_dlli(&desc[idx], state->digest_bytes_len_dma_addr,
 590                       ctx->drvdata->hash_len_sz, NS_BIT, 1);
 591         set_queue_last_ind(ctx->drvdata, &desc[idx]);
 592         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 593         set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);
 594         idx++;
 595
 596         rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
 597         if (rc != -EINPROGRESS && rc != -EBUSY) {
 598                 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
 599                 cc_unmap_hash_request(dev, state, src, true);
 600                 cc_unmap_req(dev, state, ctx);
 601         }
 602         return rc;
 603 }
 604
 605 static int cc_hash_finup(struct ahash_request *req)
 606 {
 607         struct ahash_req_ctx *state = ahash_request_ctx(req);
 608         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 609         struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 610         u32 digestsize = crypto_ahash_digestsize(tfm);
 611         struct scatterlist *src = req->src;
 612         unsigned int nbytes = req->nbytes;
 613         u8 *result = req->result;
 614         struct device *dev = drvdata_to_dev(ctx->drvdata);
 615         bool is_hmac = ctx->is_hmac;
 616         struct cc_crypto_req cc_req = {};
 617         struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
 618         unsigned int idx = 0;
 619         int rc;
 620         gfp_t flags = cc_gfp_flags(&req->base);
 621
 622         dev_dbg(dev, "===== %s-finup (%d) ====\n", is_hmac ? "hmac" : "hash",
 623                 nbytes);
 624
 625         if (cc_map_req(dev, state, ctx)) {
 626                 dev_err(dev, "map_ahash_source() failed\n");
 627                 return -EINVAL;
 628         }
 629
 630         if (cc_map_hash_request_final(ctx->drvdata, state, src, nbytes, 1,
 631                                       flags)) {
 632                 dev_err(dev, "map_ahash_request_final() failed\n");
 633                 cc_unmap_req(dev, state, ctx);
 634                 return -ENOMEM;
 635         }
 636         if (cc_map_result(dev, state, digestsize)) {
 637                 dev_err(dev, "map_ahash_digest() failed\n");
 638                 cc_unmap_hash_request(dev, state, src, true);
 639                 cc_unmap_req(dev, state, ctx);
 640                 return -ENOMEM;
 641         }
 642
 643         /* Setup request structure */
 644         cc_req.user_cb = cc_hash_complete;
 645         cc_req.user_arg = req;
 646
 647         idx = cc_restore_hash(desc, ctx, state, idx);
 648
 649         if (is_hmac)
 650                 idx = cc_fin_hmac(desc, req, idx);
 651
 652         idx = cc_fin_result(desc, req, idx);
 653
 654         rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
 655         if (rc != -EINPROGRESS && rc != -EBUSY) {
 656                 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
 657                 cc_unmap_hash_request(dev, state, src, true);
 658                 cc_unmap_result(dev, state, digestsize, result);
 659                 cc_unmap_req(dev, state, ctx);
 660         }
 661         return rc;
 662 }
 663
 664 static int cc_hash_final(struct ahash_request *req)
 665 {
 666         struct ahash_req_ctx *state = ahash_request_ctx(req);
 667         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 668         struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 669         u32 digestsize = crypto_ahash_digestsize(tfm);
 670         struct scatterlist *src = req->src;
 671         unsigned int nbytes = req->nbytes;
 672         u8 *result = req->result;
 673         struct device *dev = drvdata_to_dev(ctx->drvdata);
 674         bool is_hmac = ctx->is_hmac;
 675         struct cc_crypto_req cc_req = {};
 676         struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
 677         unsigned int idx = 0;
 678         int rc;
 679         gfp_t flags = cc_gfp_flags(&req->base);
 680
 681         dev_dbg(dev, "===== %s-final (%d) ====\n", is_hmac ? "hmac" : "hash",
 682                 nbytes);
 683
 684         if (cc_map_req(dev, state, ctx)) {
 685                 dev_err(dev, "map_ahash_source() failed\n");
 686                 return -EINVAL;
 687         }
 688
 689         if (cc_map_hash_request_final(ctx->drvdata, state, src, nbytes, 0,
 690                                       flags)) {
 691                 dev_err(dev, "map_ahash_request_final() failed\n");
 692                 cc_unmap_req(dev, state, ctx);
 693                 return -ENOMEM;
 694         }
 695
 696         if (cc_map_result(dev, state, digestsize)) {
 697                 dev_err(dev, "map_ahash_digest() failed\n");
 698                 cc_unmap_hash_request(dev, state, src, true);
 699                 cc_unmap_req(dev, state, ctx);
 700                 return -ENOMEM;
 701         }
 702
 703         /* Setup request structure */
 704         cc_req.user_cb = cc_hash_complete;
 705         cc_req.user_arg = req;
 706
 707         idx = cc_restore_hash(desc, ctx, state, idx);
 708
 709         /* "DO-PAD" must be enabled only when writing current length to HW */
 710         hw_desc_init(&desc[idx]);
 711         set_cipher_do(&desc[idx], DO_PAD);
 712         set_cipher_mode(&desc[idx], ctx->hw_mode);
 713         set_dout_dlli(&desc[idx], state->digest_bytes_len_dma_addr,
 714                       ctx->drvdata->hash_len_sz, NS_BIT, 0);
 715         set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);
 716         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 717         idx++;
 718
 719         if (is_hmac)
 720                 idx = cc_fin_hmac(desc, req, idx);
 721
 722         idx = cc_fin_result(desc, req, idx);
 723
 724         rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
 725         if (rc != -EINPROGRESS && rc != -EBUSY) {
 726                 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
 727                 cc_unmap_hash_request(dev, state, src, true);
 728                 cc_unmap_result(dev, state, digestsize, result);
 729                 cc_unmap_req(dev, state, ctx);
 730         }
 731         return rc;
 732 }
 733
 734 static int cc_hash_init(struct ahash_request *req)
 735 {
 736         struct ahash_req_ctx *state = ahash_request_ctx(req);
 737         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 738         struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 739         struct device *dev = drvdata_to_dev(ctx->drvdata);
 740
 741         dev_dbg(dev, "===== init (%d) ====\n", req->nbytes);
 742
 743         cc_init_req(dev, state, ctx);
 744
 745         return 0;
 746 }
 747
 748 static int cc_hash_setkey(struct crypto_ahash *ahash, const u8 *key,
 749                           unsigned int keylen)
 750 {
 751         unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST };
 752         struct cc_crypto_req cc_req = {};
 753         struct cc_hash_ctx *ctx = NULL;
 754         int blocksize = 0;
 755         int digestsize = 0;
 756         int i, idx = 0, rc = 0;
 757         struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
 758         cc_sram_addr_t larval_addr;
 759         struct device *dev;
 760
 761         ctx = crypto_ahash_ctx(ahash);
 762         dev = drvdata_to_dev(ctx->drvdata);
 763         dev_dbg(dev, "start keylen: %d", keylen);
 764
 765         blocksize = crypto_tfm_alg_blocksize(&ahash->base);
 766         digestsize = crypto_ahash_digestsize(ahash);
 767
 768         larval_addr = cc_larval_digest_addr(ctx->drvdata, ctx->hash_mode);
 769
 770         /* The keylen value distinguishes HASH in case keylen is ZERO bytes,
 771          * any NON-ZERO value utilizes HMAC flow
 772          */
 773         ctx->key_params.keylen = keylen;
 774         ctx->key_params.key_dma_addr = 0;
 775         ctx->is_hmac = true;
 776
 777         if (keylen) {
 778                 ctx->key_params.key_dma_addr =
 779                         dma_map_single(dev, (void *)key, keylen, DMA_TO_DEVICE);
 780                 if (dma_mapping_error(dev, ctx->key_params.key_dma_addr)) {
 781                         dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n",
 782                                 key, keylen);
 783                         return -ENOMEM;
 784                 }
 785                 dev_dbg(dev, "mapping key-buffer: key_dma_addr=%pad keylen=%u\n",
 786                         &ctx->key_params.key_dma_addr, ctx->key_params.keylen);
 787
 788                 if (keylen > blocksize) {
 789                         /* Load hash initial state */
 790                         hw_desc_init(&desc[idx]);
 791                         set_cipher_mode(&desc[idx], ctx->hw_mode);
 792                         set_din_sram(&desc[idx], larval_addr,
 793                                      ctx->inter_digestsize);
 794                         set_flow_mode(&desc[idx], S_DIN_to_HASH);
 795                         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
 796                         idx++;
 797
 798                         /* Load the hash current length*/
 799                         hw_desc_init(&desc[idx]);
 800                         set_cipher_mode(&desc[idx], ctx->hw_mode);
 801                         set_din_const(&desc[idx], 0, ctx->drvdata->hash_len_sz);
 802                         set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
 803                         set_flow_mode(&desc[idx], S_DIN_to_HASH);
 804                         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
 805                         idx++;
 806
 807                         hw_desc_init(&desc[idx]);
 808                         set_din_type(&desc[idx], DMA_DLLI,
 809                                      ctx->key_params.key_dma_addr, keylen,
 810                                      NS_BIT);
 811                         set_flow_mode(&desc[idx], DIN_HASH);
 812                         idx++;
 813
 814                         /* Get hashed key */
 815                         hw_desc_init(&desc[idx]);
 816                         set_cipher_mode(&desc[idx], ctx->hw_mode);
 817                         set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr,
 818                                       digestsize, NS_BIT, 0);
 819                         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 820                         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
 821                         set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
 822                         cc_set_endianity(ctx->hash_mode, &desc[idx]);
 823                         idx++;
 824
 825                         hw_desc_init(&desc[idx]);
 826                         set_din_const(&desc[idx], 0, (blocksize - digestsize));
 827                         set_flow_mode(&desc[idx], BYPASS);
 828                         set_dout_dlli(&desc[idx],
 829                                       (ctx->opad_tmp_keys_dma_addr +
 830                                        digestsize),
 831                                       (blocksize - digestsize), NS_BIT, 0);
 832                         idx++;
 833                 } else {
 834                         hw_desc_init(&desc[idx]);
 835                         set_din_type(&desc[idx], DMA_DLLI,
 836                                      ctx->key_params.key_dma_addr, keylen,
 837                                      NS_BIT);
 838                         set_flow_mode(&desc[idx], BYPASS);
 839                         set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr,
 840                                       keylen, NS_BIT, 0);
 841                         idx++;
 842
 843                         if ((blocksize - keylen)) {
 844                                 hw_desc_init(&desc[idx]);
 845                                 set_din_const(&desc[idx], 0,
 846                                               (blocksize - keylen));
 847                                 set_flow_mode(&desc[idx], BYPASS);
 848                                 set_dout_dlli(&desc[idx],
 849                                               (ctx->opad_tmp_keys_dma_addr +
 850                                                keylen), (blocksize - keylen),
 851                                               NS_BIT, 0);
 852                                 idx++;
 853                         }
 854                 }
 855         } else {
 856                 hw_desc_init(&desc[idx]);
 857                 set_din_const(&desc[idx], 0, blocksize);
 858                 set_flow_mode(&desc[idx], BYPASS);
 859                 set_dout_dlli(&desc[idx], (ctx->opad_tmp_keys_dma_addr),
 860                               blocksize, NS_BIT, 0);
 861                 idx++;
 862         }
 863
 864         rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);
 865         if (rc) {
 866                 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
 867                 goto out;
 868         }
 869
 870         /* calc derived HMAC key */
 871         for (idx = 0, i = 0; i < 2; i++) {
 872                 /* Load hash initial state */
 873                 hw_desc_init(&desc[idx]);
 874                 set_cipher_mode(&desc[idx], ctx->hw_mode);
 875                 set_din_sram(&desc[idx], larval_addr, ctx->inter_digestsize);
 876                 set_flow_mode(&desc[idx], S_DIN_to_HASH);
 877                 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
 878                 idx++;
 879
 880                 /* Load the hash current length*/
 881                 hw_desc_init(&desc[idx]);
 882                 set_cipher_mode(&desc[idx], ctx->hw_mode);
 883                 set_din_const(&desc[idx], 0, ctx->drvdata->hash_len_sz);
 884                 set_flow_mode(&desc[idx], S_DIN_to_HASH);
 885                 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
 886                 idx++;
 887
 888                 /* Prepare ipad key */
 889                 hw_desc_init(&desc[idx]);
 890                 set_xor_val(&desc[idx], hmac_pad_const[i]);
 891                 set_cipher_mode(&desc[idx], ctx->hw_mode);
 892                 set_flow_mode(&desc[idx], S_DIN_to_HASH);
 893                 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
 894                 idx++;
 895
 896                 /* Perform HASH update */
 897                 hw_desc_init(&desc[idx]);
 898                 set_din_type(&desc[idx], DMA_DLLI, ctx->opad_tmp_keys_dma_addr,
 899                              blocksize, NS_BIT);
 900                 set_cipher_mode(&desc[idx], ctx->hw_mode);
 901                 set_xor_active(&desc[idx]);
 902                 set_flow_mode(&desc[idx], DIN_HASH);
 903                 idx++;
 904
 905                 /* Get the IPAD/OPAD xor key (Note, IPAD is the initial digest
 906                  * of the first HASH "update" state)
 907                  */
 908                 hw_desc_init(&desc[idx]);
 909                 set_cipher_mode(&desc[idx], ctx->hw_mode);
 910                 if (i > 0) /* Not first iteration */
 911                         set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr,
 912                                       ctx->inter_digestsize, NS_BIT, 0);
 913                 else /* First iteration */
 914                         set_dout_dlli(&desc[idx], ctx->digest_buff_dma_addr,
 915                                       ctx->inter_digestsize, NS_BIT, 0);
 916                 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 917                 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
 918                 idx++;
 919         }
 920
 921         rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);
 922
 923 out:
 924         if (rc)
 925                 crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
 926
 927         if (ctx->key_params.key_dma_addr) {
 928                 dma_unmap_single(dev, ctx->key_params.key_dma_addr,
 929                                  ctx->key_params.keylen, DMA_TO_DEVICE);
 930                 dev_dbg(dev, "Unmapped key-buffer: key_dma_addr=%pad keylen=%u\n",
 931                         &ctx->key_params.key_dma_addr, ctx->key_params.keylen);
 932         }
 933         return rc;
 934 }
 935
 936 static int cc_xcbc_setkey(struct crypto_ahash *ahash,
 937                           const u8 *key, unsigned int keylen)
 938 {
 939         struct cc_crypto_req cc_req = {};
 940         struct cc_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 941         struct device *dev = drvdata_to_dev(ctx->drvdata);
 942         int rc = 0;
 943         unsigned int idx = 0;
 944         struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
 945
 946         dev_dbg(dev, "===== setkey (%d) ====\n", keylen);
 947
 948         switch (keylen) {
 949         case AES_KEYSIZE_128:
 950         case AES_KEYSIZE_192:
 951         case AES_KEYSIZE_256:
 952                 break;
 953         default:
 954                 return -EINVAL;
 955         }
 956
 957         ctx->key_params.keylen = keylen;
 958
 959         ctx->key_params.key_dma_addr =
 960                 dma_map_single(dev, (void *)key, keylen, DMA_TO_DEVICE);
 961         if (dma_mapping_error(dev, ctx->key_params.key_dma_addr)) {
 962                 dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n",
 963                         key, keylen);
 964                 return -ENOMEM;
 965         }
 966         dev_dbg(dev, "mapping key-buffer: key_dma_addr=%pad keylen=%u\n",
 967                 &ctx->key_params.key_dma_addr, ctx->key_params.keylen);
 968
 969         ctx->is_hmac = true;
 970         /* 1. Load the AES key */
 971         hw_desc_init(&desc[idx]);
 972         set_din_type(&desc[idx], DMA_DLLI, ctx->key_params.key_dma_addr,
 973                      keylen, NS_BIT);
 974         set_cipher_mode(&desc[idx], DRV_CIPHER_ECB);
 975         set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
 976         set_key_size_aes(&desc[idx], keylen);
 977         set_flow_mode(&desc[idx], S_DIN_to_AES);
 978         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
 979         idx++;
 980
 981         hw_desc_init(&desc[idx]);
 982         set_din_const(&desc[idx], 0x01010101, CC_AES_128_BIT_KEY_SIZE);
 983         set_flow_mode(&desc[idx], DIN_AES_DOUT);
 984         set_dout_dlli(&desc[idx],
 985                       (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K1_OFFSET),
 986                       CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0);
 987         idx++;
 988
 989         hw_desc_init(&desc[idx]);
 990         set_din_const(&desc[idx], 0x02020202, CC_AES_128_BIT_KEY_SIZE);
 991         set_flow_mode(&desc[idx], DIN_AES_DOUT);
 992         set_dout_dlli(&desc[idx],
 993                       (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K2_OFFSET),
 994                       CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0);
 995         idx++;
 996
 997         hw_desc_init(&desc[idx]);
 998         set_din_const(&desc[idx], 0x03030303, CC_AES_128_BIT_KEY_SIZE);
 999         set_flow_mode(&desc[idx], DIN_AES_DOUT);
1000         set_dout_dlli(&desc[idx],
1001                       (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K3_OFFSET),
1002                       CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0);
1003         idx++;
1004
1005         rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);
1006
1007         if (rc)
1008                 crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
1009
1010         dma_unmap_single(dev, ctx->key_params.key_dma_addr,
1011                          ctx->key_params.keylen, DMA_TO_DEVICE);
1012         dev_dbg(dev, "Unmapped key-buffer: key_dma_addr=%pad keylen=%u\n",
1013                 &ctx->key_params.key_dma_addr, ctx->key_params.keylen);
1014
1015         return rc;
1016 }
1017
1018 static int cc_cmac_setkey(struct crypto_ahash *ahash,
1019                           const u8 *key, unsigned int keylen)
1020 {
1021         struct cc_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1022         struct device *dev = drvdata_to_dev(ctx->drvdata);
1023
1024         dev_dbg(dev, "===== setkey (%d) ====\n", keylen);
1025
1026         ctx->is_hmac = true;
1027
1028         switch (keylen) {
1029         case AES_KEYSIZE_128:
1030         case AES_KEYSIZE_192:
1031         case AES_KEYSIZE_256:
1032                 break;
1033         default:
1034                 return -EINVAL;
1035         }
1036
1037         ctx->key_params.keylen = keylen;
1038
1039         /* STAT_PHASE_1: Copy key to ctx */
1040
1041         dma_sync_single_for_cpu(dev, ctx->opad_tmp_keys_dma_addr,
1042                                 keylen, DMA_TO_DEVICE);
1043
1044         memcpy(ctx->opad_tmp_keys_buff, key, keylen);
1045         if (keylen == 24) {
1046                 memset(ctx->opad_tmp_keys_buff + 24, 0,
1047                        CC_AES_KEY_SIZE_MAX - 24);
1048         }
1049
1050         dma_sync_single_for_device(dev, ctx->opad_tmp_keys_dma_addr,
1051                                    keylen, DMA_TO_DEVICE);
1052
1053         ctx->key_params.keylen = keylen;
1054
1055         return 0;
1056 }
1057
1058 static void cc_free_ctx(struct cc_hash_ctx *ctx)
1059 {
1060         struct device *dev = drvdata_to_dev(ctx->drvdata);
1061
1062         if (ctx->digest_buff_dma_addr) {
1063                 dma_unmap_single(dev, ctx->digest_buff_dma_addr,
1064                                  sizeof(ctx->digest_buff), DMA_BIDIRECTIONAL);
1065                 dev_dbg(dev, "Unmapped digest-buffer: digest_buff_dma_addr=%pad\n",
1066                         &ctx->digest_buff_dma_addr);
1067                 ctx->digest_buff_dma_addr = 0;
1068         }
1069         if (ctx->opad_tmp_keys_dma_addr) {
1070                 dma_unmap_single(dev, ctx->opad_tmp_keys_dma_addr,
1071                                  sizeof(ctx->opad_tmp_keys_buff),
1072                                  DMA_BIDIRECTIONAL);
1073                 dev_dbg(dev, "Unmapped opad-digest: opad_tmp_keys_dma_addr=%pad\n",
1074                         &ctx->opad_tmp_keys_dma_addr);
1075                 ctx->opad_tmp_keys_dma_addr = 0;
1076         }
1077
1078         ctx->key_params.keylen = 0;
1079 }
1080
1081 static int cc_alloc_ctx(struct cc_hash_ctx *ctx)
1082 {
1083         struct device *dev = drvdata_to_dev(ctx->drvdata);
1084
1085         ctx->key_params.keylen = 0;
1086
1087         ctx->digest_buff_dma_addr =
1088                 dma_map_single(dev, (void *)ctx->digest_buff,
1089                                sizeof(ctx->digest_buff), DMA_BIDIRECTIONAL);
1090         if (dma_mapping_error(dev, ctx->digest_buff_dma_addr)) {
1091                 dev_err(dev, "Mapping digest len %zu B at va=%pK for DMA failed\n",
1092                         sizeof(ctx->digest_buff), ctx->digest_buff);
1093                 goto fail;
1094         }
1095         dev_dbg(dev, "Mapped digest %zu B at va=%pK to dma=%pad\n",
1096                 sizeof(ctx->digest_buff), ctx->digest_buff,
1097                 &ctx->digest_buff_dma_addr);
1098
1099         ctx->opad_tmp_keys_dma_addr =
1100                 dma_map_single(dev, (void *)ctx->opad_tmp_keys_buff,
1101                                sizeof(ctx->opad_tmp_keys_buff),
1102                                DMA_BIDIRECTIONAL);
1103         if (dma_mapping_error(dev, ctx->opad_tmp_keys_dma_addr)) {
1104                 dev_err(dev, "Mapping opad digest %zu B at va=%pK for DMA failed\n",
1105                         sizeof(ctx->opad_tmp_keys_buff),
1106                         ctx->opad_tmp_keys_buff);
1107                 goto fail;
1108         }
1109         dev_dbg(dev, "Mapped opad_tmp_keys %zu B at va=%pK to dma=%pad\n",
1110                 sizeof(ctx->opad_tmp_keys_buff), ctx->opad_tmp_keys_buff,
1111                 &ctx->opad_tmp_keys_dma_addr);
1112
1113         ctx->is_hmac = false;
1114         return 0;
1115
1116 fail:
1117         cc_free_ctx(ctx);
1118         return -ENOMEM;
1119 }
1120
1121 static int cc_cra_init(struct crypto_tfm *tfm)
1122 {
1123         struct cc_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1124         struct hash_alg_common *hash_alg_common =
1125                 container_of(tfm->__crt_alg, struct hash_alg_common, base);
1126         struct ahash_alg *ahash_alg =
1127                 container_of(hash_alg_common, struct ahash_alg, halg);
1128         struct cc_hash_alg *cc_alg =
1129                         container_of(ahash_alg, struct cc_hash_alg, ahash_alg);
1130
1131         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1132                                  sizeof(struct ahash_req_ctx));
1133
1134         ctx->hash_mode = cc_alg->hash_mode;
1135         ctx->hw_mode = cc_alg->hw_mode;
1136         ctx->inter_digestsize = cc_alg->inter_digestsize;
1137         ctx->drvdata = cc_alg->drvdata;
1138
1139         return cc_alloc_ctx(ctx);
1140 }
1141
1142 static void cc_cra_exit(struct crypto_tfm *tfm)
1143 {
1144         struct cc_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1145         struct device *dev = drvdata_to_dev(ctx->drvdata);
1146
1147         dev_dbg(dev, "cc_cra_exit");
1148         cc_free_ctx(ctx);
1149 }
1150
1151 static int cc_mac_update(struct ahash_request *req)
1152 {
1153         struct ahash_req_ctx *state = ahash_request_ctx(req);
1154         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1155         struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
1156         struct device *dev = drvdata_to_dev(ctx->drvdata);
1157         unsigned int block_size = crypto_tfm_alg_blocksize(&tfm->base);
1158         struct cc_crypto_req cc_req = {};
1159         struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
1160         int rc;
1161         u32 idx = 0;
1162         gfp_t flags = cc_gfp_flags(&req->base);
1163
1164         if (req->nbytes == 0) {
1165                 /* no real updates required */
1166                 return 0;
1167         }
1168
1169         state->xcbc_count++;
1170
1171         rc = cc_map_hash_request_update(ctx->drvdata, state, req->src,
1172                                         req->nbytes, block_size, flags);
1173         if (rc) {
1174                 if (rc == 1) {
1175                         dev_dbg(dev, " data size not require HW update %x\n",
1176                                 req->nbytes);
1177                         /* No hardware updates are required */
1178                         return 0;
1179                 }
1180                 dev_err(dev, "map_ahash_request_update() failed\n");
1181                 return -ENOMEM;
1182         }
1183
1184         if (cc_map_req(dev, state, ctx)) {
1185                 dev_err(dev, "map_ahash_source() failed\n");
1186                 return -EINVAL;
1187         }
1188
1189         if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC)
1190                 cc_setup_xcbc(req, desc, &idx);
1191         else
1192                 cc_setup_cmac(req, desc, &idx);
1193
1194         cc_set_desc(state, ctx, DIN_AES_DOUT, desc, true, &idx);
1195
1196         /* store the hash digest result in context */
1197         hw_desc_init(&desc[idx]);
1198         set_cipher_mode(&desc[idx], ctx->hw_mode);
1199         set_dout_dlli(&desc[idx], state->digest_buff_dma_addr,
1200                       ctx->inter_digestsize, NS_BIT, 1);
1201         set_queue_last_ind(ctx->drvdata, &desc[idx]);
1202         set_flow_mode(&desc[idx], S_AES_to_DOUT);
1203         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1204         idx++;
1205
1206         /* Setup request structure */
1207         cc_req.user_cb = (void *)cc_update_complete;
1208         cc_req.user_arg = (void *)req;
1209
1210         rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
1211         if (rc != -EINPROGRESS && rc != -EBUSY) {
1212                 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
1213                 cc_unmap_hash_request(dev, state, req->src, true);
1214                 cc_unmap_req(dev, state, ctx);
1215         }
1216         return rc;
1217 }
1218
1219 static int cc_mac_final(struct ahash_request *req)
1220 {
1221         struct ahash_req_ctx *state = ahash_request_ctx(req);
1222         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1223         struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
1224         struct device *dev = drvdata_to_dev(ctx->drvdata);
1225         struct cc_crypto_req cc_req = {};
1226         struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
1227         int idx = 0;
1228         int rc = 0;
1229         u32 key_size, key_len;
1230         u32 digestsize = crypto_ahash_digestsize(tfm);
1231         gfp_t flags = cc_gfp_flags(&req->base);
1232         u32 rem_cnt = *cc_hash_buf_cnt(state);
1233
1234         if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) {
1235                 key_size = CC_AES_128_BIT_KEY_SIZE;
1236                 key_len  = CC_AES_128_BIT_KEY_SIZE;
1237         } else {
1238                 key_size = (ctx->key_params.keylen == 24) ? AES_MAX_KEY_SIZE :
1239                         ctx->key_params.keylen;
1240                 key_len =  ctx->key_params.keylen;
1241         }
1242
1243         dev_dbg(dev, "===== final  xcbc reminder (%d) ====\n", rem_cnt);
1244
1245         if (cc_map_req(dev, state, ctx)) {
1246                 dev_err(dev, "map_ahash_source() failed\n");
1247                 return -EINVAL;
1248         }
1249
1250         if (cc_map_hash_request_final(ctx->drvdata, state, req->src,
1251                                       req->nbytes, 0, flags)) {
1252                 dev_err(dev, "map_ahash_request_final() failed\n");
1253                 cc_unmap_req(dev, state, ctx);
1254                 return -ENOMEM;
1255         }
1256
1257         if (cc_map_result(dev, state, digestsize)) {
1258                 dev_err(dev, "map_ahash_digest() failed\n");
1259                 cc_unmap_hash_request(dev, state, req->src, true);
1260                 cc_unmap_req(dev, state, ctx);
1261                 return -ENOMEM;
1262         }
1263
1264         /* Setup request structure */
1265         cc_req.user_cb = (void *)cc_hash_complete;
1266         cc_req.user_arg = (void *)req;
1267
1268         if (state->xcbc_count && rem_cnt == 0) {
1269                 /* Load key for ECB decryption */
1270                 hw_desc_init(&desc[idx]);
1271                 set_cipher_mode(&desc[idx], DRV_CIPHER_ECB);
1272                 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_DECRYPT);
1273                 set_din_type(&desc[idx], DMA_DLLI,
1274                              (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K1_OFFSET),
1275                              key_size, NS_BIT);
1276                 set_key_size_aes(&desc[idx], key_len);
1277                 set_flow_mode(&desc[idx], S_DIN_to_AES);
1278                 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1279                 idx++;
1280
1281                 /* Initiate decryption of block state to previous
1282                  * block_state-XOR-M[n]
1283                  */
1284                 hw_desc_init(&desc[idx]);
1285                 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,
1286                              CC_AES_BLOCK_SIZE, NS_BIT);
1287                 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr,
1288                               CC_AES_BLOCK_SIZE, NS_BIT, 0);
1289                 set_flow_mode(&desc[idx], DIN_AES_DOUT);
1290                 idx++;
1291
1292                 /* Memory Barrier: wait for axi write to complete */
1293                 hw_desc_init(&desc[idx]);
1294                 set_din_no_dma(&desc[idx], 0, 0xfffff0);
1295                 set_dout_no_dma(&desc[idx], 0, 0, 1);
1296                 idx++;
1297         }
1298
1299         if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC)
1300                 cc_setup_xcbc(req, desc, &idx);
1301         else
1302                 cc_setup_cmac(req, desc, &idx);
1303
1304         if (state->xcbc_count == 0) {
1305                 hw_desc_init(&desc[idx]);
1306                 set_cipher_mode(&desc[idx], ctx->hw_mode);
1307                 set_key_size_aes(&desc[idx], key_len);
1308                 set_cmac_size0_mode(&desc[idx]);
1309                 set_flow_mode(&desc[idx], S_DIN_to_AES);
1310                 idx++;
1311         } else if (rem_cnt > 0) {
1312                 cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx);
1313         } else {
1314                 hw_desc_init(&desc[idx]);
1315                 set_din_const(&desc[idx], 0x00, CC_AES_BLOCK_SIZE);
1316                 set_flow_mode(&desc[idx], DIN_AES_DOUT);
1317                 idx++;
1318         }
1319
1320         /* Get final MAC result */
1321         hw_desc_init(&desc[idx]);
1322         /* TODO */
1323         set_dout_dlli(&desc[idx], state->digest_result_dma_addr,
1324                       digestsize, NS_BIT, 1);
1325         set_queue_last_ind(ctx->drvdata, &desc[idx]);
1326         set_flow_mode(&desc[idx], S_AES_to_DOUT);
1327         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1328         set_cipher_mode(&desc[idx], ctx->hw_mode);
1329         idx++;
1330
1331         rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
1332         if (rc != -EINPROGRESS && rc != -EBUSY) {
1333                 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
1334                 cc_unmap_hash_request(dev, state, req->src, true);
1335                 cc_unmap_result(dev, state, digestsize, req->result);
1336                 cc_unmap_req(dev, state, ctx);
1337         }
1338         return rc;
1339 }
1340
1341 static int cc_mac_finup(struct ahash_request *req)
1342 {
1343         struct ahash_req_ctx *state = ahash_request_ctx(req);
1344         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1345         struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
1346         struct device *dev = drvdata_to_dev(ctx->drvdata);
1347         struct cc_crypto_req cc_req = {};
1348         struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
1349         int idx = 0;
1350         int rc = 0;
1351         u32 key_len = 0;
1352         u32 digestsize = crypto_ahash_digestsize(tfm);
1353         gfp_t flags = cc_gfp_flags(&req->base);
1354
1355         dev_dbg(dev, "===== finup xcbc(%d) ====\n", req->nbytes);
1356         if (state->xcbc_count > 0 && req->nbytes == 0) {
1357                 dev_dbg(dev, "No data to update. Call to fdx_mac_final\n");
1358                 return cc_mac_final(req);
1359         }
1360
1361         if (cc_map_req(dev, state, ctx)) {
1362                 dev_err(dev, "map_ahash_source() failed\n");
1363                 return -EINVAL;
1364         }
1365
1366         if (cc_map_hash_request_final(ctx->drvdata, state, req->src,
1367                                       req->nbytes, 1, flags)) {
1368                 dev_err(dev, "map_ahash_request_final() failed\n");
1369                 cc_unmap_req(dev, state, ctx);
1370                 return -ENOMEM;
1371         }
1372         if (cc_map_result(dev, state, digestsize)) {
1373                 dev_err(dev, "map_ahash_digest() failed\n");
1374                 cc_unmap_hash_request(dev, state, req->src, true);
1375                 cc_unmap_req(dev, state, ctx);
1376                 return -ENOMEM;
1377         }
1378
1379         /* Setup request structure */
1380         cc_req.user_cb = (void *)cc_hash_complete;
1381         cc_req.user_arg = (void *)req;
1382
1383         if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) {
1384                 key_len = CC_AES_128_BIT_KEY_SIZE;
1385                 cc_setup_xcbc(req, desc, &idx);
1386         } else {
1387                 key_len = ctx->key_params.keylen;
1388                 cc_setup_cmac(req, desc, &idx);
1389         }
1390
1391         if (req->nbytes == 0) {
1392                 hw_desc_init(&desc[idx]);
1393                 set_cipher_mode(&desc[idx], ctx->hw_mode);
1394                 set_key_size_aes(&desc[idx], key_len);
1395                 set_cmac_size0_mode(&desc[idx]);
1396                 set_flow_mode(&desc[idx], S_DIN_to_AES);
1397                 idx++;
1398         } else {
1399                 cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx);
1400         }
1401
1402         /* Get final MAC result */
1403         hw_desc_init(&desc[idx]);
1404         /* TODO */
1405         set_dout_dlli(&desc[idx], state->digest_result_dma_addr,
1406                       digestsize, NS_BIT, 1);
1407         set_queue_last_ind(ctx->drvdata, &desc[idx]);
1408         set_flow_mode(&desc[idx], S_AES_to_DOUT);
1409         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1410         set_cipher_mode(&desc[idx], ctx->hw_mode);
1411         idx++;
1412
1413         rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
1414         if (rc != -EINPROGRESS && rc != -EBUSY) {
1415                 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
1416                 cc_unmap_hash_request(dev, state, req->src, true);
1417                 cc_unmap_result(dev, state, digestsize, req->result);
1418                 cc_unmap_req(dev, state, ctx);
1419         }
1420         return rc;
1421 }
1422
1423 static int cc_mac_digest(struct ahash_request *req)
1424 {
1425         struct ahash_req_ctx *state = ahash_request_ctx(req);
1426         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1427         struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
1428         struct device *dev = drvdata_to_dev(ctx->drvdata);
1429         u32 digestsize = crypto_ahash_digestsize(tfm);
1430         struct cc_crypto_req cc_req = {};
1431         struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
1432         u32 key_len;
1433         unsigned int idx = 0;
1434         int rc;
1435         gfp_t flags = cc_gfp_flags(&req->base);
1436
1437         dev_dbg(dev, "===== -digest mac (%d) ====\n",  req->nbytes);
1438
1439         cc_init_req(dev, state, ctx);
1440
1441         if (cc_map_req(dev, state, ctx)) {
1442                 dev_err(dev, "map_ahash_source() failed\n");
1443                 return -ENOMEM;
1444         }
1445         if (cc_map_result(dev, state, digestsize)) {
1446                 dev_err(dev, "map_ahash_digest() failed\n");
1447                 cc_unmap_req(dev, state, ctx);
1448                 return -ENOMEM;
1449         }
1450
1451         if (cc_map_hash_request_final(ctx->drvdata, state, req->src,
1452                                       req->nbytes, 1, flags)) {
1453                 dev_err(dev, "map_ahash_request_final() failed\n");
1454                 cc_unmap_req(dev, state, ctx);
1455                 return -ENOMEM;
1456         }
1457
1458         /* Setup request structure */
1459         cc_req.user_cb = (void *)cc_digest_complete;
1460         cc_req.user_arg = (void *)req;
1461
1462         if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) {
1463                 key_len = CC_AES_128_BIT_KEY_SIZE;
1464                 cc_setup_xcbc(req, desc, &idx);
1465         } else {
1466                 key_len = ctx->key_params.keylen;
1467                 cc_setup_cmac(req, desc, &idx);
1468         }
1469
1470         if (req->nbytes == 0) {
1471                 hw_desc_init(&desc[idx]);
1472                 set_cipher_mode(&desc[idx], ctx->hw_mode);
1473                 set_key_size_aes(&desc[idx], key_len);
1474                 set_cmac_size0_mode(&desc[idx]);
1475                 set_flow_mode(&desc[idx], S_DIN_to_AES);
1476                 idx++;
1477         } else {
1478                 cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx);
1479         }
1480
1481         /* Get final MAC result */
1482         hw_desc_init(&desc[idx]);
1483         set_dout_dlli(&desc[idx], state->digest_result_dma_addr,
1484                       CC_AES_BLOCK_SIZE, NS_BIT, 1);
1485         set_queue_last_ind(ctx->drvdata, &desc[idx]);
1486         set_flow_mode(&desc[idx], S_AES_to_DOUT);
1487         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1488         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1489         set_cipher_mode(&desc[idx], ctx->hw_mode);
1490         idx++;
1491
1492         rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
1493         if (rc != -EINPROGRESS && rc != -EBUSY) {
1494                 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
1495                 cc_unmap_hash_request(dev, state, req->src, true);
1496                 cc_unmap_result(dev, state, digestsize, req->result);
1497                 cc_unmap_req(dev, state, ctx);
1498         }
1499         return rc;
1500 }
1501
1502 static int cc_hash_export(struct ahash_request *req, void *out)
1503 {
1504         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1505         struct cc_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1506         struct ahash_req_ctx *state = ahash_request_ctx(req);
1507         u8 *curr_buff = cc_hash_buf(state);
1508         u32 curr_buff_cnt = *cc_hash_buf_cnt(state);
1509         const u32 tmp = CC_EXPORT_MAGIC;
1510
1511         memcpy(out, &tmp, sizeof(u32));
1512         out += sizeof(u32);
1513
1514         memcpy(out, state->digest_buff, ctx->inter_digestsize);
1515         out += ctx->inter_digestsize;
1516
1517         memcpy(out, state->digest_bytes_len, ctx->drvdata->hash_len_sz);
1518         out += ctx->drvdata->hash_len_sz;
1519
1520         memcpy(out, &curr_buff_cnt, sizeof(u32));
1521         out += sizeof(u32);
1522
1523         memcpy(out, curr_buff, curr_buff_cnt);
1524
1525         return 0;
1526 }
1527
1528 static int cc_hash_import(struct ahash_request *req, const void *in)
1529 {
1530         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1531         struct cc_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1532         struct device *dev = drvdata_to_dev(ctx->drvdata);
1533         struct ahash_req_ctx *state = ahash_request_ctx(req);
1534         u32 tmp;
1535
1536         memcpy(&tmp, in, sizeof(u32));
1537         if (tmp != CC_EXPORT_MAGIC)
1538                 return -EINVAL;
1539         in += sizeof(u32);
1540
1541         cc_init_req(dev, state, ctx);
1542
1543         memcpy(state->digest_buff, in, ctx->inter_digestsize);
1544         in += ctx->inter_digestsize;
1545
1546         memcpy(state->digest_bytes_len, in, ctx->drvdata->hash_len_sz);
1547         in += ctx->drvdata->hash_len_sz;
1548
1549         /* Sanity check the data as much as possible */
1550         memcpy(&tmp, in, sizeof(u32));
1551         if (tmp > CC_MAX_HASH_BLCK_SIZE)
1552                 return -EINVAL;
1553         in += sizeof(u32);
1554
1555         state->buf_cnt[0] = tmp;
1556         memcpy(state->buffers[0], in, tmp);
1557
1558         return 0;
1559 }
1560
1561 struct cc_hash_template {
1562         char name[CRYPTO_MAX_ALG_NAME];
1563         char driver_name[CRYPTO_MAX_ALG_NAME];
1564         char mac_name[CRYPTO_MAX_ALG_NAME];
1565         char mac_driver_name[CRYPTO_MAX_ALG_NAME];
1566         unsigned int blocksize;
1567         bool synchronize;
1568         struct ahash_alg template_ahash;
1569         int hash_mode;
1570         int hw_mode;
1571         int inter_digestsize;
1572         struct cc_drvdata *drvdata;
1573         u32 min_hw_rev;
1574 };
1575
1576 #define CC_STATE_SIZE(_x) \
1577         ((_x) + HASH_MAX_LEN_SIZE + CC_MAX_HASH_BLCK_SIZE + (2 * sizeof(u32)))
1578
1579 /* hash descriptors */
1580 static struct cc_hash_template driver_hash[] = {
1581         //Asynchronize hash template
1582         {
1583                 .name = "sha1",
1584                 .driver_name = "sha1-ccree",
1585                 .mac_name = "hmac(sha1)",
1586                 .mac_driver_name = "hmac-sha1-ccree",
1587                 .blocksize = SHA1_BLOCK_SIZE,
1588                 .synchronize = false,
1589                 .template_ahash = {
1590                         .init = cc_hash_init,
1591                         .update = cc_hash_update,
1592                         .final = cc_hash_final,
1593                         .finup = cc_hash_finup,
1594                         .digest = cc_hash_digest,
1595                         .export = cc_hash_export,
1596                         .import = cc_hash_import,
1597                         .setkey = cc_hash_setkey,
1598                         .halg = {
1599                                 .digestsize = SHA1_DIGEST_SIZE,
1600                                 .statesize = CC_STATE_SIZE(SHA1_DIGEST_SIZE),
1601                         },
1602                 },
1603                 .hash_mode = DRV_HASH_SHA1,
1604                 .hw_mode = DRV_HASH_HW_SHA1,
1605                 .inter_digestsize = SHA1_DIGEST_SIZE,
1606                 .min_hw_rev = CC_HW_REV_630,
1607         },
1608         {
1609                 .name = "sha256",
1610                 .driver_name = "sha256-ccree",
1611                 .mac_name = "hmac(sha256)",
1612                 .mac_driver_name = "hmac-sha256-ccree",
1613                 .blocksize = SHA256_BLOCK_SIZE,
1614                 .template_ahash = {
1615                         .init = cc_hash_init,
1616                         .update = cc_hash_update,
1617                         .final = cc_hash_final,
1618                         .finup = cc_hash_finup,
1619                         .digest = cc_hash_digest,
1620                         .export = cc_hash_export,
1621                         .import = cc_hash_import,
1622                         .setkey = cc_hash_setkey,
1623                         .halg = {
1624                                 .digestsize = SHA256_DIGEST_SIZE,
1625                                 .statesize = CC_STATE_SIZE(SHA256_DIGEST_SIZE)
1626                         },
1627                 },
1628                 .hash_mode = DRV_HASH_SHA256,
1629                 .hw_mode = DRV_HASH_HW_SHA256,
1630                 .inter_digestsize = SHA256_DIGEST_SIZE,
1631                 .min_hw_rev = CC_HW_REV_630,
1632         },
1633         {
1634                 .name = "sha224",
1635                 .driver_name = "sha224-ccree",
1636                 .mac_name = "hmac(sha224)",
1637                 .mac_driver_name = "hmac-sha224-ccree",
1638                 .blocksize = SHA224_BLOCK_SIZE,
1639                 .template_ahash = {
1640                         .init = cc_hash_init,
1641                         .update = cc_hash_update,
1642                         .final = cc_hash_final,
1643                         .finup = cc_hash_finup,
1644                         .digest = cc_hash_digest,
1645                         .export = cc_hash_export,
1646                         .import = cc_hash_import,
1647                         .setkey = cc_hash_setkey,
1648                         .halg = {
1649                                 .digestsize = SHA224_DIGEST_SIZE,
1650                                 .statesize = CC_STATE_SIZE(SHA224_DIGEST_SIZE),
1651                         },
1652                 },
1653                 .hash_mode = DRV_HASH_SHA224,
1654                 .hw_mode = DRV_HASH_HW_SHA256,
1655                 .inter_digestsize = SHA256_DIGEST_SIZE,
1656                 .min_hw_rev = CC_HW_REV_630,
1657         },
1658         {
1659                 .name = "sha384",
1660                 .driver_name = "sha384-ccree",
1661                 .mac_name = "hmac(sha384)",
1662                 .mac_driver_name = "hmac-sha384-ccree",
1663                 .blocksize = SHA384_BLOCK_SIZE,
1664                 .template_ahash = {
1665                         .init = cc_hash_init,
1666                         .update = cc_hash_update,
1667                         .final = cc_hash_final,
1668                         .finup = cc_hash_finup,
1669                         .digest = cc_hash_digest,
1670                         .export = cc_hash_export,
1671                         .import = cc_hash_import,
1672                         .setkey = cc_hash_setkey,
1673                         .halg = {
1674                                 .digestsize = SHA384_DIGEST_SIZE,
1675                                 .statesize = CC_STATE_SIZE(SHA384_DIGEST_SIZE),
1676                         },
1677                 },
1678                 .hash_mode = DRV_HASH_SHA384,
1679                 .hw_mode = DRV_HASH_HW_SHA512,
1680                 .inter_digestsize = SHA512_DIGEST_SIZE,
1681                 .min_hw_rev = CC_HW_REV_712,
1682         },
1683         {
1684                 .name = "sha512",
1685                 .driver_name = "sha512-ccree",
1686                 .mac_name = "hmac(sha512)",
1687                 .mac_driver_name = "hmac-sha512-ccree",
1688                 .blocksize = SHA512_BLOCK_SIZE,
1689                 .template_ahash = {
1690                         .init = cc_hash_init,
1691                         .update = cc_hash_update,
1692                         .final = cc_hash_final,
1693                         .finup = cc_hash_finup,
1694                         .digest = cc_hash_digest,
1695                         .export = cc_hash_export,
1696                         .import = cc_hash_import,
1697                         .setkey = cc_hash_setkey,
1698                         .halg = {
1699                                 .digestsize = SHA512_DIGEST_SIZE,
1700                                 .statesize = CC_STATE_SIZE(SHA512_DIGEST_SIZE),
1701                         },
1702                 },
1703                 .hash_mode = DRV_HASH_SHA512,
1704                 .hw_mode = DRV_HASH_HW_SHA512,
1705                 .inter_digestsize = SHA512_DIGEST_SIZE,
1706                 .min_hw_rev = CC_HW_REV_712,
1707         },
1708         {
1709                 .name = "md5",
1710                 .driver_name = "md5-ccree",
1711                 .mac_name = "hmac(md5)",
1712                 .mac_driver_name = "hmac-md5-ccree",
1713                 .blocksize = MD5_HMAC_BLOCK_SIZE,
1714                 .template_ahash = {
1715                         .init = cc_hash_init,
1716                         .update = cc_hash_update,
1717                         .final = cc_hash_final,
1718                         .finup = cc_hash_finup,
1719                         .digest = cc_hash_digest,
1720                         .export = cc_hash_export,
1721                         .import = cc_hash_import,
1722                         .setkey = cc_hash_setkey,
1723                         .halg = {
1724                                 .digestsize = MD5_DIGEST_SIZE,
1725                                 .statesize = CC_STATE_SIZE(MD5_DIGEST_SIZE),
1726                         },
1727                 },
1728                 .hash_mode = DRV_HASH_MD5,
1729                 .hw_mode = DRV_HASH_HW_MD5,
1730                 .inter_digestsize = MD5_DIGEST_SIZE,
1731                 .min_hw_rev = CC_HW_REV_630,
1732         },
1733         {
1734                 .mac_name = "xcbc(aes)",
1735                 .mac_driver_name = "xcbc-aes-ccree",
1736                 .blocksize = AES_BLOCK_SIZE,
1737                 .template_ahash = {
1738                         .init = cc_hash_init,
1739                         .update = cc_mac_update,
1740                         .final = cc_mac_final,
1741                         .finup = cc_mac_finup,
1742                         .digest = cc_mac_digest,
1743                         .setkey = cc_xcbc_setkey,
1744                         .export = cc_hash_export,
1745                         .import = cc_hash_import,
1746                         .halg = {
1747                                 .digestsize = AES_BLOCK_SIZE,
1748                                 .statesize = CC_STATE_SIZE(AES_BLOCK_SIZE),
1749                         },
1750                 },
1751                 .hash_mode = DRV_HASH_NULL,
1752                 .hw_mode = DRV_CIPHER_XCBC_MAC,
1753                 .inter_digestsize = AES_BLOCK_SIZE,
1754                 .min_hw_rev = CC_HW_REV_630,
1755         },
1756         {
1757                 .mac_name = "cmac(aes)",
1758                 .mac_driver_name = "cmac-aes-ccree",
1759                 .blocksize = AES_BLOCK_SIZE,
1760                 .template_ahash = {
1761                         .init = cc_hash_init,
1762                         .update = cc_mac_update,
1763                         .final = cc_mac_final,
1764                         .finup = cc_mac_finup,
1765                         .digest = cc_mac_digest,
1766                         .setkey = cc_cmac_setkey,
1767                         .export = cc_hash_export,
1768                         .import = cc_hash_import,
1769                         .halg = {
1770                                 .digestsize = AES_BLOCK_SIZE,
1771                                 .statesize = CC_STATE_SIZE(AES_BLOCK_SIZE),
1772                         },
1773                 },
1774                 .hash_mode = DRV_HASH_NULL,
1775                 .hw_mode = DRV_CIPHER_CMAC,
1776                 .inter_digestsize = AES_BLOCK_SIZE,
1777                 .min_hw_rev = CC_HW_REV_630,
1778         },
1779 };
1780
1781 static struct cc_hash_alg *cc_alloc_hash_alg(struct cc_hash_template *template,
1782                                              struct device *dev, bool keyed)
1783 {
1784         struct cc_hash_alg *t_crypto_alg;
1785         struct crypto_alg *alg;
1786         struct ahash_alg *halg;
1787
1788         t_crypto_alg = kzalloc(sizeof(*t_crypto_alg), GFP_KERNEL);
1789         if (!t_crypto_alg)
1790                 return ERR_PTR(-ENOMEM);
1791
1792         t_crypto_alg->ahash_alg = template->template_ahash;
1793         halg = &t_crypto_alg->ahash_alg;
1794         alg = &halg->halg.base;
1795
1796         if (keyed) {
1797                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1798                          template->mac_name);
1799                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1800                          template->mac_driver_name);
1801         } else {
1802                 halg->setkey = NULL;
1803                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1804                          template->name);
1805                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1806                          template->driver_name);
1807         }
1808         alg->cra_module = THIS_MODULE;
1809         alg->cra_ctxsize = sizeof(struct cc_hash_ctx);
1810         alg->cra_priority = CC_CRA_PRIO;
1811         alg->cra_blocksize = template->blocksize;
1812         alg->cra_alignmask = 0;
1813         alg->cra_exit = cc_cra_exit;
1814
1815         alg->cra_init = cc_cra_init;
1816         alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_TYPE_AHASH |
1817                         CRYPTO_ALG_KERN_DRIVER_ONLY;
1818         alg->cra_type = &crypto_ahash_type;
1819
1820         t_crypto_alg->hash_mode = template->hash_mode;
1821         t_crypto_alg->hw_mode = template->hw_mode;
1822         t_crypto_alg->inter_digestsize = template->inter_digestsize;
1823
1824         return t_crypto_alg;
1825 }
1826
1827 int cc_init_hash_sram(struct cc_drvdata *drvdata)
1828 {
1829         struct cc_hash_handle *hash_handle = drvdata->hash_handle;
1830         cc_sram_addr_t sram_buff_ofs = hash_handle->digest_len_sram_addr;
1831         unsigned int larval_seq_len = 0;
1832         struct cc_hw_desc larval_seq[CC_DIGEST_SIZE_MAX / sizeof(u32)];
1833         bool large_sha_supported = (drvdata->hw_rev >= CC_HW_REV_712);
1834         int rc = 0;
1835
1836         /* Copy-to-sram digest-len */
1837         cc_set_sram_desc(digest_len_init, sram_buff_ofs,
1838                          ARRAY_SIZE(digest_len_init), larval_seq,
1839                          &larval_seq_len);
1840         rc = send_request_init(drvdata, larval_seq, larval_seq_len);
1841         if (rc)
1842                 goto init_digest_const_err;
1843
1844         sram_buff_ofs += sizeof(digest_len_init);
1845         larval_seq_len = 0;
1846
1847         if (large_sha_supported) {
1848                 /* Copy-to-sram digest-len for sha384/512 */
1849                 cc_set_sram_desc(digest_len_sha512_init, sram_buff_ofs,
1850                                  ARRAY_SIZE(digest_len_sha512_init),
1851                                  larval_seq, &larval_seq_len);
1852                 rc = send_request_init(drvdata, larval_seq, larval_seq_len);
1853                 if (rc)
1854                         goto init_digest_const_err;
1855
1856                 sram_buff_ofs += sizeof(digest_len_sha512_init);
1857                 larval_seq_len = 0;
1858         }
1859
1860         /* The initial digests offset */
1861         hash_handle->larval_digest_sram_addr = sram_buff_ofs;
1862
1863         /* Copy-to-sram initial SHA* digests */
1864         cc_set_sram_desc(md5_init, sram_buff_ofs, ARRAY_SIZE(md5_init),
1865                          larval_seq, &larval_seq_len);
1866         rc = send_request_init(drvdata, larval_seq, larval_seq_len);
1867         if (rc)
1868                 goto init_digest_const_err;
1869         sram_buff_ofs += sizeof(md5_init);
1870         larval_seq_len = 0;
1871
1872         cc_set_sram_desc(sha1_init, sram_buff_ofs,
1873                          ARRAY_SIZE(sha1_init), larval_seq,
1874                          &larval_seq_len);
1875         rc = send_request_init(drvdata, larval_seq, larval_seq_len);
1876         if (rc)
1877                 goto init_digest_const_err;
1878         sram_buff_ofs += sizeof(sha1_init);
1879         larval_seq_len = 0;
1880
1881         cc_set_sram_desc(sha224_init, sram_buff_ofs,
1882                          ARRAY_SIZE(sha224_init), larval_seq,
1883                          &larval_seq_len);
1884         rc = send_request_init(drvdata, larval_seq, larval_seq_len);
1885         if (rc)
1886                 goto init_digest_const_err;
1887         sram_buff_ofs += sizeof(sha224_init);
1888         larval_seq_len = 0;
1889
1890         cc_set_sram_desc(sha256_init, sram_buff_ofs,
1891                          ARRAY_SIZE(sha256_init), larval_seq,
1892                          &larval_seq_len);
1893         rc = send_request_init(drvdata, larval_seq, larval_seq_len);
1894         if (rc)
1895                 goto init_digest_const_err;
1896         sram_buff_ofs += sizeof(sha256_init);
1897         larval_seq_len = 0;
1898
1899         if (large_sha_supported) {
1900                 cc_set_sram_desc((u32 *)sha384_init, sram_buff_ofs,
1901                                  (ARRAY_SIZE(sha384_init) * 2), larval_seq,
1902                                  &larval_seq_len);
1903                 rc = send_request_init(drvdata, larval_seq, larval_seq_len);
1904                 if (rc)
1905                         goto init_digest_const_err;
1906                 sram_buff_ofs += sizeof(sha384_init);
1907                 larval_seq_len = 0;
1908
1909                 cc_set_sram_desc((u32 *)sha512_init, sram_buff_ofs,
1910                                  (ARRAY_SIZE(sha512_init) * 2), larval_seq,
1911                                  &larval_seq_len);
1912                 rc = send_request_init(drvdata, larval_seq, larval_seq_len);
1913                 if (rc)
1914                         goto init_digest_const_err;
1915         }
1916
1917 init_digest_const_err:
1918         return rc;
1919 }
1920
1921 static void __init cc_swap_dwords(u32 *buf, unsigned long size)
1922 {
1923         int i;
1924         u32 tmp;
1925
1926         for (i = 0; i < size; i += 2) {
1927                 tmp = buf[i];
1928                 buf[i] = buf[i + 1];
1929                 buf[i + 1] = tmp;
1930         }
1931 }
1932
1933 /*
1934  * Due to the way the HW works we need to swap every
1935  * double word in the SHA384 and SHA512 larval hashes
1936  */
1937 void __init cc_hash_global_init(void)
1938 {
1939         cc_swap_dwords((u32 *)&sha384_init, (ARRAY_SIZE(sha384_init) * 2));
1940         cc_swap_dwords((u32 *)&sha512_init, (ARRAY_SIZE(sha512_init) * 2));
1941 }
1942
1943 int cc_hash_alloc(struct cc_drvdata *drvdata)
1944 {
1945         struct cc_hash_handle *hash_handle;
1946         cc_sram_addr_t sram_buff;
1947         u32 sram_size_to_alloc;
1948         struct device *dev = drvdata_to_dev(drvdata);
1949         int rc = 0;
1950         int alg;
1951
1952         hash_handle = kzalloc(sizeof(*hash_handle), GFP_KERNEL);
1953         if (!hash_handle)
1954                 return -ENOMEM;
1955
1956         INIT_LIST_HEAD(&hash_handle->hash_list);
1957         drvdata->hash_handle = hash_handle;
1958
1959         sram_size_to_alloc = sizeof(digest_len_init) +
1960                         sizeof(md5_init) +
1961                         sizeof(sha1_init) +
1962                         sizeof(sha224_init) +
1963                         sizeof(sha256_init);
1964
1965         if (drvdata->hw_rev >= CC_HW_REV_712)
1966                 sram_size_to_alloc += sizeof(digest_len_sha512_init) +
1967                         sizeof(sha384_init) + sizeof(sha512_init);
1968
1969         sram_buff = cc_sram_alloc(drvdata, sram_size_to_alloc);
1970         if (sram_buff == NULL_SRAM_ADDR) {
1971                 dev_err(dev, "SRAM pool exhausted\n");
1972                 rc = -ENOMEM;
1973                 goto fail;
1974         }
1975
1976         /* The initial digest-len offset */
1977         hash_handle->digest_len_sram_addr = sram_buff;
1978
1979         /*must be set before the alg registration as it is being used there*/
1980         rc = cc_init_hash_sram(drvdata);
1981         if (rc) {
1982                 dev_err(dev, "Init digest CONST failed (rc=%d)\n", rc);
1983                 goto fail;
1984         }
1985
1986         /* ahash registration */
1987         for (alg = 0; alg < ARRAY_SIZE(driver_hash); alg++) {
1988                 struct cc_hash_alg *t_alg;
1989                 int hw_mode = driver_hash[alg].hw_mode;
1990
1991                 /* We either support both HASH and MAC or none */
1992                 if (driver_hash[alg].min_hw_rev > drvdata->hw_rev)
1993                         continue;
1994
1995                 /* register hmac version */
1996                 t_alg = cc_alloc_hash_alg(&driver_hash[alg], dev, true);
1997                 if (IS_ERR(t_alg)) {
1998                         rc = PTR_ERR(t_alg);
1999                         dev_err(dev, "%s alg allocation failed\n",
2000                                 driver_hash[alg].driver_name);
2001                         goto fail;
2002                 }
2003                 t_alg->drvdata = drvdata;
2004
2005                 rc = crypto_register_ahash(&t_alg->ahash_alg);
2006                 if (rc) {
2007                         dev_err(dev, "%s alg registration failed\n",
2008                                 driver_hash[alg].driver_name);
2009                         kfree(t_alg);
2010                         goto fail;
2011                 } else {
2012                         list_add_tail(&t_alg->entry, &hash_handle->hash_list);
2013                 }
2014
2015                 if (hw_mode == DRV_CIPHER_XCBC_MAC ||
2016                     hw_mode == DRV_CIPHER_CMAC)
2017                         continue;
2018
2019                 /* register hash version */
2020                 t_alg = cc_alloc_hash_alg(&driver_hash[alg], dev, false);
2021                 if (IS_ERR(t_alg)) {
2022                         rc = PTR_ERR(t_alg);
2023                         dev_err(dev, "%s alg allocation failed\n",
2024                                 driver_hash[alg].driver_name);
2025                         goto fail;
2026                 }
2027                 t_alg->drvdata = drvdata;
2028
2029                 rc = crypto_register_ahash(&t_alg->ahash_alg);
2030                 if (rc) {
2031                         dev_err(dev, "%s alg registration failed\n",
2032                                 driver_hash[alg].driver_name);
2033                         kfree(t_alg);
2034                         goto fail;
2035                 } else {
2036                         list_add_tail(&t_alg->entry, &hash_handle->hash_list);
2037                 }
2038         }
2039
2040         return 0;
2041
2042 fail:
2043         kfree(drvdata->hash_handle);
2044         drvdata->hash_handle = NULL;
2045         return rc;
2046 }
2047
2048 int cc_hash_free(struct cc_drvdata *drvdata)
2049 {
2050         struct cc_hash_alg *t_hash_alg, *hash_n;
2051         struct cc_hash_handle *hash_handle = drvdata->hash_handle;
2052
2053         if (hash_handle) {
2054                 list_for_each_entry_safe(t_hash_alg, hash_n,
2055                                          &hash_handle->hash_list, entry) {
2056                         crypto_unregister_ahash(&t_hash_alg->ahash_alg);
2057                         list_del(&t_hash_alg->entry);
2058                         kfree(t_hash_alg);
2059                 }
2060
2061                 kfree(hash_handle);
2062                 drvdata->hash_handle = NULL;
2063         }
2064         return 0;
2065 }
2066
2067 static void cc_setup_xcbc(struct ahash_request *areq, struct cc_hw_desc desc[],
2068                           unsigned int *seq_size)
2069 {
2070         unsigned int idx = *seq_size;
2071         struct ahash_req_ctx *state = ahash_request_ctx(areq);
2072         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2073         struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
2074
2075         /* Setup XCBC MAC K1 */
2076         hw_desc_init(&desc[idx]);
2077         set_din_type(&desc[idx], DMA_DLLI, (ctx->opad_tmp_keys_dma_addr +
2078                                             XCBC_MAC_K1_OFFSET),
2079                      CC_AES_128_BIT_KEY_SIZE, NS_BIT);
2080         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
2081         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
2082         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
2083         set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
2084         set_flow_mode(&desc[idx], S_DIN_to_AES);
2085         idx++;
2086
2087         /* Setup XCBC MAC K2 */
2088         hw_desc_init(&desc[idx]);
2089         set_din_type(&desc[idx], DMA_DLLI,
2090                      (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K2_OFFSET),
2091                      CC_AES_128_BIT_KEY_SIZE, NS_BIT);
2092         set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
2093         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
2094         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
2095         set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
2096         set_flow_mode(&desc[idx], S_DIN_to_AES);
2097         idx++;
2098
2099         /* Setup XCBC MAC K3 */
2100         hw_desc_init(&desc[idx]);
2101         set_din_type(&desc[idx], DMA_DLLI,
2102                      (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K3_OFFSET),
2103                      CC_AES_128_BIT_KEY_SIZE, NS_BIT);
2104         set_setup_mode(&desc[idx], SETUP_LOAD_STATE2);
2105         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
2106         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
2107         set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
2108         set_flow_mode(&desc[idx], S_DIN_to_AES);
2109         idx++;
2110
2111         /* Loading MAC state */
2112         hw_desc_init(&desc[idx]);
2113         set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,
2114                      CC_AES_BLOCK_SIZE, NS_BIT);
2115         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
2116         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
2117         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
2118         set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
2119         set_flow_mode(&desc[idx], S_DIN_to_AES);
2120         idx++;
2121         *seq_size = idx;
2122 }
2123
2124 static void cc_setup_cmac(struct ahash_request *areq, struct cc_hw_desc desc[],
2125                           unsigned int *seq_size)
2126 {
2127         unsigned int idx = *seq_size;
2128         struct ahash_req_ctx *state = ahash_request_ctx(areq);
2129         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2130         struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
2131
2132         /* Setup CMAC Key */
2133         hw_desc_init(&desc[idx]);
2134         set_din_type(&desc[idx], DMA_DLLI, ctx->opad_tmp_keys_dma_addr,
2135                      ((ctx->key_params.keylen == 24) ? AES_MAX_KEY_SIZE :
2136                       ctx->key_params.keylen), NS_BIT);
2137         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
2138         set_cipher_mode(&desc[idx], DRV_CIPHER_CMAC);
2139         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
2140         set_key_size_aes(&desc[idx], ctx->key_params.keylen);
2141         set_flow_mode(&desc[idx], S_DIN_to_AES);
2142         idx++;
2143
2144         /* Load MAC state */
2145         hw_desc_init(&desc[idx]);
2146         set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,
2147                      CC_AES_BLOCK_SIZE, NS_BIT);
2148         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
2149         set_cipher_mode(&desc[idx], DRV_CIPHER_CMAC);
2150         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
2151         set_key_size_aes(&desc[idx], ctx->key_params.keylen);
2152         set_flow_mode(&desc[idx], S_DIN_to_AES);
2153         idx++;
2154         *seq_size = idx;
2155 }
2156
2157 static void cc_set_desc(struct ahash_req_ctx *areq_ctx,
2158                         struct cc_hash_ctx *ctx, unsigned int flow_mode,
2159                         struct cc_hw_desc desc[], bool is_not_last_data,
2160                         unsigned int *seq_size)
2161 {
2162         unsigned int idx = *seq_size;
2163         struct device *dev = drvdata_to_dev(ctx->drvdata);
2164
2165         if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_DLLI) {
2166                 hw_desc_init(&desc[idx]);
2167                 set_din_type(&desc[idx], DMA_DLLI,
2168                              sg_dma_address(areq_ctx->curr_sg),
2169                              areq_ctx->curr_sg->length, NS_BIT);
2170                 set_flow_mode(&desc[idx], flow_mode);
2171                 idx++;
2172         } else {
2173                 if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_NULL) {
2174                         dev_dbg(dev, " NULL mode\n");
2175                         /* nothing to build */
2176                         return;
2177                 }
2178                 /* bypass */
2179                 hw_desc_init(&desc[idx]);
2180                 set_din_type(&desc[idx], DMA_DLLI,
2181                              areq_ctx->mlli_params.mlli_dma_addr,
2182                              areq_ctx->mlli_params.mlli_len, NS_BIT);
2183                 set_dout_sram(&desc[idx], ctx->drvdata->mlli_sram_addr,
2184                               areq_ctx->mlli_params.mlli_len);
2185                 set_flow_mode(&desc[idx], BYPASS);
2186                 idx++;
2187                 /* process */
2188                 hw_desc_init(&desc[idx]);
2189                 set_din_type(&desc[idx], DMA_MLLI,
2190                              ctx->drvdata->mlli_sram_addr,
2191                              areq_ctx->mlli_nents, NS_BIT);
2192                 set_flow_mode(&desc[idx], flow_mode);
2193                 idx++;
2194         }
2195         if (is_not_last_data)
2196                 set_din_not_last_indication(&desc[(idx - 1)]);
2197         /* return updated desc sequence size */
2198         *seq_size = idx;
2199 }
2200
2201 static const void *cc_larval_digest(struct device *dev, u32 mode)
2202 {
2203         switch (mode) {
2204         case DRV_HASH_MD5:
2205                 return md5_init;
2206         case DRV_HASH_SHA1:
2207                 return sha1_init;
2208         case DRV_HASH_SHA224:
2209                 return sha224_init;
2210         case DRV_HASH_SHA256:
2211                 return sha256_init;
2212         case DRV_HASH_SHA384:
2213                 return sha384_init;
2214         case DRV_HASH_SHA512:
2215                 return sha512_init;
2216         default:
2217                 dev_err(dev, "Invalid hash mode (%d)\n", mode);
2218                 return md5_init;
2219         }
2220 }
2221
2222 /*!
2223  * Gets the address of the initial digest in SRAM
2224  * according to the given hash mode
2225  *
2226  * \param drvdata
2227  * \param mode The Hash mode. Supported modes: MD5/SHA1/SHA224/SHA256
2228  *
2229  * \return u32 The address of the initial digest in SRAM
2230  */
2231 cc_sram_addr_t cc_larval_digest_addr(void *drvdata, u32 mode)
2232 {
2233         struct cc_drvdata *_drvdata = (struct cc_drvdata *)drvdata;
2234         struct cc_hash_handle *hash_handle = _drvdata->hash_handle;
2235         struct device *dev = drvdata_to_dev(_drvdata);
2236
2237         switch (mode) {
2238         case DRV_HASH_NULL:
2239                 break; /*Ignore*/
2240         case DRV_HASH_MD5:
2241                 return (hash_handle->larval_digest_sram_addr);
2242         case DRV_HASH_SHA1:
2243                 return (hash_handle->larval_digest_sram_addr +
2244                         sizeof(md5_init));
2245         case DRV_HASH_SHA224:
2246                 return (hash_handle->larval_digest_sram_addr +
2247                         sizeof(md5_init) +
2248                         sizeof(sha1_init));
2249         case DRV_HASH_SHA256:
2250                 return (hash_handle->larval_digest_sram_addr +
2251                         sizeof(md5_init) +
2252                         sizeof(sha1_init) +
2253                         sizeof(sha224_init));
2254         case DRV_HASH_SHA384:
2255                 return (hash_handle->larval_digest_sram_addr +
2256                         sizeof(md5_init) +
2257                         sizeof(sha1_init) +
2258                         sizeof(sha224_init) +
2259                         sizeof(sha256_init));
2260         case DRV_HASH_SHA512:
2261                 return (hash_handle->larval_digest_sram_addr +
2262                         sizeof(md5_init) +
2263                         sizeof(sha1_init) +
2264                         sizeof(sha224_init) +
2265                         sizeof(sha256_init) +
2266                         sizeof(sha384_init));
2267         default:
2268                 dev_err(dev, "Invalid hash mode (%d)\n", mode);
2269         }
2270
2271         /*This is valid wrong value to avoid kernel crash*/
2272         return hash_handle->larval_digest_sram_addr;
2273 }
2274
2275 cc_sram_addr_t
2276 cc_digest_len_addr(void *drvdata, u32 mode)
2277 {
2278         struct cc_drvdata *_drvdata = (struct cc_drvdata *)drvdata;
2279         struct cc_hash_handle *hash_handle = _drvdata->hash_handle;
2280         cc_sram_addr_t digest_len_addr = hash_handle->digest_len_sram_addr;
2281
2282         switch (mode) {
2283         case DRV_HASH_SHA1:
2284         case DRV_HASH_SHA224:
2285         case DRV_HASH_SHA256:
2286         case DRV_HASH_MD5:
2287                 return digest_len_addr;
2288 #if (CC_DEV_SHA_MAX > 256)
2289         case DRV_HASH_SHA384:
2290         case DRV_HASH_SHA512:
2291                 return  digest_len_addr + sizeof(digest_len_init);
2292 #endif
2293         default:
2294                 return digest_len_addr; /*to avoid kernel crash*/
2295         }
2296 }