drivers/crypto/ccree/cc_aead.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <crypto/algapi.h>
   7 #include <crypto/internal/aead.h>
   8 #include <crypto/authenc.h>
   9 #include <crypto/des.h>
  10 #include <linux/rtnetlink.h>
  11 #include "cc_driver.h"
  12 #include "cc_buffer_mgr.h"
  13 #include "cc_aead.h"
  14 #include "cc_request_mgr.h"
  15 #include "cc_hash.h"
  16 #include "cc_sram_mgr.h"
  17
  18 #define template_aead   template_u.aead
  19
  20 #define MAX_AEAD_SETKEY_SEQ 12
  21 #define MAX_AEAD_PROCESS_SEQ 23
  22
  23 #define MAX_HMAC_DIGEST_SIZE (SHA256_DIGEST_SIZE)
  24 #define MAX_HMAC_BLOCK_SIZE (SHA256_BLOCK_SIZE)
  25
  26 #define MAX_NONCE_SIZE CTR_RFC3686_NONCE_SIZE
  27
  28 struct cc_aead_handle {
  29         cc_sram_addr_t sram_workspace_addr;
  30         struct list_head aead_list;
  31 };
  32
  33 struct cc_hmac_s {
  34         u8 *padded_authkey;
  35         u8 *ipad_opad; /* IPAD, OPAD*/
  36         dma_addr_t padded_authkey_dma_addr;
  37         dma_addr_t ipad_opad_dma_addr;
  38 };
  39
  40 struct cc_xcbc_s {
  41         u8 *xcbc_keys; /* K1,K2,K3 */
  42         dma_addr_t xcbc_keys_dma_addr;
  43 };
  44
  45 struct cc_aead_ctx {
  46         struct cc_drvdata *drvdata;
  47         u8 ctr_nonce[MAX_NONCE_SIZE]; /* used for ctr3686 iv and aes ccm */
  48         u8 *enckey;
  49         dma_addr_t enckey_dma_addr;
  50         union {
  51                 struct cc_hmac_s hmac;
  52                 struct cc_xcbc_s xcbc;
  53         } auth_state;
  54         unsigned int enc_keylen;
  55         unsigned int auth_keylen;
  56         unsigned int authsize; /* Actual (reduced?) size of the MAC/ICv */
  57         unsigned int hash_len;
  58         enum drv_cipher_mode cipher_mode;
  59         enum cc_flow_mode flow_mode;
  60         enum drv_hash_mode auth_mode;
  61 };
  62
  63 static inline bool valid_assoclen(struct aead_request *req)
  64 {
  65         return ((req->assoclen == 16) || (req->assoclen == 20));
  66 }
  67
  68 static void cc_aead_exit(struct crypto_aead *tfm)
  69 {
  70         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
  71         struct device *dev = drvdata_to_dev(ctx->drvdata);
  72
  73         dev_dbg(dev, "Clearing context @%p for %s\n", crypto_aead_ctx(tfm),
  74                 crypto_tfm_alg_name(&tfm->base));
  75
  76         /* Unmap enckey buffer */
  77         if (ctx->enckey) {
  78                 dma_free_coherent(dev, AES_MAX_KEY_SIZE, ctx->enckey,
  79                                   ctx->enckey_dma_addr);
  80                 dev_dbg(dev, "Freed enckey DMA buffer enckey_dma_addr=%pad\n",
  81                         &ctx->enckey_dma_addr);
  82                 ctx->enckey_dma_addr = 0;
  83                 ctx->enckey = NULL;
  84         }
  85
  86         if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */
  87                 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc;
  88
  89                 if (xcbc->xcbc_keys) {
  90                         dma_free_coherent(dev, CC_AES_128_BIT_KEY_SIZE * 3,
  91                                           xcbc->xcbc_keys,
  92                                           xcbc->xcbc_keys_dma_addr);
  93                 }
  94                 dev_dbg(dev, "Freed xcbc_keys DMA buffer xcbc_keys_dma_addr=%pad\n",
  95                         &xcbc->xcbc_keys_dma_addr);
  96                 xcbc->xcbc_keys_dma_addr = 0;
  97                 xcbc->xcbc_keys = NULL;
  98         } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC auth. */
  99                 struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
 100
 101                 if (hmac->ipad_opad) {
 102                         dma_free_coherent(dev, 2 * MAX_HMAC_DIGEST_SIZE,
 103                                           hmac->ipad_opad,
 104                                           hmac->ipad_opad_dma_addr);
 105                         dev_dbg(dev, "Freed ipad_opad DMA buffer ipad_opad_dma_addr=%pad\n",
 106                                 &hmac->ipad_opad_dma_addr);
 107                         hmac->ipad_opad_dma_addr = 0;
 108                         hmac->ipad_opad = NULL;
 109                 }
 110                 if (hmac->padded_authkey) {
 111                         dma_free_coherent(dev, MAX_HMAC_BLOCK_SIZE,
 112                                           hmac->padded_authkey,
 113                                           hmac->padded_authkey_dma_addr);
 114                         dev_dbg(dev, "Freed padded_authkey DMA buffer padded_authkey_dma_addr=%pad\n",
 115                                 &hmac->padded_authkey_dma_addr);
 116                         hmac->padded_authkey_dma_addr = 0;
 117                         hmac->padded_authkey = NULL;
 118                 }
 119         }
 120 }
 121
 122 static unsigned int cc_get_aead_hash_len(struct crypto_aead *tfm)
 123 {
 124         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 125
 126         return cc_get_default_hash_len(ctx->drvdata);
 127 }
 128
 129 static int cc_aead_init(struct crypto_aead *tfm)
 130 {
 131         struct aead_alg *alg = crypto_aead_alg(tfm);
 132         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 133         struct cc_crypto_alg *cc_alg =
 134                         container_of(alg, struct cc_crypto_alg, aead_alg);
 135         struct device *dev = drvdata_to_dev(cc_alg->drvdata);
 136
 137         dev_dbg(dev, "Initializing context @%p for %s\n", ctx,
 138                 crypto_tfm_alg_name(&tfm->base));
 139
 140         /* Initialize modes in instance */
 141         ctx->cipher_mode = cc_alg->cipher_mode;
 142         ctx->flow_mode = cc_alg->flow_mode;
 143         ctx->auth_mode = cc_alg->auth_mode;
 144         ctx->drvdata = cc_alg->drvdata;
 145         crypto_aead_set_reqsize(tfm, sizeof(struct aead_req_ctx));
 146
 147         /* Allocate key buffer, cache line aligned */
 148         ctx->enckey = dma_alloc_coherent(dev, AES_MAX_KEY_SIZE,
 149                                          &ctx->enckey_dma_addr, GFP_KERNEL);
 150         if (!ctx->enckey) {
 151                 dev_err(dev, "Failed allocating key buffer\n");
 152                 goto init_failed;
 153         }
 154         dev_dbg(dev, "Allocated enckey buffer in context ctx->enckey=@%p\n",
 155                 ctx->enckey);
 156
 157         /* Set default authlen value */
 158
 159         if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */
 160                 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc;
 161                 const unsigned int key_size = CC_AES_128_BIT_KEY_SIZE * 3;
 162
 163                 /* Allocate dma-coherent buffer for XCBC's K1+K2+K3 */
 164                 /* (and temporary for user key - up to 256b) */
 165                 xcbc->xcbc_keys = dma_alloc_coherent(dev, key_size,
 166                                                      &xcbc->xcbc_keys_dma_addr,
 167                                                      GFP_KERNEL);
 168                 if (!xcbc->xcbc_keys) {
 169                         dev_err(dev, "Failed allocating buffer for XCBC keys\n");
 170                         goto init_failed;
 171                 }
 172         } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC authentication */
 173                 struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
 174                 const unsigned int digest_size = 2 * MAX_HMAC_DIGEST_SIZE;
 175                 dma_addr_t *pkey_dma = &hmac->padded_authkey_dma_addr;
 176
 177                 /* Allocate dma-coherent buffer for IPAD + OPAD */
 178                 hmac->ipad_opad = dma_alloc_coherent(dev, digest_size,
 179                                                      &hmac->ipad_opad_dma_addr,
 180                                                      GFP_KERNEL);
 181
 182                 if (!hmac->ipad_opad) {
 183                         dev_err(dev, "Failed allocating IPAD/OPAD buffer\n");
 184                         goto init_failed;
 185                 }
 186
 187                 dev_dbg(dev, "Allocated authkey buffer in context ctx->authkey=@%p\n",
 188                         hmac->ipad_opad);
 189
 190                 hmac->padded_authkey = dma_alloc_coherent(dev,
 191                                                           MAX_HMAC_BLOCK_SIZE,
 192                                                           pkey_dma,
 193                                                           GFP_KERNEL);
 194
 195                 if (!hmac->padded_authkey) {
 196                         dev_err(dev, "failed to allocate padded_authkey\n");
 197                         goto init_failed;
 198                 }
 199         } else {
 200                 ctx->auth_state.hmac.ipad_opad = NULL;
 201                 ctx->auth_state.hmac.padded_authkey = NULL;
 202         }
 203         ctx->hash_len = cc_get_aead_hash_len(tfm);
 204
 205         return 0;
 206
 207 init_failed:
 208         cc_aead_exit(tfm);
 209         return -ENOMEM;
 210 }
 211
 212 static void cc_aead_complete(struct device *dev, void *cc_req, int err)
 213 {
 214         struct aead_request *areq = (struct aead_request *)cc_req;
 215         struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
 216         struct crypto_aead *tfm = crypto_aead_reqtfm(cc_req);
 217         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 218
 219         /* BACKLOG notification */
 220         if (err == -EINPROGRESS)
 221                 goto done;
 222
 223         cc_unmap_aead_request(dev, areq);
 224
 225         /* Restore ordinary iv pointer */
 226         areq->iv = areq_ctx->backup_iv;
 227
 228         if (err)
 229                 goto done;
 230
 231         if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
 232                 if (memcmp(areq_ctx->mac_buf, areq_ctx->icv_virt_addr,
 233                            ctx->authsize) != 0) {
 234                         dev_dbg(dev, "Payload authentication failure, (auth-size=%d, cipher=%d)\n",
 235                                 ctx->authsize, ctx->cipher_mode);
 236                         /* In case of payload authentication failure, MUST NOT
 237                          * revealed the decrypted message --> zero its memory.
 238                          */
 239                         cc_zero_sgl(areq->dst, areq_ctx->cryptlen);
 240                         err = -EBADMSG;
 241                 }
 242         } else { /*ENCRYPT*/
 243                 if (areq_ctx->is_icv_fragmented) {
 244                         u32 skip = areq->cryptlen + areq_ctx->dst_offset;
 245
 246                         cc_copy_sg_portion(dev, areq_ctx->mac_buf,
 247                                            areq_ctx->dst_sgl, skip,
 248                                            (skip + ctx->authsize),
 249                                            CC_SG_FROM_BUF);
 250                 }
 251
 252                 /* If an IV was generated, copy it back to the user provided
 253                  * buffer.
 254                  */
 255                 if (areq_ctx->backup_giv) {
 256                         if (ctx->cipher_mode == DRV_CIPHER_CTR)
 257                                 memcpy(areq_ctx->backup_giv, areq_ctx->ctr_iv +
 258                                        CTR_RFC3686_NONCE_SIZE,
 259                                        CTR_RFC3686_IV_SIZE);
 260                         else if (ctx->cipher_mode == DRV_CIPHER_CCM)
 261                                 memcpy(areq_ctx->backup_giv, areq_ctx->ctr_iv +
 262                                        CCM_BLOCK_IV_OFFSET, CCM_BLOCK_IV_SIZE);
 263                 }
 264         }
 265 done:
 266         aead_request_complete(areq, err);
 267 }
 268
 269 static unsigned int xcbc_setkey(struct cc_hw_desc *desc,
 270                                 struct cc_aead_ctx *ctx)
 271 {
 272         /* Load the AES key */
 273         hw_desc_init(&desc[0]);
 274         /* We are using for the source/user key the same buffer
 275          * as for the output keys, * because after this key loading it
 276          * is not needed anymore
 277          */
 278         set_din_type(&desc[0], DMA_DLLI,
 279                      ctx->auth_state.xcbc.xcbc_keys_dma_addr, ctx->auth_keylen,
 280                      NS_BIT);
 281         set_cipher_mode(&desc[0], DRV_CIPHER_ECB);
 282         set_cipher_config0(&desc[0], DRV_CRYPTO_DIRECTION_ENCRYPT);
 283         set_key_size_aes(&desc[0], ctx->auth_keylen);
 284         set_flow_mode(&desc[0], S_DIN_to_AES);
 285         set_setup_mode(&desc[0], SETUP_LOAD_KEY0);
 286
 287         hw_desc_init(&desc[1]);
 288         set_din_const(&desc[1], 0x01010101, CC_AES_128_BIT_KEY_SIZE);
 289         set_flow_mode(&desc[1], DIN_AES_DOUT);
 290         set_dout_dlli(&desc[1], ctx->auth_state.xcbc.xcbc_keys_dma_addr,
 291                       AES_KEYSIZE_128, NS_BIT, 0);
 292
 293         hw_desc_init(&desc[2]);
 294         set_din_const(&desc[2], 0x02020202, CC_AES_128_BIT_KEY_SIZE);
 295         set_flow_mode(&desc[2], DIN_AES_DOUT);
 296         set_dout_dlli(&desc[2], (ctx->auth_state.xcbc.xcbc_keys_dma_addr
 297                                          + AES_KEYSIZE_128),
 298                               AES_KEYSIZE_128, NS_BIT, 0);
 299
 300         hw_desc_init(&desc[3]);
 301         set_din_const(&desc[3], 0x03030303, CC_AES_128_BIT_KEY_SIZE);
 302         set_flow_mode(&desc[3], DIN_AES_DOUT);
 303         set_dout_dlli(&desc[3], (ctx->auth_state.xcbc.xcbc_keys_dma_addr
 304                                           + 2 * AES_KEYSIZE_128),
 305                               AES_KEYSIZE_128, NS_BIT, 0);
 306
 307         return 4;
 308 }
 309
 310 static int hmac_setkey(struct cc_hw_desc *desc, struct cc_aead_ctx *ctx)
 311 {
 312         unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST };
 313         unsigned int digest_ofs = 0;
 314         unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
 315                         DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
 316         unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
 317                         CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
 318         struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
 319
 320         unsigned int idx = 0;
 321         int i;
 322
 323         /* calc derived HMAC key */
 324         for (i = 0; i < 2; i++) {
 325                 /* Load hash initial state */
 326                 hw_desc_init(&desc[idx]);
 327                 set_cipher_mode(&desc[idx], hash_mode);
 328                 set_din_sram(&desc[idx],
 329                              cc_larval_digest_addr(ctx->drvdata,
 330                                                    ctx->auth_mode),
 331                              digest_size);
 332                 set_flow_mode(&desc[idx], S_DIN_to_HASH);
 333                 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
 334                 idx++;
 335
 336                 /* Load the hash current length*/
 337                 hw_desc_init(&desc[idx]);
 338                 set_cipher_mode(&desc[idx], hash_mode);
 339                 set_din_const(&desc[idx], 0, ctx->hash_len);
 340                 set_flow_mode(&desc[idx], S_DIN_to_HASH);
 341                 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
 342                 idx++;
 343
 344                 /* Prepare ipad key */
 345                 hw_desc_init(&desc[idx]);
 346                 set_xor_val(&desc[idx], hmac_pad_const[i]);
 347                 set_cipher_mode(&desc[idx], hash_mode);
 348                 set_flow_mode(&desc[idx], S_DIN_to_HASH);
 349                 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
 350                 idx++;
 351
 352                 /* Perform HASH update */
 353                 hw_desc_init(&desc[idx]);
 354                 set_din_type(&desc[idx], DMA_DLLI,
 355                              hmac->padded_authkey_dma_addr,
 356                              SHA256_BLOCK_SIZE, NS_BIT);
 357                 set_cipher_mode(&desc[idx], hash_mode);
 358                 set_xor_active(&desc[idx]);
 359                 set_flow_mode(&desc[idx], DIN_HASH);
 360                 idx++;
 361
 362                 /* Get the digset */
 363                 hw_desc_init(&desc[idx]);
 364                 set_cipher_mode(&desc[idx], hash_mode);
 365                 set_dout_dlli(&desc[idx],
 366                               (hmac->ipad_opad_dma_addr + digest_ofs),
 367                               digest_size, NS_BIT, 0);
 368                 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 369                 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
 370                 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
 371                 idx++;
 372
 373                 digest_ofs += digest_size;
 374         }
 375
 376         return idx;
 377 }
 378
 379 static int validate_keys_sizes(struct cc_aead_ctx *ctx)
 380 {
 381         struct device *dev = drvdata_to_dev(ctx->drvdata);
 382
 383         dev_dbg(dev, "enc_keylen=%u  authkeylen=%u\n",
 384                 ctx->enc_keylen, ctx->auth_keylen);
 385
 386         switch (ctx->auth_mode) {
 387         case DRV_HASH_SHA1:
 388         case DRV_HASH_SHA256:
 389                 break;
 390         case DRV_HASH_XCBC_MAC:
 391                 if (ctx->auth_keylen != AES_KEYSIZE_128 &&
 392                     ctx->auth_keylen != AES_KEYSIZE_192 &&
 393                     ctx->auth_keylen != AES_KEYSIZE_256)
 394                         return -ENOTSUPP;
 395                 break;
 396         case DRV_HASH_NULL: /* Not authenc (e.g., CCM) - no auth_key) */
 397                 if (ctx->auth_keylen > 0)
 398                         return -EINVAL;
 399                 break;
 400         default:
 401                 dev_err(dev, "Invalid auth_mode=%d\n", ctx->auth_mode);
 402                 return -EINVAL;
 403         }
 404         /* Check cipher key size */
 405         if (ctx->flow_mode == S_DIN_to_DES) {
 406                 if (ctx->enc_keylen != DES3_EDE_KEY_SIZE) {
 407                         dev_err(dev, "Invalid cipher(3DES) key size: %u\n",
 408                                 ctx->enc_keylen);
 409                         return -EINVAL;
 410                 }
 411         } else { /* Default assumed to be AES ciphers */
 412                 if (ctx->enc_keylen != AES_KEYSIZE_128 &&
 413                     ctx->enc_keylen != AES_KEYSIZE_192 &&
 414                     ctx->enc_keylen != AES_KEYSIZE_256) {
 415                         dev_err(dev, "Invalid cipher(AES) key size: %u\n",
 416                                 ctx->enc_keylen);
 417                         return -EINVAL;
 418                 }
 419         }
 420
 421         return 0; /* All tests of keys sizes passed */
 422 }
 423
 424 /* This function prepers the user key so it can pass to the hmac processing
 425  * (copy to intenral buffer or hash in case of key longer than block
 426  */
 427 static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *authkey,
 428                                  unsigned int keylen)
 429 {
 430         dma_addr_t key_dma_addr = 0;
 431         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 432         struct device *dev = drvdata_to_dev(ctx->drvdata);
 433         u32 larval_addr = cc_larval_digest_addr(ctx->drvdata, ctx->auth_mode);
 434         struct cc_crypto_req cc_req = {};
 435         unsigned int blocksize;
 436         unsigned int digestsize;
 437         unsigned int hashmode;
 438         unsigned int idx = 0;
 439         int rc = 0;
 440         u8 *key = NULL;
 441         struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ];
 442         dma_addr_t padded_authkey_dma_addr =
 443                 ctx->auth_state.hmac.padded_authkey_dma_addr;
 444
 445         switch (ctx->auth_mode) { /* auth_key required and >0 */
 446         case DRV_HASH_SHA1:
 447                 blocksize = SHA1_BLOCK_SIZE;
 448                 digestsize = SHA1_DIGEST_SIZE;
 449                 hashmode = DRV_HASH_HW_SHA1;
 450                 break;
 451         case DRV_HASH_SHA256:
 452         default:
 453                 blocksize = SHA256_BLOCK_SIZE;
 454                 digestsize = SHA256_DIGEST_SIZE;
 455                 hashmode = DRV_HASH_HW_SHA256;
 456         }
 457
 458         if (keylen != 0) {
 459
 460                 key = kmemdup(authkey, keylen, GFP_KERNEL);
 461                 if (!key)
 462                         return -ENOMEM;
 463
 464                 key_dma_addr = dma_map_single(dev, (void *)key, keylen,
 465                                               DMA_TO_DEVICE);
 466                 if (dma_mapping_error(dev, key_dma_addr)) {
 467                         dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n",
 468                                 key, keylen);
 469                         kzfree(key);
 470                         return -ENOMEM;
 471                 }
 472                 if (keylen > blocksize) {
 473                         /* Load hash initial state */
 474                         hw_desc_init(&desc[idx]);
 475                         set_cipher_mode(&desc[idx], hashmode);
 476                         set_din_sram(&desc[idx], larval_addr, digestsize);
 477                         set_flow_mode(&desc[idx], S_DIN_to_HASH);
 478                         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
 479                         idx++;
 480
 481                         /* Load the hash current length*/
 482                         hw_desc_init(&desc[idx]);
 483                         set_cipher_mode(&desc[idx], hashmode);
 484                         set_din_const(&desc[idx], 0, ctx->hash_len);
 485                         set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
 486                         set_flow_mode(&desc[idx], S_DIN_to_HASH);
 487                         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
 488                         idx++;
 489
 490                         hw_desc_init(&desc[idx]);
 491                         set_din_type(&desc[idx], DMA_DLLI,
 492                                      key_dma_addr, keylen, NS_BIT);
 493                         set_flow_mode(&desc[idx], DIN_HASH);
 494                         idx++;
 495
 496                         /* Get hashed key */
 497                         hw_desc_init(&desc[idx]);
 498                         set_cipher_mode(&desc[idx], hashmode);
 499                         set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
 500                                       digestsize, NS_BIT, 0);
 501                         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 502                         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
 503                         set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
 504                         set_cipher_config0(&desc[idx],
 505                                            HASH_DIGEST_RESULT_LITTLE_ENDIAN);
 506                         idx++;
 507
 508                         hw_desc_init(&desc[idx]);
 509                         set_din_const(&desc[idx], 0, (blocksize - digestsize));
 510                         set_flow_mode(&desc[idx], BYPASS);
 511                         set_dout_dlli(&desc[idx], (padded_authkey_dma_addr +
 512                                       digestsize), (blocksize - digestsize),
 513                                       NS_BIT, 0);
 514                         idx++;
 515                 } else {
 516                         hw_desc_init(&desc[idx]);
 517                         set_din_type(&desc[idx], DMA_DLLI, key_dma_addr,
 518                                      keylen, NS_BIT);
 519                         set_flow_mode(&desc[idx], BYPASS);
 520                         set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
 521                                       keylen, NS_BIT, 0);
 522                         idx++;
 523
 524                         if ((blocksize - keylen) != 0) {
 525                                 hw_desc_init(&desc[idx]);
 526                                 set_din_const(&desc[idx], 0,
 527                                               (blocksize - keylen));
 528                                 set_flow_mode(&desc[idx], BYPASS);
 529                                 set_dout_dlli(&desc[idx],
 530                                               (padded_authkey_dma_addr +
 531                                                keylen),
 532                                               (blocksize - keylen), NS_BIT, 0);
 533                                 idx++;
 534                         }
 535                 }
 536         } else {
 537                 hw_desc_init(&desc[idx]);
 538                 set_din_const(&desc[idx], 0, (blocksize - keylen));
 539                 set_flow_mode(&desc[idx], BYPASS);
 540                 set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
 541                               blocksize, NS_BIT, 0);
 542                 idx++;
 543         }
 544
 545         rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);
 546         if (rc)
 547                 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
 548
 549         if (key_dma_addr)
 550                 dma_unmap_single(dev, key_dma_addr, keylen, DMA_TO_DEVICE);
 551
 552         kzfree(key);
 553
 554         return rc;
 555 }
 556
 557 static int cc_aead_setkey(struct crypto_aead *tfm, const u8 *key,
 558                           unsigned int keylen)
 559 {
 560         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 561         struct cc_crypto_req cc_req = {};
 562         struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ];
 563         unsigned int seq_len = 0;
 564         struct device *dev = drvdata_to_dev(ctx->drvdata);
 565         const u8 *enckey, *authkey;
 566         int rc;
 567
 568         dev_dbg(dev, "Setting key in context @%p for %s. key=%p keylen=%u\n",
 569                 ctx, crypto_tfm_alg_name(crypto_aead_tfm(tfm)), key, keylen);
 570
 571         /* STAT_PHASE_0: Init and sanity checks */
 572
 573         if (ctx->auth_mode != DRV_HASH_NULL) { /* authenc() alg. */
 574                 struct crypto_authenc_keys keys;
 575
 576                 rc = crypto_authenc_extractkeys(&keys, key, keylen);
 577                 if (rc)
 578                         goto badkey;
 579                 enckey = keys.enckey;
 580                 authkey = keys.authkey;
 581                 ctx->enc_keylen = keys.enckeylen;
 582                 ctx->auth_keylen = keys.authkeylen;
 583
 584                 if (ctx->cipher_mode == DRV_CIPHER_CTR) {
 585                         /* the nonce is stored in bytes at end of key */
 586                         rc = -EINVAL;
 587                         if (ctx->enc_keylen <
 588                             (AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE))
 589                                 goto badkey;
 590                         /* Copy nonce from last 4 bytes in CTR key to
 591                          *  first 4 bytes in CTR IV
 592                          */
 593                         memcpy(ctx->ctr_nonce, enckey + ctx->enc_keylen -
 594                                CTR_RFC3686_NONCE_SIZE, CTR_RFC3686_NONCE_SIZE);
 595                         /* Set CTR key size */
 596                         ctx->enc_keylen -= CTR_RFC3686_NONCE_SIZE;
 597                 }
 598         } else { /* non-authenc - has just one key */
 599                 enckey = key;
 600                 authkey = NULL;
 601                 ctx->enc_keylen = keylen;
 602                 ctx->auth_keylen = 0;
 603         }
 604
 605         rc = validate_keys_sizes(ctx);
 606         if (rc)
 607                 goto badkey;
 608
 609         /* STAT_PHASE_1: Copy key to ctx */
 610
 611         /* Get key material */
 612         memcpy(ctx->enckey, enckey, ctx->enc_keylen);
 613         if (ctx->enc_keylen == 24)
 614                 memset(ctx->enckey + 24, 0, CC_AES_KEY_SIZE_MAX - 24);
 615         if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
 616                 memcpy(ctx->auth_state.xcbc.xcbc_keys, authkey,
 617                        ctx->auth_keylen);
 618         } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC */
 619                 rc = cc_get_plain_hmac_key(tfm, authkey, ctx->auth_keylen);
 620                 if (rc)
 621                         goto badkey;
 622         }
 623
 624         /* STAT_PHASE_2: Create sequence */
 625
 626         switch (ctx->auth_mode) {
 627         case DRV_HASH_SHA1:
 628         case DRV_HASH_SHA256:
 629                 seq_len = hmac_setkey(desc, ctx);
 630                 break;
 631         case DRV_HASH_XCBC_MAC:
 632                 seq_len = xcbc_setkey(desc, ctx);
 633                 break;
 634         case DRV_HASH_NULL: /* non-authenc modes, e.g., CCM */
 635                 break; /* No auth. key setup */
 636         default:
 637                 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode);
 638                 rc = -ENOTSUPP;
 639                 goto badkey;
 640         }
 641
 642         /* STAT_PHASE_3: Submit sequence to HW */
 643
 644         if (seq_len > 0) { /* For CCM there is no sequence to setup the key */
 645                 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, seq_len);
 646                 if (rc) {
 647                         dev_err(dev, "send_request() failed (rc=%d)\n", rc);
 648                         goto setkey_error;
 649                 }
 650         }
 651
 652         /* Update STAT_PHASE_3 */
 653         return rc;
 654
 655 badkey:
 656         crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 657
 658 setkey_error:
 659         return rc;
 660 }
 661
 662 static int cc_des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
 663                                unsigned int keylen)
 664 {
 665         struct crypto_authenc_keys keys;
 666         u32 flags;
 667         int err;
 668
 669         err = crypto_authenc_extractkeys(&keys, key, keylen);
 670         if (unlikely(err))
 671                 goto badkey;
 672
 673         err = -EINVAL;
 674         if (keys.enckeylen != DES3_EDE_KEY_SIZE)
 675                 goto badkey;
 676
 677         flags = crypto_aead_get_flags(aead);
 678         err = __des3_verify_key(&flags, keys.enckey);
 679         if (unlikely(err)) {
 680                 crypto_aead_set_flags(aead, flags);
 681                 goto out;
 682         }
 683
 684         err = cc_aead_setkey(aead, key, keylen);
 685
 686 out:
 687         memzero_explicit(&keys, sizeof(keys));
 688         return err;
 689
 690 badkey:
 691         crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 692         goto out;
 693 }
 694
 695 static int cc_rfc4309_ccm_setkey(struct crypto_aead *tfm, const u8 *key,
 696                                  unsigned int keylen)
 697 {
 698         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 699
 700         if (keylen < 3)
 701                 return -EINVAL;
 702
 703         keylen -= 3;
 704         memcpy(ctx->ctr_nonce, key + keylen, 3);
 705
 706         return cc_aead_setkey(tfm, key, keylen);
 707 }
 708
 709 static int cc_aead_setauthsize(struct crypto_aead *authenc,
 710                                unsigned int authsize)
 711 {
 712         struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
 713         struct device *dev = drvdata_to_dev(ctx->drvdata);
 714
 715         /* Unsupported auth. sizes */
 716         if (authsize == 0 ||
 717             authsize > crypto_aead_maxauthsize(authenc)) {
 718                 return -ENOTSUPP;
 719         }
 720
 721         ctx->authsize = authsize;
 722         dev_dbg(dev, "authlen=%d\n", ctx->authsize);
 723
 724         return 0;
 725 }
 726
 727 static int cc_rfc4309_ccm_setauthsize(struct crypto_aead *authenc,
 728                                       unsigned int authsize)
 729 {
 730         switch (authsize) {
 731         case 8:
 732         case 12:
 733         case 16:
 734                 break;
 735         default:
 736                 return -EINVAL;
 737         }
 738
 739         return cc_aead_setauthsize(authenc, authsize);
 740 }
 741
 742 static int cc_ccm_setauthsize(struct crypto_aead *authenc,
 743                               unsigned int authsize)
 744 {
 745         switch (authsize) {
 746         case 4:
 747         case 6:
 748         case 8:
 749         case 10:
 750         case 12:
 751         case 14:
 752         case 16:
 753                 break;
 754         default:
 755                 return -EINVAL;
 756         }
 757
 758         return cc_aead_setauthsize(authenc, authsize);
 759 }
 760
 761 static void cc_set_assoc_desc(struct aead_request *areq, unsigned int flow_mode,
 762                               struct cc_hw_desc desc[], unsigned int *seq_size)
 763 {
 764         struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
 765         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 766         struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
 767         enum cc_req_dma_buf_type assoc_dma_type = areq_ctx->assoc_buff_type;
 768         unsigned int idx = *seq_size;
 769         struct device *dev = drvdata_to_dev(ctx->drvdata);
 770
 771         switch (assoc_dma_type) {
 772         case CC_DMA_BUF_DLLI:
 773                 dev_dbg(dev, "ASSOC buffer type DLLI\n");
 774                 hw_desc_init(&desc[idx]);
 775                 set_din_type(&desc[idx], DMA_DLLI, sg_dma_address(areq->src),
 776                              areq_ctx->assoclen, NS_BIT);
 777                 set_flow_mode(&desc[idx], flow_mode);
 778                 if (ctx->auth_mode == DRV_HASH_XCBC_MAC &&
 779                     areq_ctx->cryptlen > 0)
 780                         set_din_not_last_indication(&desc[idx]);
 781                 break;
 782         case CC_DMA_BUF_MLLI:
 783                 dev_dbg(dev, "ASSOC buffer type MLLI\n");
 784                 hw_desc_init(&desc[idx]);
 785                 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->assoc.sram_addr,
 786                              areq_ctx->assoc.mlli_nents, NS_BIT);
 787                 set_flow_mode(&desc[idx], flow_mode);
 788                 if (ctx->auth_mode == DRV_HASH_XCBC_MAC &&
 789                     areq_ctx->cryptlen > 0)
 790                         set_din_not_last_indication(&desc[idx]);
 791                 break;
 792         case CC_DMA_BUF_NULL:
 793         default:
 794                 dev_err(dev, "Invalid ASSOC buffer type\n");
 795         }
 796
 797         *seq_size = (++idx);
 798 }
 799
 800 static void cc_proc_authen_desc(struct aead_request *areq,
 801                                 unsigned int flow_mode,
 802                                 struct cc_hw_desc desc[],
 803                                 unsigned int *seq_size, int direct)
 804 {
 805         struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
 806         enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;
 807         unsigned int idx = *seq_size;
 808         struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
 809         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 810         struct device *dev = drvdata_to_dev(ctx->drvdata);
 811
 812         switch (data_dma_type) {
 813         case CC_DMA_BUF_DLLI:
 814         {
 815                 struct scatterlist *cipher =
 816                         (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
 817                         areq_ctx->dst_sgl : areq_ctx->src_sgl;
 818
 819                 unsigned int offset =
 820                         (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
 821                         areq_ctx->dst_offset : areq_ctx->src_offset;
 822                 dev_dbg(dev, "AUTHENC: SRC/DST buffer type DLLI\n");
 823                 hw_desc_init(&desc[idx]);
 824                 set_din_type(&desc[idx], DMA_DLLI,
 825                              (sg_dma_address(cipher) + offset),
 826                              areq_ctx->cryptlen, NS_BIT);
 827                 set_flow_mode(&desc[idx], flow_mode);
 828                 break;
 829         }
 830         case CC_DMA_BUF_MLLI:
 831         {
 832                 /* DOUBLE-PASS flow (as default)
 833                  * assoc. + iv + data -compact in one table
 834                  * if assoclen is ZERO only IV perform
 835                  */
 836                 cc_sram_addr_t mlli_addr = areq_ctx->assoc.sram_addr;
 837                 u32 mlli_nents = areq_ctx->assoc.mlli_nents;
 838
 839                 if (areq_ctx->is_single_pass) {
 840                         if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
 841                                 mlli_addr = areq_ctx->dst.sram_addr;
 842                                 mlli_nents = areq_ctx->dst.mlli_nents;
 843                         } else {
 844                                 mlli_addr = areq_ctx->src.sram_addr;
 845                                 mlli_nents = areq_ctx->src.mlli_nents;
 846                         }
 847                 }
 848
 849                 dev_dbg(dev, "AUTHENC: SRC/DST buffer type MLLI\n");
 850                 hw_desc_init(&desc[idx]);
 851                 set_din_type(&desc[idx], DMA_MLLI, mlli_addr, mlli_nents,
 852                              NS_BIT);
 853                 set_flow_mode(&desc[idx], flow_mode);
 854                 break;
 855         }
 856         case CC_DMA_BUF_NULL:
 857         default:
 858                 dev_err(dev, "AUTHENC: Invalid SRC/DST buffer type\n");
 859         }
 860
 861         *seq_size = (++idx);
 862 }
 863
 864 static void cc_proc_cipher_desc(struct aead_request *areq,
 865                                 unsigned int flow_mode,
 866                                 struct cc_hw_desc desc[],
 867                                 unsigned int *seq_size)
 868 {
 869         unsigned int idx = *seq_size;
 870         struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
 871         enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;
 872         struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
 873         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 874         struct device *dev = drvdata_to_dev(ctx->drvdata);
 875
 876         if (areq_ctx->cryptlen == 0)
 877                 return; /*null processing*/
 878
 879         switch (data_dma_type) {
 880         case CC_DMA_BUF_DLLI:
 881                 dev_dbg(dev, "CIPHER: SRC/DST buffer type DLLI\n");
 882                 hw_desc_init(&desc[idx]);
 883                 set_din_type(&desc[idx], DMA_DLLI,
 884                              (sg_dma_address(areq_ctx->src_sgl) +
 885                               areq_ctx->src_offset), areq_ctx->cryptlen,
 886                               NS_BIT);
 887                 set_dout_dlli(&desc[idx],
 888                               (sg_dma_address(areq_ctx->dst_sgl) +
 889                                areq_ctx->dst_offset),
 890                               areq_ctx->cryptlen, NS_BIT, 0);
 891                 set_flow_mode(&desc[idx], flow_mode);
 892                 break;
 893         case CC_DMA_BUF_MLLI:
 894                 dev_dbg(dev, "CIPHER: SRC/DST buffer type MLLI\n");
 895                 hw_desc_init(&desc[idx]);
 896                 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->src.sram_addr,
 897                              areq_ctx->src.mlli_nents, NS_BIT);
 898                 set_dout_mlli(&desc[idx], areq_ctx->dst.sram_addr,
 899                               areq_ctx->dst.mlli_nents, NS_BIT, 0);
 900                 set_flow_mode(&desc[idx], flow_mode);
 901                 break;
 902         case CC_DMA_BUF_NULL:
 903         default:
 904                 dev_err(dev, "CIPHER: Invalid SRC/DST buffer type\n");
 905         }
 906
 907         *seq_size = (++idx);
 908 }
 909
 910 static void cc_proc_digest_desc(struct aead_request *req,
 911                                 struct cc_hw_desc desc[],
 912                                 unsigned int *seq_size)
 913 {
 914         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 915         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 916         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
 917         unsigned int idx = *seq_size;
 918         unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
 919                                 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
 920         int direct = req_ctx->gen_ctx.op_type;
 921
 922         /* Get final ICV result */
 923         if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
 924                 hw_desc_init(&desc[idx]);
 925                 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 926                 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
 927                 set_dout_dlli(&desc[idx], req_ctx->icv_dma_addr, ctx->authsize,
 928                               NS_BIT, 1);
 929                 set_queue_last_ind(ctx->drvdata, &desc[idx]);
 930                 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
 931                         set_aes_not_hash_mode(&desc[idx]);
 932                         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
 933                 } else {
 934                         set_cipher_config0(&desc[idx],
 935                                            HASH_DIGEST_RESULT_LITTLE_ENDIAN);
 936                         set_cipher_mode(&desc[idx], hash_mode);
 937                 }
 938         } else { /*Decrypt*/
 939                 /* Get ICV out from hardware */
 940                 hw_desc_init(&desc[idx]);
 941                 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
 942                 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 943                 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr,
 944                               ctx->authsize, NS_BIT, 1);
 945                 set_queue_last_ind(ctx->drvdata, &desc[idx]);
 946                 set_cipher_config0(&desc[idx],
 947                                    HASH_DIGEST_RESULT_LITTLE_ENDIAN);
 948                 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
 949                 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
 950                         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
 951                         set_aes_not_hash_mode(&desc[idx]);
 952                 } else {
 953                         set_cipher_mode(&desc[idx], hash_mode);
 954                 }
 955         }
 956
 957         *seq_size = (++idx);
 958 }
 959
 960 static void cc_set_cipher_desc(struct aead_request *req,
 961                                struct cc_hw_desc desc[],
 962                                unsigned int *seq_size)
 963 {
 964         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 965         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 966         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
 967         unsigned int hw_iv_size = req_ctx->hw_iv_size;
 968         unsigned int idx = *seq_size;
 969         int direct = req_ctx->gen_ctx.op_type;
 970
 971         /* Setup cipher state */
 972         hw_desc_init(&desc[idx]);
 973         set_cipher_config0(&desc[idx], direct);
 974         set_flow_mode(&desc[idx], ctx->flow_mode);
 975         set_din_type(&desc[idx], DMA_DLLI, req_ctx->gen_ctx.iv_dma_addr,
 976                      hw_iv_size, NS_BIT);
 977         if (ctx->cipher_mode == DRV_CIPHER_CTR)
 978                 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
 979         else
 980                 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
 981         set_cipher_mode(&desc[idx], ctx->cipher_mode);
 982         idx++;
 983
 984         /* Setup enc. key */
 985         hw_desc_init(&desc[idx]);
 986         set_cipher_config0(&desc[idx], direct);
 987         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
 988         set_flow_mode(&desc[idx], ctx->flow_mode);
 989         if (ctx->flow_mode == S_DIN_to_AES) {
 990                 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
 991                              ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX :
 992                               ctx->enc_keylen), NS_BIT);
 993                 set_key_size_aes(&desc[idx], ctx->enc_keylen);
 994         } else {
 995                 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
 996                              ctx->enc_keylen, NS_BIT);
 997                 set_key_size_des(&desc[idx], ctx->enc_keylen);
 998         }
 999         set_cipher_mode(&desc[idx], ctx->cipher_mode);
1000         idx++;
1001
1002         *seq_size = idx;
1003 }
1004
1005 static void cc_proc_cipher(struct aead_request *req, struct cc_hw_desc desc[],
1006                            unsigned int *seq_size, unsigned int data_flow_mode)
1007 {
1008         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1009         int direct = req_ctx->gen_ctx.op_type;
1010         unsigned int idx = *seq_size;
1011
1012         if (req_ctx->cryptlen == 0)
1013                 return; /*null processing*/
1014
1015         cc_set_cipher_desc(req, desc, &idx);
1016         cc_proc_cipher_desc(req, data_flow_mode, desc, &idx);
1017         if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1018                 /* We must wait for DMA to write all cipher */
1019                 hw_desc_init(&desc[idx]);
1020                 set_din_no_dma(&desc[idx], 0, 0xfffff0);
1021                 set_dout_no_dma(&desc[idx], 0, 0, 1);
1022                 idx++;
1023         }
1024
1025         *seq_size = idx;
1026 }
1027
1028 static void cc_set_hmac_desc(struct aead_request *req, struct cc_hw_desc desc[],
1029                              unsigned int *seq_size)
1030 {
1031         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1032         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1033         unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
1034                                 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
1035         unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
1036                                 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
1037         unsigned int idx = *seq_size;
1038
1039         /* Loading hash ipad xor key state */
1040         hw_desc_init(&desc[idx]);
1041         set_cipher_mode(&desc[idx], hash_mode);
1042         set_din_type(&desc[idx], DMA_DLLI,
1043                      ctx->auth_state.hmac.ipad_opad_dma_addr, digest_size,
1044                      NS_BIT);
1045         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1046         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1047         idx++;
1048
1049         /* Load init. digest len (64 bytes) */
1050         hw_desc_init(&desc[idx]);
1051         set_cipher_mode(&desc[idx], hash_mode);
1052         set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode),
1053                      ctx->hash_len);
1054         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1055         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1056         idx++;
1057
1058         *seq_size = idx;
1059 }
1060
1061 static void cc_set_xcbc_desc(struct aead_request *req, struct cc_hw_desc desc[],
1062                              unsigned int *seq_size)
1063 {
1064         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1065         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1066         unsigned int idx = *seq_size;
1067
1068         /* Loading MAC state */
1069         hw_desc_init(&desc[idx]);
1070         set_din_const(&desc[idx], 0, CC_AES_BLOCK_SIZE);
1071         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1072         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1073         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1074         set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1075         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1076         set_aes_not_hash_mode(&desc[idx]);
1077         idx++;
1078
1079         /* Setup XCBC MAC K1 */
1080         hw_desc_init(&desc[idx]);
1081         set_din_type(&desc[idx], DMA_DLLI,
1082                      ctx->auth_state.xcbc.xcbc_keys_dma_addr,
1083                      AES_KEYSIZE_128, NS_BIT);
1084         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1085         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1086         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1087         set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1088         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1089         set_aes_not_hash_mode(&desc[idx]);
1090         idx++;
1091
1092         /* Setup XCBC MAC K2 */
1093         hw_desc_init(&desc[idx]);
1094         set_din_type(&desc[idx], DMA_DLLI,
1095                      (ctx->auth_state.xcbc.xcbc_keys_dma_addr +
1096                       AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT);
1097         set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1098         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1099         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1100         set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1101         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1102         set_aes_not_hash_mode(&desc[idx]);
1103         idx++;
1104
1105         /* Setup XCBC MAC K3 */
1106         hw_desc_init(&desc[idx]);
1107         set_din_type(&desc[idx], DMA_DLLI,
1108                      (ctx->auth_state.xcbc.xcbc_keys_dma_addr +
1109                       2 * AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT);
1110         set_setup_mode(&desc[idx], SETUP_LOAD_STATE2);
1111         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1112         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1113         set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1114         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1115         set_aes_not_hash_mode(&desc[idx]);
1116         idx++;
1117
1118         *seq_size = idx;
1119 }
1120
1121 static void cc_proc_header_desc(struct aead_request *req,
1122                                 struct cc_hw_desc desc[],
1123                                 unsigned int *seq_size)
1124 {
1125         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1126         unsigned int idx = *seq_size;
1127
1128         /* Hash associated data */
1129         if (areq_ctx->assoclen > 0)
1130                 cc_set_assoc_desc(req, DIN_HASH, desc, &idx);
1131
1132         /* Hash IV */
1133         *seq_size = idx;
1134 }
1135
1136 static void cc_proc_scheme_desc(struct aead_request *req,
1137                                 struct cc_hw_desc desc[],
1138                                 unsigned int *seq_size)
1139 {
1140         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1141         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1142         struct cc_aead_handle *aead_handle = ctx->drvdata->aead_handle;
1143         unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
1144                                 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
1145         unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
1146                                 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
1147         unsigned int idx = *seq_size;
1148
1149         hw_desc_init(&desc[idx]);
1150         set_cipher_mode(&desc[idx], hash_mode);
1151         set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr,
1152                       ctx->hash_len);
1153         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1154         set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);
1155         set_cipher_do(&desc[idx], DO_PAD);
1156         idx++;
1157
1158         /* Get final ICV result */
1159         hw_desc_init(&desc[idx]);
1160         set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr,
1161                       digest_size);
1162         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1163         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1164         set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN);
1165         set_cipher_mode(&desc[idx], hash_mode);
1166         idx++;
1167
1168         /* Loading hash opad xor key state */
1169         hw_desc_init(&desc[idx]);
1170         set_cipher_mode(&desc[idx], hash_mode);
1171         set_din_type(&desc[idx], DMA_DLLI,
1172                      (ctx->auth_state.hmac.ipad_opad_dma_addr + digest_size),
1173                      digest_size, NS_BIT);
1174         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1175         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1176         idx++;
1177
1178         /* Load init. digest len (64 bytes) */
1179         hw_desc_init(&desc[idx]);
1180         set_cipher_mode(&desc[idx], hash_mode);
1181         set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode),
1182                      ctx->hash_len);
1183         set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1184         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1185         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1186         idx++;
1187
1188         /* Perform HASH update */
1189         hw_desc_init(&desc[idx]);
1190         set_din_sram(&desc[idx], aead_handle->sram_workspace_addr,
1191                      digest_size);
1192         set_flow_mode(&desc[idx], DIN_HASH);
1193         idx++;
1194
1195         *seq_size = idx;
1196 }
1197
1198 static void cc_mlli_to_sram(struct aead_request *req,
1199                             struct cc_hw_desc desc[], unsigned int *seq_size)
1200 {
1201         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1202         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1203         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1204         struct device *dev = drvdata_to_dev(ctx->drvdata);
1205
1206         if ((req_ctx->assoc_buff_type == CC_DMA_BUF_MLLI ||
1207             req_ctx->data_buff_type == CC_DMA_BUF_MLLI ||
1208             !req_ctx->is_single_pass) && req_ctx->mlli_params.mlli_len) {
1209                 dev_dbg(dev, "Copy-to-sram: mlli_dma=%08x, mlli_size=%u\n",
1210                         (unsigned int)ctx->drvdata->mlli_sram_addr,
1211                         req_ctx->mlli_params.mlli_len);
1212                 /* Copy MLLI table host-to-sram */
1213                 hw_desc_init(&desc[*seq_size]);
1214                 set_din_type(&desc[*seq_size], DMA_DLLI,
1215                              req_ctx->mlli_params.mlli_dma_addr,
1216                              req_ctx->mlli_params.mlli_len, NS_BIT);
1217                 set_dout_sram(&desc[*seq_size],
1218                               ctx->drvdata->mlli_sram_addr,
1219                               req_ctx->mlli_params.mlli_len);
1220                 set_flow_mode(&desc[*seq_size], BYPASS);
1221                 (*seq_size)++;
1222         }
1223 }
1224
1225 static enum cc_flow_mode cc_get_data_flow(enum drv_crypto_direction direct,
1226                                           enum cc_flow_mode setup_flow_mode,
1227                                           bool is_single_pass)
1228 {
1229         enum cc_flow_mode data_flow_mode;
1230
1231         if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1232                 if (setup_flow_mode == S_DIN_to_AES)
1233                         data_flow_mode = is_single_pass ?
1234                                 AES_to_HASH_and_DOUT : DIN_AES_DOUT;
1235                 else
1236                         data_flow_mode = is_single_pass ?
1237                                 DES_to_HASH_and_DOUT : DIN_DES_DOUT;
1238         } else { /* Decrypt */
1239                 if (setup_flow_mode == S_DIN_to_AES)
1240                         data_flow_mode = is_single_pass ?
1241                                 AES_and_HASH : DIN_AES_DOUT;
1242                 else
1243                         data_flow_mode = is_single_pass ?
1244                                 DES_and_HASH : DIN_DES_DOUT;
1245         }
1246
1247         return data_flow_mode;
1248 }
1249
1250 static void cc_hmac_authenc(struct aead_request *req, struct cc_hw_desc desc[],
1251                             unsigned int *seq_size)
1252 {
1253         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1254         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1255         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1256         int direct = req_ctx->gen_ctx.op_type;
1257         unsigned int data_flow_mode =
1258                 cc_get_data_flow(direct, ctx->flow_mode,
1259                                  req_ctx->is_single_pass);
1260
1261         if (req_ctx->is_single_pass) {
1262                 /**
1263                  * Single-pass flow
1264                  */
1265                 cc_set_hmac_desc(req, desc, seq_size);
1266                 cc_set_cipher_desc(req, desc, seq_size);
1267                 cc_proc_header_desc(req, desc, seq_size);
1268                 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size);
1269                 cc_proc_scheme_desc(req, desc, seq_size);
1270                 cc_proc_digest_desc(req, desc, seq_size);
1271                 return;
1272         }
1273
1274         /**
1275          * Double-pass flow
1276          * Fallback for unsupported single-pass modes,
1277          * i.e. using assoc. data of non-word-multiple
1278          */
1279         if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1280                 /* encrypt first.. */
1281                 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1282                 /* authenc after..*/
1283                 cc_set_hmac_desc(req, desc, seq_size);
1284                 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1285                 cc_proc_scheme_desc(req, desc, seq_size);
1286                 cc_proc_digest_desc(req, desc, seq_size);
1287
1288         } else { /*DECRYPT*/
1289                 /* authenc first..*/
1290                 cc_set_hmac_desc(req, desc, seq_size);
1291                 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1292                 cc_proc_scheme_desc(req, desc, seq_size);
1293                 /* decrypt after.. */
1294                 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1295                 /* read the digest result with setting the completion bit
1296                  * must be after the cipher operation
1297                  */
1298                 cc_proc_digest_desc(req, desc, seq_size);
1299         }
1300 }
1301
1302 static void
1303 cc_xcbc_authenc(struct aead_request *req, struct cc_hw_desc desc[],
1304                 unsigned int *seq_size)
1305 {
1306         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1307         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1308         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1309         int direct = req_ctx->gen_ctx.op_type;
1310         unsigned int data_flow_mode =
1311                 cc_get_data_flow(direct, ctx->flow_mode,
1312                                  req_ctx->is_single_pass);
1313
1314         if (req_ctx->is_single_pass) {
1315                 /**
1316                  * Single-pass flow
1317                  */
1318                 cc_set_xcbc_desc(req, desc, seq_size);
1319                 cc_set_cipher_desc(req, desc, seq_size);
1320                 cc_proc_header_desc(req, desc, seq_size);
1321                 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size);
1322                 cc_proc_digest_desc(req, desc, seq_size);
1323                 return;
1324         }
1325
1326         /**
1327          * Double-pass flow
1328          * Fallback for unsupported single-pass modes,
1329          * i.e. using assoc. data of non-word-multiple
1330          */
1331         if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1332                 /* encrypt first.. */
1333                 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1334                 /* authenc after.. */
1335                 cc_set_xcbc_desc(req, desc, seq_size);
1336                 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1337                 cc_proc_digest_desc(req, desc, seq_size);
1338         } else { /*DECRYPT*/
1339                 /* authenc first.. */
1340                 cc_set_xcbc_desc(req, desc, seq_size);
1341                 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1342                 /* decrypt after..*/
1343                 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1344                 /* read the digest result with setting the completion bit
1345                  * must be after the cipher operation
1346                  */
1347                 cc_proc_digest_desc(req, desc, seq_size);
1348         }
1349 }
1350
1351 static int validate_data_size(struct cc_aead_ctx *ctx,
1352                               enum drv_crypto_direction direct,
1353                               struct aead_request *req)
1354 {
1355         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1356         struct device *dev = drvdata_to_dev(ctx->drvdata);
1357         unsigned int assoclen = areq_ctx->assoclen;
1358         unsigned int cipherlen = (direct == DRV_CRYPTO_DIRECTION_DECRYPT) ?
1359                         (req->cryptlen - ctx->authsize) : req->cryptlen;
1360
1361         if (direct == DRV_CRYPTO_DIRECTION_DECRYPT &&
1362             req->cryptlen < ctx->authsize)
1363                 goto data_size_err;
1364
1365         areq_ctx->is_single_pass = true; /*defaulted to fast flow*/
1366
1367         switch (ctx->flow_mode) {
1368         case S_DIN_to_AES:
1369                 if (ctx->cipher_mode == DRV_CIPHER_CBC &&
1370                     !IS_ALIGNED(cipherlen, AES_BLOCK_SIZE))
1371                         goto data_size_err;
1372                 if (ctx->cipher_mode == DRV_CIPHER_CCM)
1373                         break;
1374                 if (ctx->cipher_mode == DRV_CIPHER_GCTR) {
1375                         if (areq_ctx->plaintext_authenticate_only)
1376                                 areq_ctx->is_single_pass = false;
1377                         break;
1378                 }
1379
1380                 if (!IS_ALIGNED(assoclen, sizeof(u32)))
1381                         areq_ctx->is_single_pass = false;
1382
1383                 if (ctx->cipher_mode == DRV_CIPHER_CTR &&
1384                     !IS_ALIGNED(cipherlen, sizeof(u32)))
1385                         areq_ctx->is_single_pass = false;
1386
1387                 break;
1388         case S_DIN_to_DES:
1389                 if (!IS_ALIGNED(cipherlen, DES_BLOCK_SIZE))
1390                         goto data_size_err;
1391                 if (!IS_ALIGNED(assoclen, DES_BLOCK_SIZE))
1392                         areq_ctx->is_single_pass = false;
1393                 break;
1394         default:
1395                 dev_err(dev, "Unexpected flow mode (%d)\n", ctx->flow_mode);
1396                 goto data_size_err;
1397         }
1398
1399         return 0;
1400
1401 data_size_err:
1402         return -EINVAL;
1403 }
1404
1405 static unsigned int format_ccm_a0(u8 *pa0_buff, u32 header_size)
1406 {
1407         unsigned int len = 0;
1408
1409         if (header_size == 0)
1410                 return 0;
1411
1412         if (header_size < ((1UL << 16) - (1UL << 8))) {
1413                 len = 2;
1414
1415                 pa0_buff[0] = (header_size >> 8) & 0xFF;
1416                 pa0_buff[1] = header_size & 0xFF;
1417         } else {
1418                 len = 6;
1419
1420                 pa0_buff[0] = 0xFF;
1421                 pa0_buff[1] = 0xFE;
1422                 pa0_buff[2] = (header_size >> 24) & 0xFF;
1423                 pa0_buff[3] = (header_size >> 16) & 0xFF;
1424                 pa0_buff[4] = (header_size >> 8) & 0xFF;
1425                 pa0_buff[5] = header_size & 0xFF;
1426         }
1427
1428         return len;
1429 }
1430
1431 static int set_msg_len(u8 *block, unsigned int msglen, unsigned int csize)
1432 {
1433         __be32 data;
1434
1435         memset(block, 0, csize);
1436         block += csize;
1437
1438         if (csize >= 4)
1439                 csize = 4;
1440         else if (msglen > (1 << (8 * csize)))
1441                 return -EOVERFLOW;
1442
1443         data = cpu_to_be32(msglen);
1444         memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
1445
1446         return 0;
1447 }
1448
1449 static int cc_ccm(struct aead_request *req, struct cc_hw_desc desc[],
1450                   unsigned int *seq_size)
1451 {
1452         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1453         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1454         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1455         unsigned int idx = *seq_size;
1456         unsigned int cipher_flow_mode;
1457         dma_addr_t mac_result;
1458
1459         if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1460                 cipher_flow_mode = AES_to_HASH_and_DOUT;
1461                 mac_result = req_ctx->mac_buf_dma_addr;
1462         } else { /* Encrypt */
1463                 cipher_flow_mode = AES_and_HASH;
1464                 mac_result = req_ctx->icv_dma_addr;
1465         }
1466
1467         /* load key */
1468         hw_desc_init(&desc[idx]);
1469         set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1470         set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1471                      ((ctx->enc_keylen == 24) ?  CC_AES_KEY_SIZE_MAX :
1472                       ctx->enc_keylen), NS_BIT);
1473         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1474         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1475         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1476         set_flow_mode(&desc[idx], S_DIN_to_AES);
1477         idx++;
1478
1479         /* load ctr state */
1480         hw_desc_init(&desc[idx]);
1481         set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1482         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1483         set_din_type(&desc[idx], DMA_DLLI,
1484                      req_ctx->gen_ctx.iv_dma_addr, AES_BLOCK_SIZE, NS_BIT);
1485         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1486         set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1487         set_flow_mode(&desc[idx], S_DIN_to_AES);
1488         idx++;
1489
1490         /* load MAC key */
1491         hw_desc_init(&desc[idx]);
1492         set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1493         set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1494                      ((ctx->enc_keylen == 24) ?  CC_AES_KEY_SIZE_MAX :
1495                       ctx->enc_keylen), NS_BIT);
1496         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1497         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1498         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1499         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1500         set_aes_not_hash_mode(&desc[idx]);
1501         idx++;
1502
1503         /* load MAC state */
1504         hw_desc_init(&desc[idx]);
1505         set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1506         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1507         set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1508                      AES_BLOCK_SIZE, NS_BIT);
1509         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1510         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1511         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1512         set_aes_not_hash_mode(&desc[idx]);
1513         idx++;
1514
1515         /* process assoc data */
1516         if (req_ctx->assoclen > 0) {
1517                 cc_set_assoc_desc(req, DIN_HASH, desc, &idx);
1518         } else {
1519                 hw_desc_init(&desc[idx]);
1520                 set_din_type(&desc[idx], DMA_DLLI,
1521                              sg_dma_address(&req_ctx->ccm_adata_sg),
1522                              AES_BLOCK_SIZE + req_ctx->ccm_hdr_size, NS_BIT);
1523                 set_flow_mode(&desc[idx], DIN_HASH);
1524                 idx++;
1525         }
1526
1527         /* process the cipher */
1528         if (req_ctx->cryptlen)
1529                 cc_proc_cipher_desc(req, cipher_flow_mode, desc, &idx);
1530
1531         /* Read temporal MAC */
1532         hw_desc_init(&desc[idx]);
1533         set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1534         set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, ctx->authsize,
1535                       NS_BIT, 0);
1536         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1537         set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN);
1538         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1539         set_aes_not_hash_mode(&desc[idx]);
1540         idx++;
1541
1542         /* load AES-CTR state (for last MAC calculation)*/
1543         hw_desc_init(&desc[idx]);
1544         set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1545         set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1546         set_din_type(&desc[idx], DMA_DLLI, req_ctx->ccm_iv0_dma_addr,
1547                      AES_BLOCK_SIZE, NS_BIT);
1548         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1549         set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1550         set_flow_mode(&desc[idx], S_DIN_to_AES);
1551         idx++;
1552
1553         hw_desc_init(&desc[idx]);
1554         set_din_no_dma(&desc[idx], 0, 0xfffff0);
1555         set_dout_no_dma(&desc[idx], 0, 0, 1);
1556         idx++;
1557
1558         /* encrypt the "T" value and store MAC in mac_state */
1559         hw_desc_init(&desc[idx]);
1560         set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1561                      ctx->authsize, NS_BIT);
1562         set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1);
1563         set_queue_last_ind(ctx->drvdata, &desc[idx]);
1564         set_flow_mode(&desc[idx], DIN_AES_DOUT);
1565         idx++;
1566
1567         *seq_size = idx;
1568         return 0;
1569 }
1570
1571 static int config_ccm_adata(struct aead_request *req)
1572 {
1573         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1574         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1575         struct device *dev = drvdata_to_dev(ctx->drvdata);
1576         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1577         //unsigned int size_of_a = 0, rem_a_size = 0;
1578         unsigned int lp = req->iv[0];
1579         /* Note: The code assume that req->iv[0] already contains the value
1580          * of L' of RFC3610
1581          */
1582         unsigned int l = lp + 1;  /* This is L' of RFC 3610. */
1583         unsigned int m = ctx->authsize;  /* This is M' of RFC 3610. */
1584         u8 *b0 = req_ctx->ccm_config + CCM_B0_OFFSET;
1585         u8 *a0 = req_ctx->ccm_config + CCM_A0_OFFSET;
1586         u8 *ctr_count_0 = req_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET;
1587         unsigned int cryptlen = (req_ctx->gen_ctx.op_type ==
1588                                  DRV_CRYPTO_DIRECTION_ENCRYPT) ?
1589                                 req->cryptlen :
1590                                 (req->cryptlen - ctx->authsize);
1591         int rc;
1592
1593         memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE);
1594         memset(req_ctx->ccm_config, 0, AES_BLOCK_SIZE * 3);
1595
1596         /* taken from crypto/ccm.c */
1597         /* 2 <= L <= 8, so 1 <= L' <= 7. */
1598         if (l < 2 || l > 8) {
1599                 dev_err(dev, "illegal iv value %X\n", req->iv[0]);
1600                 return -EINVAL;
1601         }
1602         memcpy(b0, req->iv, AES_BLOCK_SIZE);
1603
1604         /* format control info per RFC 3610 and
1605          * NIST Special Publication 800-38C
1606          */
1607         *b0 |= (8 * ((m - 2) / 2));
1608         if (req_ctx->assoclen > 0)
1609                 *b0 |= 64;  /* Enable bit 6 if Adata exists. */
1610
1611         rc = set_msg_len(b0 + 16 - l, cryptlen, l);  /* Write L'. */
1612         if (rc) {
1613                 dev_err(dev, "message len overflow detected");
1614                 return rc;
1615         }
1616          /* END of "taken from crypto/ccm.c" */
1617
1618         /* l(a) - size of associated data. */
1619         req_ctx->ccm_hdr_size = format_ccm_a0(a0, req_ctx->assoclen);
1620
1621         memset(req->iv + 15 - req->iv[0], 0, req->iv[0] + 1);
1622         req->iv[15] = 1;
1623
1624         memcpy(ctr_count_0, req->iv, AES_BLOCK_SIZE);
1625         ctr_count_0[15] = 0;
1626
1627         return 0;
1628 }
1629
1630 static void cc_proc_rfc4309_ccm(struct aead_request *req)
1631 {
1632         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1633         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1634         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1635
1636         /* L' */
1637         memset(areq_ctx->ctr_iv, 0, AES_BLOCK_SIZE);
1638         /* For RFC 4309, always use 4 bytes for message length
1639          * (at most 2^32-1 bytes).
1640          */
1641         areq_ctx->ctr_iv[0] = 3;
1642
1643         /* In RFC 4309 there is an 11-bytes nonce+IV part,
1644          * that we build here.
1645          */
1646         memcpy(areq_ctx->ctr_iv + CCM_BLOCK_NONCE_OFFSET, ctx->ctr_nonce,
1647                CCM_BLOCK_NONCE_SIZE);
1648         memcpy(areq_ctx->ctr_iv + CCM_BLOCK_IV_OFFSET, req->iv,
1649                CCM_BLOCK_IV_SIZE);
1650         req->iv = areq_ctx->ctr_iv;
1651         areq_ctx->assoclen -= CCM_BLOCK_IV_SIZE;
1652 }
1653
1654 static void cc_set_ghash_desc(struct aead_request *req,
1655                               struct cc_hw_desc desc[], unsigned int *seq_size)
1656 {
1657         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1658         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1659         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1660         unsigned int idx = *seq_size;
1661
1662         /* load key to AES*/
1663         hw_desc_init(&desc[idx]);
1664         set_cipher_mode(&desc[idx], DRV_CIPHER_ECB);
1665         set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1666         set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1667                      ctx->enc_keylen, NS_BIT);
1668         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1669         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1670         set_flow_mode(&desc[idx], S_DIN_to_AES);
1671         idx++;
1672
1673         /* process one zero block to generate hkey */
1674         hw_desc_init(&desc[idx]);
1675         set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE);
1676         set_dout_dlli(&desc[idx], req_ctx->hkey_dma_addr, AES_BLOCK_SIZE,
1677                       NS_BIT, 0);
1678         set_flow_mode(&desc[idx], DIN_AES_DOUT);
1679         idx++;
1680
1681         /* Memory Barrier */
1682         hw_desc_init(&desc[idx]);
1683         set_din_no_dma(&desc[idx], 0, 0xfffff0);
1684         set_dout_no_dma(&desc[idx], 0, 0, 1);
1685         idx++;
1686
1687         /* Load GHASH subkey */
1688         hw_desc_init(&desc[idx]);
1689         set_din_type(&desc[idx], DMA_DLLI, req_ctx->hkey_dma_addr,
1690                      AES_BLOCK_SIZE, NS_BIT);
1691         set_dout_no_dma(&desc[idx], 0, 0, 1);
1692         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1693         set_aes_not_hash_mode(&desc[idx]);
1694         set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1695         set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1696         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1697         idx++;
1698
1699         /* Configure Hash Engine to work with GHASH.
1700          * Since it was not possible to extend HASH submodes to add GHASH,
1701          * The following command is necessary in order to
1702          * select GHASH (according to HW designers)
1703          */
1704         hw_desc_init(&desc[idx]);
1705         set_din_no_dma(&desc[idx], 0, 0xfffff0);
1706         set_dout_no_dma(&desc[idx], 0, 0, 1);
1707         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1708         set_aes_not_hash_mode(&desc[idx]);
1709         set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1710         set_cipher_do(&desc[idx], 1); //1=AES_SK RKEK
1711         set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1712         set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1713         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1714         idx++;
1715
1716         /* Load GHASH initial STATE (which is 0). (for any hash there is an
1717          * initial state)
1718          */
1719         hw_desc_init(&desc[idx]);
1720         set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE);
1721         set_dout_no_dma(&desc[idx], 0, 0, 1);
1722         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1723         set_aes_not_hash_mode(&desc[idx]);
1724         set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1725         set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1726         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1727         idx++;
1728
1729         *seq_size = idx;
1730 }
1731
1732 static void cc_set_gctr_desc(struct aead_request *req, struct cc_hw_desc desc[],
1733                              unsigned int *seq_size)
1734 {
1735         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1736         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1737         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1738         unsigned int idx = *seq_size;
1739
1740         /* load key to AES*/
1741         hw_desc_init(&desc[idx]);
1742         set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1743         set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1744         set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1745                      ctx->enc_keylen, NS_BIT);
1746         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1747         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1748         set_flow_mode(&desc[idx], S_DIN_to_AES);
1749         idx++;
1750
1751         if (req_ctx->cryptlen && !req_ctx->plaintext_authenticate_only) {
1752                 /* load AES/CTR initial CTR value inc by 2*/
1753                 hw_desc_init(&desc[idx]);
1754                 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1755                 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1756                 set_din_type(&desc[idx], DMA_DLLI,
1757                              req_ctx->gcm_iv_inc2_dma_addr, AES_BLOCK_SIZE,
1758                              NS_BIT);
1759                 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1760                 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1761                 set_flow_mode(&desc[idx], S_DIN_to_AES);
1762                 idx++;
1763         }
1764
1765         *seq_size = idx;
1766 }
1767
1768 static void cc_proc_gcm_result(struct aead_request *req,
1769                                struct cc_hw_desc desc[],
1770                                unsigned int *seq_size)
1771 {
1772         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1773         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1774         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1775         dma_addr_t mac_result;
1776         unsigned int idx = *seq_size;
1777
1778         if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1779                 mac_result = req_ctx->mac_buf_dma_addr;
1780         } else { /* Encrypt */
1781                 mac_result = req_ctx->icv_dma_addr;
1782         }
1783
1784         /* process(ghash) gcm_block_len */
1785         hw_desc_init(&desc[idx]);
1786         set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_block_len_dma_addr,
1787                      AES_BLOCK_SIZE, NS_BIT);
1788         set_flow_mode(&desc[idx], DIN_HASH);
1789         idx++;
1790
1791         /* Store GHASH state after GHASH(Associated Data + Cipher +LenBlock) */
1792         hw_desc_init(&desc[idx]);
1793         set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1794         set_din_no_dma(&desc[idx], 0, 0xfffff0);
1795         set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, AES_BLOCK_SIZE,
1796                       NS_BIT, 0);
1797         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1798         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1799         set_aes_not_hash_mode(&desc[idx]);
1800
1801         idx++;
1802
1803         /* load AES/CTR initial CTR value inc by 1*/
1804         hw_desc_init(&desc[idx]);
1805         set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1806         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1807         set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_iv_inc1_dma_addr,
1808                      AES_BLOCK_SIZE, NS_BIT);
1809         set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1810         set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1811         set_flow_mode(&desc[idx], S_DIN_to_AES);
1812         idx++;
1813
1814         /* Memory Barrier */
1815         hw_desc_init(&desc[idx]);
1816         set_din_no_dma(&desc[idx], 0, 0xfffff0);
1817         set_dout_no_dma(&desc[idx], 0, 0, 1);
1818         idx++;
1819
1820         /* process GCTR on stored GHASH and store MAC in mac_state*/
1821         hw_desc_init(&desc[idx]);
1822         set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1823         set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1824                      AES_BLOCK_SIZE, NS_BIT);
1825         set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1);
1826         set_queue_last_ind(ctx->drvdata, &desc[idx]);
1827         set_flow_mode(&desc[idx], DIN_AES_DOUT);
1828         idx++;
1829
1830         *seq_size = idx;
1831 }
1832
1833 static int cc_gcm(struct aead_request *req, struct cc_hw_desc desc[],
1834                   unsigned int *seq_size)
1835 {
1836         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1837         unsigned int cipher_flow_mode;
1838
1839         if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1840                 cipher_flow_mode = AES_and_HASH;
1841         } else { /* Encrypt */
1842                 cipher_flow_mode = AES_to_HASH_and_DOUT;
1843         }
1844
1845         //in RFC4543 no data to encrypt. just copy data from src to dest.
1846         if (req_ctx->plaintext_authenticate_only) {
1847                 cc_proc_cipher_desc(req, BYPASS, desc, seq_size);
1848                 cc_set_ghash_desc(req, desc, seq_size);
1849                 /* process(ghash) assoc data */
1850                 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size);
1851                 cc_set_gctr_desc(req, desc, seq_size);
1852                 cc_proc_gcm_result(req, desc, seq_size);
1853                 return 0;
1854         }
1855
1856         // for gcm and rfc4106.
1857         cc_set_ghash_desc(req, desc, seq_size);
1858         /* process(ghash) assoc data */
1859         if (req_ctx->assoclen > 0)
1860                 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size);
1861         cc_set_gctr_desc(req, desc, seq_size);
1862         /* process(gctr+ghash) */
1863         if (req_ctx->cryptlen)
1864                 cc_proc_cipher_desc(req, cipher_flow_mode, desc, seq_size);
1865         cc_proc_gcm_result(req, desc, seq_size);
1866
1867         return 0;
1868 }
1869
1870 static int config_gcm_context(struct aead_request *req)
1871 {
1872         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1873         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1874         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1875         struct device *dev = drvdata_to_dev(ctx->drvdata);
1876
1877         unsigned int cryptlen = (req_ctx->gen_ctx.op_type ==
1878                                  DRV_CRYPTO_DIRECTION_ENCRYPT) ?
1879                                 req->cryptlen :
1880                                 (req->cryptlen - ctx->authsize);
1881         __be32 counter = cpu_to_be32(2);
1882
1883         dev_dbg(dev, "%s() cryptlen = %d, req_ctx->assoclen = %d ctx->authsize = %d\n",
1884                 __func__, cryptlen, req_ctx->assoclen, ctx->authsize);
1885
1886         memset(req_ctx->hkey, 0, AES_BLOCK_SIZE);
1887
1888         memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE);
1889
1890         memcpy(req->iv + 12, &counter, 4);
1891         memcpy(req_ctx->gcm_iv_inc2, req->iv, 16);
1892
1893         counter = cpu_to_be32(1);
1894         memcpy(req->iv + 12, &counter, 4);
1895         memcpy(req_ctx->gcm_iv_inc1, req->iv, 16);
1896
1897         if (!req_ctx->plaintext_authenticate_only) {
1898                 __be64 temp64;
1899
1900                 temp64 = cpu_to_be64(req_ctx->assoclen * 8);
1901                 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64));
1902                 temp64 = cpu_to_be64(cryptlen * 8);
1903                 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8);
1904         } else {
1905                 /* rfc4543=>  all data(AAD,IV,Plain) are considered additional
1906                  * data that is nothing is encrypted.
1907                  */
1908                 __be64 temp64;
1909
1910                 temp64 = cpu_to_be64((req_ctx->assoclen +
1911                                       GCM_BLOCK_RFC4_IV_SIZE + cryptlen) * 8);
1912                 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64));
1913                 temp64 = 0;
1914                 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8);
1915         }
1916
1917         return 0;
1918 }
1919
1920 static void cc_proc_rfc4_gcm(struct aead_request *req)
1921 {
1922         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1923         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1924         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1925
1926         memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_NONCE_OFFSET,
1927                ctx->ctr_nonce, GCM_BLOCK_RFC4_NONCE_SIZE);
1928         memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_IV_OFFSET, req->iv,
1929                GCM_BLOCK_RFC4_IV_SIZE);
1930         req->iv = areq_ctx->ctr_iv;
1931         areq_ctx->assoclen -= GCM_BLOCK_RFC4_IV_SIZE;
1932 }
1933
1934 static int cc_proc_aead(struct aead_request *req,
1935                         enum drv_crypto_direction direct)
1936 {
1937         int rc = 0;
1938         int seq_len = 0;
1939         struct cc_hw_desc desc[MAX_AEAD_PROCESS_SEQ];
1940         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1941         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1942         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1943         struct device *dev = drvdata_to_dev(ctx->drvdata);
1944         struct cc_crypto_req cc_req = {};
1945
1946         dev_dbg(dev, "%s context=%p req=%p iv=%p src=%p src_ofs=%d dst=%p dst_ofs=%d cryptolen=%d\n",
1947                 ((direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Enc" : "Dec"),
1948                 ctx, req, req->iv, sg_virt(req->src), req->src->offset,
1949                 sg_virt(req->dst), req->dst->offset, req->cryptlen);
1950
1951         /* STAT_PHASE_0: Init and sanity checks */
1952
1953         /* Check data length according to mode */
1954         if (validate_data_size(ctx, direct, req)) {
1955                 dev_err(dev, "Unsupported crypt/assoc len %d/%d.\n",
1956                         req->cryptlen, areq_ctx->assoclen);
1957                 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_BLOCK_LEN);
1958                 return -EINVAL;
1959         }
1960
1961         /* Setup request structure */
1962         cc_req.user_cb = (void *)cc_aead_complete;
1963         cc_req.user_arg = (void *)req;
1964
1965         /* Setup request context */
1966         areq_ctx->gen_ctx.op_type = direct;
1967         areq_ctx->req_authsize = ctx->authsize;
1968         areq_ctx->cipher_mode = ctx->cipher_mode;
1969
1970         /* STAT_PHASE_1: Map buffers */
1971
1972         if (ctx->cipher_mode == DRV_CIPHER_CTR) {
1973                 /* Build CTR IV - Copy nonce from last 4 bytes in
1974                  * CTR key to first 4 bytes in CTR IV
1975                  */
1976                 memcpy(areq_ctx->ctr_iv, ctx->ctr_nonce,
1977                        CTR_RFC3686_NONCE_SIZE);
1978                 if (!areq_ctx->backup_giv) /*User none-generated IV*/
1979                         memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE,
1980                                req->iv, CTR_RFC3686_IV_SIZE);
1981                 /* Initialize counter portion of counter block */
1982                 *(__be32 *)(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE +
1983                             CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
1984
1985                 /* Replace with counter iv */
1986                 req->iv = areq_ctx->ctr_iv;
1987                 areq_ctx->hw_iv_size = CTR_RFC3686_BLOCK_SIZE;
1988         } else if ((ctx->cipher_mode == DRV_CIPHER_CCM) ||
1989                    (ctx->cipher_mode == DRV_CIPHER_GCTR)) {
1990                 areq_ctx->hw_iv_size = AES_BLOCK_SIZE;
1991                 if (areq_ctx->ctr_iv != req->iv) {
1992                         memcpy(areq_ctx->ctr_iv, req->iv,
1993                                crypto_aead_ivsize(tfm));
1994                         req->iv = areq_ctx->ctr_iv;
1995                 }
1996         }  else {
1997                 areq_ctx->hw_iv_size = crypto_aead_ivsize(tfm);
1998         }
1999
2000         if (ctx->cipher_mode == DRV_CIPHER_CCM) {
2001                 rc = config_ccm_adata(req);
2002                 if (rc) {
2003                         dev_dbg(dev, "config_ccm_adata() returned with a failure %d!",
2004                                 rc);
2005                         goto exit;
2006                 }
2007         } else {
2008                 areq_ctx->ccm_hdr_size = ccm_header_size_null;
2009         }
2010
2011         if (ctx->cipher_mode == DRV_CIPHER_GCTR) {
2012                 rc = config_gcm_context(req);
2013                 if (rc) {
2014                         dev_dbg(dev, "config_gcm_context() returned with a failure %d!",
2015                                 rc);
2016                         goto exit;
2017                 }
2018         }
2019
2020         rc = cc_map_aead_request(ctx->drvdata, req);
2021         if (rc) {
2022                 dev_err(dev, "map_request() failed\n");
2023                 goto exit;
2024         }
2025
2026         /* do we need to generate IV? */
2027         if (areq_ctx->backup_giv) {
2028                 /* set the DMA mapped IV address*/
2029                 if (ctx->cipher_mode == DRV_CIPHER_CTR) {
2030                         cc_req.ivgen_dma_addr[0] =
2031                                 areq_ctx->gen_ctx.iv_dma_addr +
2032                                 CTR_RFC3686_NONCE_SIZE;
2033                         cc_req.ivgen_dma_addr_len = 1;
2034                 } else if (ctx->cipher_mode == DRV_CIPHER_CCM) {
2035                         /* In ccm, the IV needs to exist both inside B0 and
2036                          * inside the counter.It is also copied to iv_dma_addr
2037                          * for other reasons (like returning it to the user).
2038                          * So, using 3 (identical) IV outputs.
2039                          */
2040                         cc_req.ivgen_dma_addr[0] =
2041                                 areq_ctx->gen_ctx.iv_dma_addr +
2042                                 CCM_BLOCK_IV_OFFSET;
2043                         cc_req.ivgen_dma_addr[1] =
2044                                 sg_dma_address(&areq_ctx->ccm_adata_sg) +
2045                                 CCM_B0_OFFSET + CCM_BLOCK_IV_OFFSET;
2046                         cc_req.ivgen_dma_addr[2] =
2047                                 sg_dma_address(&areq_ctx->ccm_adata_sg) +
2048                                 CCM_CTR_COUNT_0_OFFSET + CCM_BLOCK_IV_OFFSET;
2049                         cc_req.ivgen_dma_addr_len = 3;
2050                 } else {
2051                         cc_req.ivgen_dma_addr[0] =
2052                                 areq_ctx->gen_ctx.iv_dma_addr;
2053                         cc_req.ivgen_dma_addr_len = 1;
2054                 }
2055
2056                 /* set the IV size (8/16 B long)*/
2057                 cc_req.ivgen_size = crypto_aead_ivsize(tfm);
2058         }
2059
2060         /* STAT_PHASE_2: Create sequence */
2061
2062         /* Load MLLI tables to SRAM if necessary */
2063         cc_mlli_to_sram(req, desc, &seq_len);
2064
2065         /*TODO: move seq len by reference */
2066         switch (ctx->auth_mode) {
2067         case DRV_HASH_SHA1:
2068         case DRV_HASH_SHA256:
2069                 cc_hmac_authenc(req, desc, &seq_len);
2070                 break;
2071         case DRV_HASH_XCBC_MAC:
2072                 cc_xcbc_authenc(req, desc, &seq_len);
2073                 break;
2074         case DRV_HASH_NULL:
2075                 if (ctx->cipher_mode == DRV_CIPHER_CCM)
2076                         cc_ccm(req, desc, &seq_len);
2077                 if (ctx->cipher_mode == DRV_CIPHER_GCTR)
2078                         cc_gcm(req, desc, &seq_len);
2079                 break;
2080         default:
2081                 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode);
2082                 cc_unmap_aead_request(dev, req);
2083                 rc = -ENOTSUPP;
2084                 goto exit;
2085         }
2086
2087         /* STAT_PHASE_3: Lock HW and push sequence */
2088
2089         rc = cc_send_request(ctx->drvdata, &cc_req, desc, seq_len, &req->base);
2090
2091         if (rc != -EINPROGRESS && rc != -EBUSY) {
2092                 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
2093                 cc_unmap_aead_request(dev, req);
2094         }
2095
2096 exit:
2097         return rc;
2098 }
2099
2100 static int cc_aead_encrypt(struct aead_request *req)
2101 {
2102         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2103         int rc;
2104
2105         memset(areq_ctx, 0, sizeof(*areq_ctx));
2106
2107         /* No generated IV required */
2108         areq_ctx->backup_iv = req->iv;
2109         areq_ctx->assoclen = req->assoclen;
2110         areq_ctx->backup_giv = NULL;
2111         areq_ctx->is_gcm4543 = false;
2112
2113         areq_ctx->plaintext_authenticate_only = false;
2114
2115         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2116         if (rc != -EINPROGRESS && rc != -EBUSY)
2117                 req->iv = areq_ctx->backup_iv;
2118
2119         return rc;
2120 }
2121
2122 static int cc_rfc4309_ccm_encrypt(struct aead_request *req)
2123 {
2124         /* Very similar to cc_aead_encrypt() above. */
2125
2126         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2127         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2128         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2129         struct device *dev = drvdata_to_dev(ctx->drvdata);
2130         int rc = -EINVAL;
2131
2132         if (!valid_assoclen(req)) {
2133                 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2134                 goto out;
2135         }
2136
2137         memset(areq_ctx, 0, sizeof(*areq_ctx));
2138
2139         /* No generated IV required */
2140         areq_ctx->backup_iv = req->iv;
2141         areq_ctx->assoclen = req->assoclen;
2142         areq_ctx->backup_giv = NULL;
2143         areq_ctx->is_gcm4543 = true;
2144
2145         cc_proc_rfc4309_ccm(req);
2146
2147         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2148         if (rc != -EINPROGRESS && rc != -EBUSY)
2149                 req->iv = areq_ctx->backup_iv;
2150 out:
2151         return rc;
2152 }
2153
2154 static int cc_aead_decrypt(struct aead_request *req)
2155 {
2156         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2157         int rc;
2158
2159         memset(areq_ctx, 0, sizeof(*areq_ctx));
2160
2161         /* No generated IV required */
2162         areq_ctx->backup_iv = req->iv;
2163         areq_ctx->assoclen = req->assoclen;
2164         areq_ctx->backup_giv = NULL;
2165         areq_ctx->is_gcm4543 = false;
2166
2167         areq_ctx->plaintext_authenticate_only = false;
2168
2169         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2170         if (rc != -EINPROGRESS && rc != -EBUSY)
2171                 req->iv = areq_ctx->backup_iv;
2172
2173         return rc;
2174 }
2175
2176 static int cc_rfc4309_ccm_decrypt(struct aead_request *req)
2177 {
2178         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2179         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2180         struct device *dev = drvdata_to_dev(ctx->drvdata);
2181         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2182         int rc = -EINVAL;
2183
2184         if (!valid_assoclen(req)) {
2185                 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2186                 goto out;
2187         }
2188
2189         memset(areq_ctx, 0, sizeof(*areq_ctx));
2190
2191         /* No generated IV required */
2192         areq_ctx->backup_iv = req->iv;
2193         areq_ctx->assoclen = req->assoclen;
2194         areq_ctx->backup_giv = NULL;
2195
2196         areq_ctx->is_gcm4543 = true;
2197         cc_proc_rfc4309_ccm(req);
2198
2199         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2200         if (rc != -EINPROGRESS && rc != -EBUSY)
2201                 req->iv = areq_ctx->backup_iv;
2202
2203 out:
2204         return rc;
2205 }
2206
2207 static int cc_rfc4106_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
2208                                  unsigned int keylen)
2209 {
2210         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2211         struct device *dev = drvdata_to_dev(ctx->drvdata);
2212
2213         dev_dbg(dev, "%s()  keylen %d, key %p\n", __func__, keylen, key);
2214
2215         if (keylen < 4)
2216                 return -EINVAL;
2217
2218         keylen -= 4;
2219         memcpy(ctx->ctr_nonce, key + keylen, 4);
2220
2221         return cc_aead_setkey(tfm, key, keylen);
2222 }
2223
2224 static int cc_rfc4543_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
2225                                  unsigned int keylen)
2226 {
2227         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2228         struct device *dev = drvdata_to_dev(ctx->drvdata);
2229
2230         dev_dbg(dev, "%s()  keylen %d, key %p\n", __func__, keylen, key);
2231
2232         if (keylen < 4)
2233                 return -EINVAL;
2234
2235         keylen -= 4;
2236         memcpy(ctx->ctr_nonce, key + keylen, 4);
2237
2238         return cc_aead_setkey(tfm, key, keylen);
2239 }
2240
2241 static int cc_gcm_setauthsize(struct crypto_aead *authenc,
2242                               unsigned int authsize)
2243 {
2244         switch (authsize) {
2245         case 4:
2246         case 8:
2247         case 12:
2248         case 13:
2249         case 14:
2250         case 15:
2251         case 16:
2252                 break;
2253         default:
2254                 return -EINVAL;
2255         }
2256
2257         return cc_aead_setauthsize(authenc, authsize);
2258 }
2259
2260 static int cc_rfc4106_gcm_setauthsize(struct crypto_aead *authenc,
2261                                       unsigned int authsize)
2262 {
2263         struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
2264         struct device *dev = drvdata_to_dev(ctx->drvdata);
2265
2266         dev_dbg(dev, "authsize %d\n", authsize);
2267
2268         switch (authsize) {
2269         case 8:
2270         case 12:
2271         case 16:
2272                 break;
2273         default:
2274                 return -EINVAL;
2275         }
2276
2277         return cc_aead_setauthsize(authenc, authsize);
2278 }
2279
2280 static int cc_rfc4543_gcm_setauthsize(struct crypto_aead *authenc,
2281                                       unsigned int authsize)
2282 {
2283         struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
2284         struct device *dev = drvdata_to_dev(ctx->drvdata);
2285
2286         dev_dbg(dev, "authsize %d\n", authsize);
2287
2288         if (authsize != 16)
2289                 return -EINVAL;
2290
2291         return cc_aead_setauthsize(authenc, authsize);
2292 }
2293
2294 static int cc_rfc4106_gcm_encrypt(struct aead_request *req)
2295 {
2296         /* Very similar to cc_aead_encrypt() above. */
2297
2298         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2299         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2300         struct device *dev = drvdata_to_dev(ctx->drvdata);
2301         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2302         int rc = -EINVAL;
2303
2304         if (!valid_assoclen(req)) {
2305                 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2306                 goto out;
2307         }
2308
2309         memset(areq_ctx, 0, sizeof(*areq_ctx));
2310
2311         /* No generated IV required */
2312         areq_ctx->backup_iv = req->iv;
2313         areq_ctx->assoclen = req->assoclen;
2314         areq_ctx->backup_giv = NULL;
2315
2316         areq_ctx->plaintext_authenticate_only = false;
2317
2318         cc_proc_rfc4_gcm(req);
2319         areq_ctx->is_gcm4543 = true;
2320
2321         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2322         if (rc != -EINPROGRESS && rc != -EBUSY)
2323                 req->iv = areq_ctx->backup_iv;
2324 out:
2325         return rc;
2326 }
2327
2328 static int cc_rfc4543_gcm_encrypt(struct aead_request *req)
2329 {
2330         /* Very similar to cc_aead_encrypt() above. */
2331
2332         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2333         int rc;
2334
2335         memset(areq_ctx, 0, sizeof(*areq_ctx));
2336
2337         //plaintext is not encryped with rfc4543
2338         areq_ctx->plaintext_authenticate_only = true;
2339
2340         /* No generated IV required */
2341         areq_ctx->backup_iv = req->iv;
2342         areq_ctx->assoclen = req->assoclen;
2343         areq_ctx->backup_giv = NULL;
2344
2345         cc_proc_rfc4_gcm(req);
2346         areq_ctx->is_gcm4543 = true;
2347
2348         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2349         if (rc != -EINPROGRESS && rc != -EBUSY)
2350                 req->iv = areq_ctx->backup_iv;
2351
2352         return rc;
2353 }
2354
2355 static int cc_rfc4106_gcm_decrypt(struct aead_request *req)
2356 {
2357         /* Very similar to cc_aead_decrypt() above. */
2358
2359         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2360         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2361         struct device *dev = drvdata_to_dev(ctx->drvdata);
2362         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2363         int rc = -EINVAL;
2364
2365         if (!valid_assoclen(req)) {
2366                 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2367                 goto out;
2368         }
2369
2370         memset(areq_ctx, 0, sizeof(*areq_ctx));
2371
2372         /* No generated IV required */
2373         areq_ctx->backup_iv = req->iv;
2374         areq_ctx->assoclen = req->assoclen;
2375         areq_ctx->backup_giv = NULL;
2376
2377         areq_ctx->plaintext_authenticate_only = false;
2378
2379         cc_proc_rfc4_gcm(req);
2380         areq_ctx->is_gcm4543 = true;
2381
2382         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2383         if (rc != -EINPROGRESS && rc != -EBUSY)
2384                 req->iv = areq_ctx->backup_iv;
2385 out:
2386         return rc;
2387 }
2388
2389 static int cc_rfc4543_gcm_decrypt(struct aead_request *req)
2390 {
2391         /* Very similar to cc_aead_decrypt() above. */
2392
2393         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2394         int rc;
2395
2396         memset(areq_ctx, 0, sizeof(*areq_ctx));
2397
2398         //plaintext is not decryped with rfc4543
2399         areq_ctx->plaintext_authenticate_only = true;
2400
2401         /* No generated IV required */
2402         areq_ctx->backup_iv = req->iv;
2403         areq_ctx->assoclen = req->assoclen;
2404         areq_ctx->backup_giv = NULL;
2405
2406         cc_proc_rfc4_gcm(req);
2407         areq_ctx->is_gcm4543 = true;
2408
2409         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2410         if (rc != -EINPROGRESS && rc != -EBUSY)
2411                 req->iv = areq_ctx->backup_iv;
2412
2413         return rc;
2414 }
2415
2416 /* aead alg */
2417 static struct cc_alg_template aead_algs[] = {
2418         {
2419                 .name = "authenc(hmac(sha1),cbc(aes))",
2420                 .driver_name = "authenc-hmac-sha1-cbc-aes-ccree",
2421                 .blocksize = AES_BLOCK_SIZE,
2422                 .template_aead = {
2423                         .setkey = cc_aead_setkey,
2424                         .setauthsize = cc_aead_setauthsize,
2425                         .encrypt = cc_aead_encrypt,
2426                         .decrypt = cc_aead_decrypt,
2427                         .init = cc_aead_init,
2428                         .exit = cc_aead_exit,
2429                         .ivsize = AES_BLOCK_SIZE,
2430                         .maxauthsize = SHA1_DIGEST_SIZE,
2431                 },
2432                 .cipher_mode = DRV_CIPHER_CBC,
2433                 .flow_mode = S_DIN_to_AES,
2434                 .auth_mode = DRV_HASH_SHA1,
2435                 .min_hw_rev = CC_HW_REV_630,
2436                 .std_body = CC_STD_NIST,
2437         },
2438         {
2439                 .name = "authenc(hmac(sha1),cbc(des3_ede))",
2440                 .driver_name = "authenc-hmac-sha1-cbc-des3-ccree",
2441                 .blocksize = DES3_EDE_BLOCK_SIZE,
2442                 .template_aead = {
2443                         .setkey = cc_des3_aead_setkey,
2444                         .setauthsize = cc_aead_setauthsize,
2445                         .encrypt = cc_aead_encrypt,
2446                         .decrypt = cc_aead_decrypt,
2447                         .init = cc_aead_init,
2448                         .exit = cc_aead_exit,
2449                         .ivsize = DES3_EDE_BLOCK_SIZE,
2450                         .maxauthsize = SHA1_DIGEST_SIZE,
2451                 },
2452                 .cipher_mode = DRV_CIPHER_CBC,
2453                 .flow_mode = S_DIN_to_DES,
2454                 .auth_mode = DRV_HASH_SHA1,
2455                 .min_hw_rev = CC_HW_REV_630,
2456                 .std_body = CC_STD_NIST,
2457         },
2458         {
2459                 .name = "authenc(hmac(sha256),cbc(aes))",
2460                 .driver_name = "authenc-hmac-sha256-cbc-aes-ccree",
2461                 .blocksize = AES_BLOCK_SIZE,
2462                 .template_aead = {
2463                         .setkey = cc_aead_setkey,
2464                         .setauthsize = cc_aead_setauthsize,
2465                         .encrypt = cc_aead_encrypt,
2466                         .decrypt = cc_aead_decrypt,
2467                         .init = cc_aead_init,
2468                         .exit = cc_aead_exit,
2469                         .ivsize = AES_BLOCK_SIZE,
2470                         .maxauthsize = SHA256_DIGEST_SIZE,
2471                 },
2472                 .cipher_mode = DRV_CIPHER_CBC,
2473                 .flow_mode = S_DIN_to_AES,
2474                 .auth_mode = DRV_HASH_SHA256,
2475                 .min_hw_rev = CC_HW_REV_630,
2476                 .std_body = CC_STD_NIST,
2477         },
2478         {
2479                 .name = "authenc(hmac(sha256),cbc(des3_ede))",
2480                 .driver_name = "authenc-hmac-sha256-cbc-des3-ccree",
2481                 .blocksize = DES3_EDE_BLOCK_SIZE,
2482                 .template_aead = {
2483                         .setkey = cc_des3_aead_setkey,
2484                         .setauthsize = cc_aead_setauthsize,
2485                         .encrypt = cc_aead_encrypt,
2486                         .decrypt = cc_aead_decrypt,
2487                         .init = cc_aead_init,
2488                         .exit = cc_aead_exit,
2489                         .ivsize = DES3_EDE_BLOCK_SIZE,
2490                         .maxauthsize = SHA256_DIGEST_SIZE,
2491                 },
2492                 .cipher_mode = DRV_CIPHER_CBC,
2493                 .flow_mode = S_DIN_to_DES,
2494                 .auth_mode = DRV_HASH_SHA256,
2495                 .min_hw_rev = CC_HW_REV_630,
2496                 .std_body = CC_STD_NIST,
2497         },
2498         {
2499                 .name = "authenc(xcbc(aes),cbc(aes))",
2500                 .driver_name = "authenc-xcbc-aes-cbc-aes-ccree",
2501                 .blocksize = AES_BLOCK_SIZE,
2502                 .template_aead = {
2503                         .setkey = cc_aead_setkey,
2504                         .setauthsize = cc_aead_setauthsize,
2505                         .encrypt = cc_aead_encrypt,
2506                         .decrypt = cc_aead_decrypt,
2507                         .init = cc_aead_init,
2508                         .exit = cc_aead_exit,
2509                         .ivsize = AES_BLOCK_SIZE,
2510                         .maxauthsize = AES_BLOCK_SIZE,
2511                 },
2512                 .cipher_mode = DRV_CIPHER_CBC,
2513                 .flow_mode = S_DIN_to_AES,
2514                 .auth_mode = DRV_HASH_XCBC_MAC,
2515                 .min_hw_rev = CC_HW_REV_630,
2516                 .std_body = CC_STD_NIST,
2517         },
2518         {
2519                 .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
2520                 .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-ccree",
2521                 .blocksize = 1,
2522                 .template_aead = {
2523                         .setkey = cc_aead_setkey,
2524                         .setauthsize = cc_aead_setauthsize,
2525                         .encrypt = cc_aead_encrypt,
2526                         .decrypt = cc_aead_decrypt,
2527                         .init = cc_aead_init,
2528                         .exit = cc_aead_exit,
2529                         .ivsize = CTR_RFC3686_IV_SIZE,
2530                         .maxauthsize = SHA1_DIGEST_SIZE,
2531                 },
2532                 .cipher_mode = DRV_CIPHER_CTR,
2533                 .flow_mode = S_DIN_to_AES,
2534                 .auth_mode = DRV_HASH_SHA1,
2535                 .min_hw_rev = CC_HW_REV_630,
2536                 .std_body = CC_STD_NIST,
2537         },
2538         {
2539                 .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
2540                 .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-ccree",
2541                 .blocksize = 1,
2542                 .template_aead = {
2543                         .setkey = cc_aead_setkey,
2544                         .setauthsize = cc_aead_setauthsize,
2545                         .encrypt = cc_aead_encrypt,
2546                         .decrypt = cc_aead_decrypt,
2547                         .init = cc_aead_init,
2548                         .exit = cc_aead_exit,
2549                         .ivsize = CTR_RFC3686_IV_SIZE,
2550                         .maxauthsize = SHA256_DIGEST_SIZE,
2551                 },
2552                 .cipher_mode = DRV_CIPHER_CTR,
2553                 .flow_mode = S_DIN_to_AES,
2554                 .auth_mode = DRV_HASH_SHA256,
2555                 .min_hw_rev = CC_HW_REV_630,
2556                 .std_body = CC_STD_NIST,
2557         },
2558         {
2559                 .name = "authenc(xcbc(aes),rfc3686(ctr(aes)))",
2560                 .driver_name = "authenc-xcbc-aes-rfc3686-ctr-aes-ccree",
2561                 .blocksize = 1,
2562                 .template_aead = {
2563                         .setkey = cc_aead_setkey,
2564                         .setauthsize = cc_aead_setauthsize,
2565                         .encrypt = cc_aead_encrypt,
2566                         .decrypt = cc_aead_decrypt,
2567                         .init = cc_aead_init,
2568                         .exit = cc_aead_exit,
2569                         .ivsize = CTR_RFC3686_IV_SIZE,
2570                         .maxauthsize = AES_BLOCK_SIZE,
2571                 },
2572                 .cipher_mode = DRV_CIPHER_CTR,
2573                 .flow_mode = S_DIN_to_AES,
2574                 .auth_mode = DRV_HASH_XCBC_MAC,
2575                 .min_hw_rev = CC_HW_REV_630,
2576                 .std_body = CC_STD_NIST,
2577         },
2578         {
2579                 .name = "ccm(aes)",
2580                 .driver_name = "ccm-aes-ccree",
2581                 .blocksize = 1,
2582                 .template_aead = {
2583                         .setkey = cc_aead_setkey,
2584                         .setauthsize = cc_ccm_setauthsize,
2585                         .encrypt = cc_aead_encrypt,
2586                         .decrypt = cc_aead_decrypt,
2587                         .init = cc_aead_init,
2588                         .exit = cc_aead_exit,
2589                         .ivsize = AES_BLOCK_SIZE,
2590                         .maxauthsize = AES_BLOCK_SIZE,
2591                 },
2592                 .cipher_mode = DRV_CIPHER_CCM,
2593                 .flow_mode = S_DIN_to_AES,
2594                 .auth_mode = DRV_HASH_NULL,
2595                 .min_hw_rev = CC_HW_REV_630,
2596                 .std_body = CC_STD_NIST,
2597         },
2598         {
2599                 .name = "rfc4309(ccm(aes))",
2600                 .driver_name = "rfc4309-ccm-aes-ccree",
2601                 .blocksize = 1,
2602                 .template_aead = {
2603                         .setkey = cc_rfc4309_ccm_setkey,
2604                         .setauthsize = cc_rfc4309_ccm_setauthsize,
2605                         .encrypt = cc_rfc4309_ccm_encrypt,
2606                         .decrypt = cc_rfc4309_ccm_decrypt,
2607                         .init = cc_aead_init,
2608                         .exit = cc_aead_exit,
2609                         .ivsize = CCM_BLOCK_IV_SIZE,
2610                         .maxauthsize = AES_BLOCK_SIZE,
2611                 },
2612                 .cipher_mode = DRV_CIPHER_CCM,
2613                 .flow_mode = S_DIN_to_AES,
2614                 .auth_mode = DRV_HASH_NULL,
2615                 .min_hw_rev = CC_HW_REV_630,
2616                 .std_body = CC_STD_NIST,
2617         },
2618         {
2619                 .name = "gcm(aes)",
2620                 .driver_name = "gcm-aes-ccree",
2621                 .blocksize = 1,
2622                 .template_aead = {
2623                         .setkey = cc_aead_setkey,
2624                         .setauthsize = cc_gcm_setauthsize,
2625                         .encrypt = cc_aead_encrypt,
2626                         .decrypt = cc_aead_decrypt,
2627                         .init = cc_aead_init,
2628                         .exit = cc_aead_exit,
2629                         .ivsize = 12,
2630                         .maxauthsize = AES_BLOCK_SIZE,
2631                 },
2632                 .cipher_mode = DRV_CIPHER_GCTR,
2633                 .flow_mode = S_DIN_to_AES,
2634                 .auth_mode = DRV_HASH_NULL,
2635                 .min_hw_rev = CC_HW_REV_630,
2636                 .std_body = CC_STD_NIST,
2637         },
2638         {
2639                 .name = "rfc4106(gcm(aes))",
2640                 .driver_name = "rfc4106-gcm-aes-ccree",
2641                 .blocksize = 1,
2642                 .template_aead = {
2643                         .setkey = cc_rfc4106_gcm_setkey,
2644                         .setauthsize = cc_rfc4106_gcm_setauthsize,
2645                         .encrypt = cc_rfc4106_gcm_encrypt,
2646                         .decrypt = cc_rfc4106_gcm_decrypt,
2647                         .init = cc_aead_init,
2648                         .exit = cc_aead_exit,
2649                         .ivsize = GCM_BLOCK_RFC4_IV_SIZE,
2650                         .maxauthsize = AES_BLOCK_SIZE,
2651                 },
2652                 .cipher_mode = DRV_CIPHER_GCTR,
2653                 .flow_mode = S_DIN_to_AES,
2654                 .auth_mode = DRV_HASH_NULL,
2655                 .min_hw_rev = CC_HW_REV_630,
2656                 .std_body = CC_STD_NIST,
2657         },
2658         {
2659                 .name = "rfc4543(gcm(aes))",
2660                 .driver_name = "rfc4543-gcm-aes-ccree",
2661                 .blocksize = 1,
2662                 .template_aead = {
2663                         .setkey = cc_rfc4543_gcm_setkey,
2664                         .setauthsize = cc_rfc4543_gcm_setauthsize,
2665                         .encrypt = cc_rfc4543_gcm_encrypt,
2666                         .decrypt = cc_rfc4543_gcm_decrypt,
2667                         .init = cc_aead_init,
2668                         .exit = cc_aead_exit,
2669                         .ivsize = GCM_BLOCK_RFC4_IV_SIZE,
2670                         .maxauthsize = AES_BLOCK_SIZE,
2671                 },
2672                 .cipher_mode = DRV_CIPHER_GCTR,
2673                 .flow_mode = S_DIN_to_AES,
2674                 .auth_mode = DRV_HASH_NULL,
2675                 .min_hw_rev = CC_HW_REV_630,
2676                 .std_body = CC_STD_NIST,
2677         },
2678 };
2679
2680 static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl,
2681                                                 struct device *dev)
2682 {
2683         struct cc_crypto_alg *t_alg;
2684         struct aead_alg *alg;
2685
2686         t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
2687         if (!t_alg)
2688                 return ERR_PTR(-ENOMEM);
2689
2690         alg = &tmpl->template_aead;
2691
2692         snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
2693         snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
2694                  tmpl->driver_name);
2695         alg->base.cra_module = THIS_MODULE;
2696         alg->base.cra_priority = CC_CRA_PRIO;
2697
2698         alg->base.cra_ctxsize = sizeof(struct cc_aead_ctx);
2699         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
2700         alg->init = cc_aead_init;
2701         alg->exit = cc_aead_exit;
2702
2703         t_alg->aead_alg = *alg;
2704
2705         t_alg->cipher_mode = tmpl->cipher_mode;
2706         t_alg->flow_mode = tmpl->flow_mode;
2707         t_alg->auth_mode = tmpl->auth_mode;
2708
2709         return t_alg;
2710 }
2711
2712 int cc_aead_free(struct cc_drvdata *drvdata)
2713 {
2714         struct cc_crypto_alg *t_alg, *n;
2715         struct cc_aead_handle *aead_handle =
2716                 (struct cc_aead_handle *)drvdata->aead_handle;
2717
2718         if (aead_handle) {
2719                 /* Remove registered algs */
2720                 list_for_each_entry_safe(t_alg, n, &aead_handle->aead_list,
2721                                          entry) {
2722                         crypto_unregister_aead(&t_alg->aead_alg);
2723                         list_del(&t_alg->entry);
2724                         kfree(t_alg);
2725                 }
2726                 kfree(aead_handle);
2727                 drvdata->aead_handle = NULL;
2728         }
2729
2730         return 0;
2731 }
2732
2733 int cc_aead_alloc(struct cc_drvdata *drvdata)
2734 {
2735         struct cc_aead_handle *aead_handle;
2736         struct cc_crypto_alg *t_alg;
2737         int rc = -ENOMEM;
2738         int alg;
2739         struct device *dev = drvdata_to_dev(drvdata);
2740
2741         aead_handle = kmalloc(sizeof(*aead_handle), GFP_KERNEL);
2742         if (!aead_handle) {
2743                 rc = -ENOMEM;
2744                 goto fail0;
2745         }
2746
2747         INIT_LIST_HEAD(&aead_handle->aead_list);
2748         drvdata->aead_handle = aead_handle;
2749
2750         aead_handle->sram_workspace_addr = cc_sram_alloc(drvdata,
2751                                                          MAX_HMAC_DIGEST_SIZE);
2752
2753         if (aead_handle->sram_workspace_addr == NULL_SRAM_ADDR) {
2754                 dev_err(dev, "SRAM pool exhausted\n");
2755                 rc = -ENOMEM;
2756                 goto fail1;
2757         }
2758
2759         /* Linux crypto */
2760         for (alg = 0; alg < ARRAY_SIZE(aead_algs); alg++) {
2761                 if ((aead_algs[alg].min_hw_rev > drvdata->hw_rev) ||
2762                     !(drvdata->std_bodies & aead_algs[alg].std_body))
2763                         continue;
2764
2765                 t_alg = cc_create_aead_alg(&aead_algs[alg], dev);
2766                 if (IS_ERR(t_alg)) {
2767                         rc = PTR_ERR(t_alg);
2768                         dev_err(dev, "%s alg allocation failed\n",
2769                                 aead_algs[alg].driver_name);
2770                         goto fail1;
2771                 }
2772                 t_alg->drvdata = drvdata;
2773                 rc = crypto_register_aead(&t_alg->aead_alg);
2774                 if (rc) {
2775                         dev_err(dev, "%s alg registration failed\n",
2776                                 t_alg->aead_alg.base.cra_driver_name);
2777                         goto fail2;
2778                 } else {
2779                         list_add_tail(&t_alg->entry, &aead_handle->aead_list);
2780                         dev_dbg(dev, "Registered %s\n",
2781                                 t_alg->aead_alg.base.cra_driver_name);
2782                 }
2783         }
2784
2785         return 0;
2786
2787 fail2:
2788         kfree(t_alg);
2789 fail1:
2790         cc_aead_free(drvdata);
2791 fail0:
2792         return rc;
2793 }