drivers/crypto/bcm/cipher.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright 2016 Broadcom
   4  */
   5
   6 #include <linux/err.h>
   7 #include <linux/module.h>
   8 #include <linux/init.h>
   9 #include <linux/errno.h>
  10 #include <linux/kernel.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/platform_device.h>
  13 #include <linux/scatterlist.h>
  14 #include <linux/crypto.h>
  15 #include <linux/kthread.h>
  16 #include <linux/rtnetlink.h>
  17 #include <linux/sched.h>
  18 #include <linux/of_address.h>
  19 #include <linux/of_device.h>
  20 #include <linux/io.h>
  21 #include <linux/bitops.h>
  22
  23 #include <crypto/algapi.h>
  24 #include <crypto/aead.h>
  25 #include <crypto/internal/aead.h>
  26 #include <crypto/aes.h>
  27 #include <crypto/internal/des.h>
  28 #include <crypto/hmac.h>
  29 #include <crypto/md5.h>
  30 #include <crypto/authenc.h>
  31 #include <crypto/skcipher.h>
  32 #include <crypto/hash.h>
  33 #include <crypto/sha1.h>
  34 #include <crypto/sha2.h>
  35 #include <crypto/sha3.h>
  36
  37 #include "util.h"
  38 #include "cipher.h"
  39 #include "spu.h"
  40 #include "spum.h"
  41 #include "spu2.h"
  42
  43 /* ================= Device Structure ================== */
  44
  45 struct bcm_device_private iproc_priv;
  46
  47 /* ==================== Parameters ===================== */
  48
  49 int flow_debug_logging;
  50 module_param(flow_debug_logging, int, 0644);
  51 MODULE_PARM_DESC(flow_debug_logging, "Enable Flow Debug Logging");
  52
  53 int packet_debug_logging;
  54 module_param(packet_debug_logging, int, 0644);
  55 MODULE_PARM_DESC(packet_debug_logging, "Enable Packet Debug Logging");
  56
  57 int debug_logging_sleep;
  58 module_param(debug_logging_sleep, int, 0644);
  59 MODULE_PARM_DESC(debug_logging_sleep, "Packet Debug Logging Sleep");
  60
  61 /*
  62  * The value of these module parameters is used to set the priority for each
  63  * algo type when this driver registers algos with the kernel crypto API.
  64  * To use a priority other than the default, set the priority in the insmod or
  65  * modprobe. Changing the module priority after init time has no effect.
  66  *
  67  * The default priorities are chosen to be lower (less preferred) than ARMv8 CE
  68  * algos, but more preferred than generic software algos.
  69  */
  70 static int cipher_pri = 150;
  71 module_param(cipher_pri, int, 0644);
  72 MODULE_PARM_DESC(cipher_pri, "Priority for cipher algos");
  73
  74 static int hash_pri = 100;
  75 module_param(hash_pri, int, 0644);
  76 MODULE_PARM_DESC(hash_pri, "Priority for hash algos");
  77
  78 static int aead_pri = 150;
  79 module_param(aead_pri, int, 0644);
  80 MODULE_PARM_DESC(aead_pri, "Priority for AEAD algos");
  81
  82 /* A type 3 BCM header, expected to precede the SPU header for SPU-M.
  83  * Bits 3 and 4 in the first byte encode the channel number (the dma ringset).
  84  * 0x60 - ring 0
  85  * 0x68 - ring 1
  86  * 0x70 - ring 2
  87  * 0x78 - ring 3
  88  */
  89 static char BCMHEADER[] = { 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28 };
  90 /*
  91  * Some SPU hw does not use BCM header on SPU messages. So BCM_HDR_LEN
  92  * is set dynamically after reading SPU type from device tree.
  93  */
  94 #define BCM_HDR_LEN  iproc_priv.bcm_hdr_len
  95
  96 /* min and max time to sleep before retrying when mbox queue is full. usec */
  97 #define MBOX_SLEEP_MIN  800
  98 #define MBOX_SLEEP_MAX 1000
  99
 100 /**
 101  * select_channel() - Select a SPU channel to handle a crypto request. Selects
 102  * channel in round robin order.
 103  *
 104  * Return:  channel index
 105  */
 106 static u8 select_channel(void)
 107 {
 108         u8 chan_idx = atomic_inc_return(&iproc_priv.next_chan);
 109
 110         return chan_idx % iproc_priv.spu.num_chan;
 111 }
 112
 113 /**
 114  * spu_skcipher_rx_sg_create() - Build up the scatterlist of buffers used to
 115  * receive a SPU response message for an skcipher request. Includes buffers to
 116  * catch SPU message headers and the response data.
 117  * @mssg:       mailbox message containing the receive sg
 118  * @rctx:       crypto request context
 119  * @rx_frag_num: number of scatterlist elements required to hold the
 120  *              SPU response message
 121  * @chunksize:  Number of bytes of response data expected
 122  * @stat_pad_len: Number of bytes required to pad the STAT field to
 123  *              a 4-byte boundary
 124  *
 125  * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
 126  * when the request completes, whether the request is handled successfully or
 127  * there is an error.
 128  *
 129  * Returns:
 130  *   0 if successful
 131  *   < 0 if an error
 132  */
 133 static int
 134 spu_skcipher_rx_sg_create(struct brcm_message *mssg,
 135                             struct iproc_reqctx_s *rctx,
 136                             u8 rx_frag_num,
 137                             unsigned int chunksize, u32 stat_pad_len)
 138 {
 139         struct spu_hw *spu = &iproc_priv.spu;
 140         struct scatterlist *sg; /* used to build sgs in mbox message */
 141         struct iproc_ctx_s *ctx = rctx->ctx;
 142         u32 datalen;            /* Number of bytes of response data expected */
 143
 144         mssg->spu.dst = kcalloc(rx_frag_num, sizeof(struct scatterlist),
 145                                 rctx->gfp);
 146         if (!mssg->spu.dst)
 147                 return -ENOMEM;
 148
 149         sg = mssg->spu.dst;
 150         sg_init_table(sg, rx_frag_num);
 151         /* Space for SPU message header */
 152         sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len);
 153
 154         /* If XTS tweak in payload, add buffer to receive encrypted tweak */
 155         if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
 156             spu->spu_xts_tweak_in_payload())
 157                 sg_set_buf(sg++, rctx->msg_buf.c.supdt_tweak,
 158                            SPU_XTS_TWEAK_SIZE);
 159
 160         /* Copy in each dst sg entry from request, up to chunksize */
 161         datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip,
 162                                  rctx->dst_nents, chunksize);
 163         if (datalen < chunksize) {
 164                 pr_err("%s(): failed to copy dst sg to mbox msg. chunksize %u, datalen %u",
 165                        __func__, chunksize, datalen);
 166                 return -EFAULT;
 167         }
 168
 169         if (stat_pad_len)
 170                 sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len);
 171
 172         memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN);
 173         sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len());
 174
 175         return 0;
 176 }
 177
 178 /**
 179  * spu_skcipher_tx_sg_create() - Build up the scatterlist of buffers used to
 180  * send a SPU request message for an skcipher request. Includes SPU message
 181  * headers and the request data.
 182  * @mssg:       mailbox message containing the transmit sg
 183  * @rctx:       crypto request context
 184  * @tx_frag_num: number of scatterlist elements required to construct the
 185  *              SPU request message
 186  * @chunksize:  Number of bytes of request data
 187  * @pad_len:    Number of pad bytes
 188  *
 189  * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
 190  * when the request completes, whether the request is handled successfully or
 191  * there is an error.
 192  *
 193  * Returns:
 194  *   0 if successful
 195  *   < 0 if an error
 196  */
 197 static int
 198 spu_skcipher_tx_sg_create(struct brcm_message *mssg,
 199                             struct iproc_reqctx_s *rctx,
 200                             u8 tx_frag_num, unsigned int chunksize, u32 pad_len)
 201 {
 202         struct spu_hw *spu = &iproc_priv.spu;
 203         struct scatterlist *sg; /* used to build sgs in mbox message */
 204         struct iproc_ctx_s *ctx = rctx->ctx;
 205         u32 datalen;            /* Number of bytes of response data expected */
 206         u32 stat_len;
 207
 208         mssg->spu.src = kcalloc(tx_frag_num, sizeof(struct scatterlist),
 209                                 rctx->gfp);
 210         if (unlikely(!mssg->spu.src))
 211                 return -ENOMEM;
 212
 213         sg = mssg->spu.src;
 214         sg_init_table(sg, tx_frag_num);
 215
 216         sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr,
 217                    BCM_HDR_LEN + ctx->spu_req_hdr_len);
 218
 219         /* if XTS tweak in payload, copy from IV (where crypto API puts it) */
 220         if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
 221             spu->spu_xts_tweak_in_payload())
 222                 sg_set_buf(sg++, rctx->msg_buf.iv_ctr, SPU_XTS_TWEAK_SIZE);
 223
 224         /* Copy in each src sg entry from request, up to chunksize */
 225         datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip,
 226                                  rctx->src_nents, chunksize);
 227         if (unlikely(datalen < chunksize)) {
 228                 pr_err("%s(): failed to copy src sg to mbox msg",
 229                        __func__);
 230                 return -EFAULT;
 231         }
 232
 233         if (pad_len)
 234                 sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len);
 235
 236         stat_len = spu->spu_tx_status_len();
 237         if (stat_len) {
 238                 memset(rctx->msg_buf.tx_stat, 0, stat_len);
 239                 sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len);
 240         }
 241         return 0;
 242 }
 243
 244 static int mailbox_send_message(struct brcm_message *mssg, u32 flags,
 245                                 u8 chan_idx)
 246 {
 247         int err;
 248         int retry_cnt = 0;
 249         struct device *dev = &(iproc_priv.pdev->dev);
 250
 251         err = mbox_send_message(iproc_priv.mbox[chan_idx], mssg);
 252         if (flags & CRYPTO_TFM_REQ_MAY_SLEEP) {
 253                 while ((err == -ENOBUFS) && (retry_cnt < SPU_MB_RETRY_MAX)) {
 254                         /*
 255                          * Mailbox queue is full. Since MAY_SLEEP is set, assume
 256                          * not in atomic context and we can wait and try again.
 257                          */
 258                         retry_cnt++;
 259                         usleep_range(MBOX_SLEEP_MIN, MBOX_SLEEP_MAX);
 260                         err = mbox_send_message(iproc_priv.mbox[chan_idx],
 261                                                 mssg);
 262                         atomic_inc(&iproc_priv.mb_no_spc);
 263                 }
 264         }
 265         if (err < 0) {
 266                 atomic_inc(&iproc_priv.mb_send_fail);
 267                 return err;
 268         }
 269
 270         /* Check error returned by mailbox controller */
 271         err = mssg->error;
 272         if (unlikely(err < 0)) {
 273                 dev_err(dev, "message error %d", err);
 274                 /* Signal txdone for mailbox channel */
 275         }
 276
 277         /* Signal txdone for mailbox channel */
 278         mbox_client_txdone(iproc_priv.mbox[chan_idx], err);
 279         return err;
 280 }
 281
 282 /**
 283  * handle_skcipher_req() - Submit as much of a block cipher request as fits in
 284  * a single SPU request message, starting at the current position in the request
 285  * data.
 286  * @rctx:       Crypto request context
 287  *
 288  * This may be called on the crypto API thread, or, when a request is so large
 289  * it must be broken into multiple SPU messages, on the thread used to invoke
 290  * the response callback. When requests are broken into multiple SPU
 291  * messages, we assume subsequent messages depend on previous results, and
 292  * thus always wait for previous results before submitting the next message.
 293  * Because requests are submitted in lock step like this, there is no need
 294  * to synchronize access to request data structures.
 295  *
 296  * Return: -EINPROGRESS: request has been accepted and result will be returned
 297  *                       asynchronously
 298  *         Any other value indicates an error
 299  */
 300 static int handle_skcipher_req(struct iproc_reqctx_s *rctx)
 301 {
 302         struct spu_hw *spu = &iproc_priv.spu;
 303         struct crypto_async_request *areq = rctx->parent;
 304         struct skcipher_request *req =
 305             container_of(areq, struct skcipher_request, base);
 306         struct iproc_ctx_s *ctx = rctx->ctx;
 307         struct spu_cipher_parms cipher_parms;
 308         int err;
 309         unsigned int chunksize; /* Num bytes of request to submit */
 310         int remaining;  /* Bytes of request still to process */
 311         int chunk_start;        /* Beginning of data for current SPU msg */
 312
 313         /* IV or ctr value to use in this SPU msg */
 314         u8 local_iv_ctr[MAX_IV_SIZE];
 315         u32 stat_pad_len;       /* num bytes to align status field */
 316         u32 pad_len;            /* total length of all padding */
 317         struct brcm_message *mssg;      /* mailbox message */
 318
 319         /* number of entries in src and dst sg in mailbox message. */
 320         u8 rx_frag_num = 2;     /* response header and STATUS */
 321         u8 tx_frag_num = 1;     /* request header */
 322
 323         flow_log("%s\n", __func__);
 324
 325         cipher_parms.alg = ctx->cipher.alg;
 326         cipher_parms.mode = ctx->cipher.mode;
 327         cipher_parms.type = ctx->cipher_type;
 328         cipher_parms.key_len = ctx->enckeylen;
 329         cipher_parms.key_buf = ctx->enckey;
 330         cipher_parms.iv_buf = local_iv_ctr;
 331         cipher_parms.iv_len = rctx->iv_ctr_len;
 332
 333         mssg = &rctx->mb_mssg;
 334         chunk_start = rctx->src_sent;
 335         remaining = rctx->total_todo - chunk_start;
 336
 337         /* determine the chunk we are breaking off and update the indexes */
 338         if ((ctx->max_payload != SPU_MAX_PAYLOAD_INF) &&
 339             (remaining > ctx->max_payload))
 340                 chunksize = ctx->max_payload;
 341         else
 342                 chunksize = remaining;
 343
 344         rctx->src_sent += chunksize;
 345         rctx->total_sent = rctx->src_sent;
 346
 347         /* Count number of sg entries to be included in this request */
 348         rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize);
 349         rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize);
 350
 351         if ((ctx->cipher.mode == CIPHER_MODE_CBC) &&
 352             rctx->is_encrypt && chunk_start)
 353                 /*
 354                  * Encrypting non-first first chunk. Copy last block of
 355                  * previous result to IV for this chunk.
 356                  */
 357                 sg_copy_part_to_buf(req->dst, rctx->msg_buf.iv_ctr,
 358                                     rctx->iv_ctr_len,
 359                                     chunk_start - rctx->iv_ctr_len);
 360
 361         if (rctx->iv_ctr_len) {
 362                 /* get our local copy of the iv */
 363                 __builtin_memcpy(local_iv_ctr, rctx->msg_buf.iv_ctr,
 364                                  rctx->iv_ctr_len);
 365
 366                 /* generate the next IV if possible */
 367                 if ((ctx->cipher.mode == CIPHER_MODE_CBC) &&
 368                     !rctx->is_encrypt) {
 369                         /*
 370                          * CBC Decrypt: next IV is the last ciphertext block in
 371                          * this chunk
 372                          */
 373                         sg_copy_part_to_buf(req->src, rctx->msg_buf.iv_ctr,
 374                                             rctx->iv_ctr_len,
 375                                             rctx->src_sent - rctx->iv_ctr_len);
 376                 } else if (ctx->cipher.mode == CIPHER_MODE_CTR) {
 377                         /*
 378                          * The SPU hardware increments the counter once for
 379                          * each AES block of 16 bytes. So update the counter
 380                          * for the next chunk, if there is one. Note that for
 381                          * this chunk, the counter has already been copied to
 382                          * local_iv_ctr. We can assume a block size of 16,
 383                          * because we only support CTR mode for AES, not for
 384                          * any other cipher alg.
 385                          */
 386                         add_to_ctr(rctx->msg_buf.iv_ctr, chunksize >> 4);
 387                 }
 388         }
 389
 390         if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
 391                 flow_log("max_payload infinite\n");
 392         else
 393                 flow_log("max_payload %u\n", ctx->max_payload);
 394
 395         flow_log("sent:%u start:%u remains:%u size:%u\n",
 396                  rctx->src_sent, chunk_start, remaining, chunksize);
 397
 398         /* Copy SPU header template created at setkey time */
 399         memcpy(rctx->msg_buf.bcm_spu_req_hdr, ctx->bcm_spu_req_hdr,
 400                sizeof(rctx->msg_buf.bcm_spu_req_hdr));
 401
 402         spu->spu_cipher_req_finish(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
 403                                    ctx->spu_req_hdr_len, !(rctx->is_encrypt),
 404                                    &cipher_parms, chunksize);
 405
 406         atomic64_add(chunksize, &iproc_priv.bytes_out);
 407
 408         stat_pad_len = spu->spu_wordalign_padlen(chunksize);
 409         if (stat_pad_len)
 410                 rx_frag_num++;
 411         pad_len = stat_pad_len;
 412         if (pad_len) {
 413                 tx_frag_num++;
 414                 spu->spu_request_pad(rctx->msg_buf.spu_req_pad, 0,
 415                                      0, ctx->auth.alg, ctx->auth.mode,
 416                                      rctx->total_sent, stat_pad_len);
 417         }
 418
 419         spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
 420                               ctx->spu_req_hdr_len);
 421         packet_log("payload:\n");
 422         dump_sg(rctx->src_sg, rctx->src_skip, chunksize);
 423         packet_dump("   pad: ", rctx->msg_buf.spu_req_pad, pad_len);
 424
 425         /*
 426          * Build mailbox message containing SPU request msg and rx buffers
 427          * to catch response message
 428          */
 429         memset(mssg, 0, sizeof(*mssg));
 430         mssg->type = BRCM_MESSAGE_SPU;
 431         mssg->ctx = rctx;       /* Will be returned in response */
 432
 433         /* Create rx scatterlist to catch result */
 434         rx_frag_num += rctx->dst_nents;
 435
 436         if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
 437             spu->spu_xts_tweak_in_payload())
 438                 rx_frag_num++;  /* extra sg to insert tweak */
 439
 440         err = spu_skcipher_rx_sg_create(mssg, rctx, rx_frag_num, chunksize,
 441                                           stat_pad_len);
 442         if (err)
 443                 return err;
 444
 445         /* Create tx scatterlist containing SPU request message */
 446         tx_frag_num += rctx->src_nents;
 447         if (spu->spu_tx_status_len())
 448                 tx_frag_num++;
 449
 450         if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
 451             spu->spu_xts_tweak_in_payload())
 452                 tx_frag_num++;  /* extra sg to insert tweak */
 453
 454         err = spu_skcipher_tx_sg_create(mssg, rctx, tx_frag_num, chunksize,
 455                                           pad_len);
 456         if (err)
 457                 return err;
 458
 459         err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx);
 460         if (unlikely(err < 0))
 461                 return err;
 462
 463         return -EINPROGRESS;
 464 }
 465
 466 /**
 467  * handle_skcipher_resp() - Process a block cipher SPU response. Updates the
 468  * total received count for the request and updates global stats.
 469  * @rctx:       Crypto request context
 470  */
 471 static void handle_skcipher_resp(struct iproc_reqctx_s *rctx)
 472 {
 473         struct spu_hw *spu = &iproc_priv.spu;
 474         struct crypto_async_request *areq = rctx->parent;
 475         struct skcipher_request *req = skcipher_request_cast(areq);
 476         struct iproc_ctx_s *ctx = rctx->ctx;
 477         u32 payload_len;
 478
 479         /* See how much data was returned */
 480         payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr);
 481
 482         /*
 483          * In XTS mode, the first SPU_XTS_TWEAK_SIZE bytes may be the
 484          * encrypted tweak ("i") value; we don't count those.
 485          */
 486         if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
 487             spu->spu_xts_tweak_in_payload() &&
 488             (payload_len >= SPU_XTS_TWEAK_SIZE))
 489                 payload_len -= SPU_XTS_TWEAK_SIZE;
 490
 491         atomic64_add(payload_len, &iproc_priv.bytes_in);
 492
 493         flow_log("%s() offset: %u, bd_len: %u BD:\n",
 494                  __func__, rctx->total_received, payload_len);
 495
 496         dump_sg(req->dst, rctx->total_received, payload_len);
 497
 498         rctx->total_received += payload_len;
 499         if (rctx->total_received == rctx->total_todo) {
 500                 atomic_inc(&iproc_priv.op_counts[SPU_OP_CIPHER]);
 501                 atomic_inc(
 502                    &iproc_priv.cipher_cnt[ctx->cipher.alg][ctx->cipher.mode]);
 503         }
 504 }
 505
 506 /**
 507  * spu_ahash_rx_sg_create() - Build up the scatterlist of buffers used to
 508  * receive a SPU response message for an ahash request.
 509  * @mssg:       mailbox message containing the receive sg
 510  * @rctx:       crypto request context
 511  * @rx_frag_num: number of scatterlist elements required to hold the
 512  *              SPU response message
 513  * @digestsize: length of hash digest, in bytes
 514  * @stat_pad_len: Number of bytes required to pad the STAT field to
 515  *              a 4-byte boundary
 516  *
 517  * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
 518  * when the request completes, whether the request is handled successfully or
 519  * there is an error.
 520  *
 521  * Return:
 522  *   0 if successful
 523  *   < 0 if an error
 524  */
 525 static int
 526 spu_ahash_rx_sg_create(struct brcm_message *mssg,
 527                        struct iproc_reqctx_s *rctx,
 528                        u8 rx_frag_num, unsigned int digestsize,
 529                        u32 stat_pad_len)
 530 {
 531         struct spu_hw *spu = &iproc_priv.spu;
 532         struct scatterlist *sg; /* used to build sgs in mbox message */
 533         struct iproc_ctx_s *ctx = rctx->ctx;
 534
 535         mssg->spu.dst = kcalloc(rx_frag_num, sizeof(struct scatterlist),
 536                                 rctx->gfp);
 537         if (!mssg->spu.dst)
 538                 return -ENOMEM;
 539
 540         sg = mssg->spu.dst;
 541         sg_init_table(sg, rx_frag_num);
 542         /* Space for SPU message header */
 543         sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len);
 544
 545         /* Space for digest */
 546         sg_set_buf(sg++, rctx->msg_buf.digest, digestsize);
 547
 548         if (stat_pad_len)
 549                 sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len);
 550
 551         memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN);
 552         sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len());
 553         return 0;
 554 }
 555
 556 /**
 557  * spu_ahash_tx_sg_create() -  Build up the scatterlist of buffers used to send
 558  * a SPU request message for an ahash request. Includes SPU message headers and
 559  * the request data.
 560  * @mssg:       mailbox message containing the transmit sg
 561  * @rctx:       crypto request context
 562  * @tx_frag_num: number of scatterlist elements required to construct the
 563  *              SPU request message
 564  * @spu_hdr_len: length in bytes of SPU message header
 565  * @hash_carry_len: Number of bytes of data carried over from previous req
 566  * @new_data_len: Number of bytes of new request data
 567  * @pad_len:    Number of pad bytes
 568  *
 569  * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
 570  * when the request completes, whether the request is handled successfully or
 571  * there is an error.
 572  *
 573  * Return:
 574  *   0 if successful
 575  *   < 0 if an error
 576  */
 577 static int
 578 spu_ahash_tx_sg_create(struct brcm_message *mssg,
 579                        struct iproc_reqctx_s *rctx,
 580                        u8 tx_frag_num,
 581                        u32 spu_hdr_len,
 582                        unsigned int hash_carry_len,
 583                        unsigned int new_data_len, u32 pad_len)
 584 {
 585         struct spu_hw *spu = &iproc_priv.spu;
 586         struct scatterlist *sg; /* used to build sgs in mbox message */
 587         u32 datalen;            /* Number of bytes of response data expected */
 588         u32 stat_len;
 589
 590         mssg->spu.src = kcalloc(tx_frag_num, sizeof(struct scatterlist),
 591                                 rctx->gfp);
 592         if (!mssg->spu.src)
 593                 return -ENOMEM;
 594
 595         sg = mssg->spu.src;
 596         sg_init_table(sg, tx_frag_num);
 597
 598         sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr,
 599                    BCM_HDR_LEN + spu_hdr_len);
 600
 601         if (hash_carry_len)
 602                 sg_set_buf(sg++, rctx->hash_carry, hash_carry_len);
 603
 604         if (new_data_len) {
 605                 /* Copy in each src sg entry from request, up to chunksize */
 606                 datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip,
 607                                          rctx->src_nents, new_data_len);
 608                 if (datalen < new_data_len) {
 609                         pr_err("%s(): failed to copy src sg to mbox msg",
 610                                __func__);
 611                         return -EFAULT;
 612                 }
 613         }
 614
 615         if (pad_len)
 616                 sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len);
 617
 618         stat_len = spu->spu_tx_status_len();
 619         if (stat_len) {
 620                 memset(rctx->msg_buf.tx_stat, 0, stat_len);
 621                 sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len);
 622         }
 623
 624         return 0;
 625 }
 626
 627 /**
 628  * handle_ahash_req() - Process an asynchronous hash request from the crypto
 629  * API.
 630  * @rctx:  Crypto request context
 631  *
 632  * Builds a SPU request message embedded in a mailbox message and submits the
 633  * mailbox message on a selected mailbox channel. The SPU request message is
 634  * constructed as a scatterlist, including entries from the crypto API's
 635  * src scatterlist to avoid copying the data to be hashed. This function is
 636  * called either on the thread from the crypto API, or, in the case that the
 637  * crypto API request is too large to fit in a single SPU request message,
 638  * on the thread that invokes the receive callback with a response message.
 639  * Because some operations require the response from one chunk before the next
 640  * chunk can be submitted, we always wait for the response for the previous
 641  * chunk before submitting the next chunk. Because requests are submitted in
 642  * lock step like this, there is no need to synchronize access to request data
 643  * structures.
 644  *
 645  * Return:
 646  *   -EINPROGRESS: request has been submitted to SPU and response will be
 647  *                 returned asynchronously
 648  *   -EAGAIN:      non-final request included a small amount of data, which for
 649  *                 efficiency we did not submit to the SPU, but instead stored
 650  *                 to be submitted to the SPU with the next part of the request
 651  *   other:        an error code
 652  */
 653 static int handle_ahash_req(struct iproc_reqctx_s *rctx)
 654 {
 655         struct spu_hw *spu = &iproc_priv.spu;
 656         struct crypto_async_request *areq = rctx->parent;
 657         struct ahash_request *req = ahash_request_cast(areq);
 658         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 659         struct crypto_tfm *tfm = crypto_ahash_tfm(ahash);
 660         unsigned int blocksize = crypto_tfm_alg_blocksize(tfm);
 661         struct iproc_ctx_s *ctx = rctx->ctx;
 662
 663         /* number of bytes still to be hashed in this req */
 664         unsigned int nbytes_to_hash = 0;
 665         int err;
 666         unsigned int chunksize = 0;     /* length of hash carry + new data */
 667         /*
 668          * length of new data, not from hash carry, to be submitted in
 669          * this hw request
 670          */
 671         unsigned int new_data_len;
 672
 673         unsigned int __maybe_unused chunk_start = 0;
 674         u32 db_size;     /* Length of data field, incl gcm and hash padding */
 675         int pad_len = 0; /* total pad len, including gcm, hash, stat padding */
 676         u32 data_pad_len = 0;   /* length of GCM/CCM padding */
 677         u32 stat_pad_len = 0;   /* length of padding to align STATUS word */
 678         struct brcm_message *mssg;      /* mailbox message */
 679         struct spu_request_opts req_opts;
 680         struct spu_cipher_parms cipher_parms;
 681         struct spu_hash_parms hash_parms;
 682         struct spu_aead_parms aead_parms;
 683         unsigned int local_nbuf;
 684         u32 spu_hdr_len;
 685         unsigned int digestsize;
 686         u16 rem = 0;
 687
 688         /*
 689          * number of entries in src and dst sg. Always includes SPU msg header.
 690          * rx always includes a buffer to catch digest and STATUS.
 691          */
 692         u8 rx_frag_num = 3;
 693         u8 tx_frag_num = 1;
 694
 695         flow_log("total_todo %u, total_sent %u\n",
 696                  rctx->total_todo, rctx->total_sent);
 697
 698         memset(&req_opts, 0, sizeof(req_opts));
 699         memset(&cipher_parms, 0, sizeof(cipher_parms));
 700         memset(&hash_parms, 0, sizeof(hash_parms));
 701         memset(&aead_parms, 0, sizeof(aead_parms));
 702
 703         req_opts.bd_suppress = true;
 704         hash_parms.alg = ctx->auth.alg;
 705         hash_parms.mode = ctx->auth.mode;
 706         hash_parms.type = HASH_TYPE_NONE;
 707         hash_parms.key_buf = (u8 *)ctx->authkey;
 708         hash_parms.key_len = ctx->authkeylen;
 709
 710         /*
 711          * For hash algorithms below assignment looks bit odd but
 712          * it's needed for AES-XCBC and AES-CMAC hash algorithms
 713          * to differentiate between 128, 192, 256 bit key values.
 714          * Based on the key values, hash algorithm is selected.
 715          * For example for 128 bit key, hash algorithm is AES-128.
 716          */
 717         cipher_parms.type = ctx->cipher_type;
 718
 719         mssg = &rctx->mb_mssg;
 720         chunk_start = rctx->src_sent;
 721
 722         /*
 723          * Compute the amount remaining to hash. This may include data
 724          * carried over from previous requests.
 725          */
 726         nbytes_to_hash = rctx->total_todo - rctx->total_sent;
 727         chunksize = nbytes_to_hash;
 728         if ((ctx->max_payload != SPU_MAX_PAYLOAD_INF) &&
 729             (chunksize > ctx->max_payload))
 730                 chunksize = ctx->max_payload;
 731
 732         /*
 733          * If this is not a final request and the request data is not a multiple
 734          * of a full block, then simply park the extra data and prefix it to the
 735          * data for the next request.
 736          */
 737         if (!rctx->is_final) {
 738                 u8 *dest = rctx->hash_carry + rctx->hash_carry_len;
 739                 u16 new_len;  /* len of data to add to hash carry */
 740
 741                 rem = chunksize % blocksize;   /* remainder */
 742                 if (rem) {
 743                         /* chunksize not a multiple of blocksize */
 744                         chunksize -= rem;
 745                         if (chunksize == 0) {
 746                                 /* Don't have a full block to submit to hw */
 747                                 new_len = rem - rctx->hash_carry_len;
 748                                 sg_copy_part_to_buf(req->src, dest, new_len,
 749                                                     rctx->src_sent);
 750                                 rctx->hash_carry_len = rem;
 751                                 flow_log("Exiting with hash carry len: %u\n",
 752                                          rctx->hash_carry_len);
 753                                 packet_dump("  buf: ",
 754                                             rctx->hash_carry,
 755                                             rctx->hash_carry_len);
 756                                 return -EAGAIN;
 757                         }
 758                 }
 759         }
 760
 761         /* if we have hash carry, then prefix it to the data in this request */
 762         local_nbuf = rctx->hash_carry_len;
 763         rctx->hash_carry_len = 0;
 764         if (local_nbuf)
 765                 tx_frag_num++;
 766         new_data_len = chunksize - local_nbuf;
 767
 768         /* Count number of sg entries to be used in this request */
 769         rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip,
 770                                        new_data_len);
 771
 772         /* AES hashing keeps key size in type field, so need to copy it here */
 773         if (hash_parms.alg == HASH_ALG_AES)
 774                 hash_parms.type = (enum hash_type)cipher_parms.type;
 775         else
 776                 hash_parms.type = spu->spu_hash_type(rctx->total_sent);
 777
 778         digestsize = spu->spu_digest_size(ctx->digestsize, ctx->auth.alg,
 779                                           hash_parms.type);
 780         hash_parms.digestsize = digestsize;
 781
 782         /* update the indexes */
 783         rctx->total_sent += chunksize;
 784         /* if you sent a prebuf then that wasn't from this req->src */
 785         rctx->src_sent += new_data_len;
 786
 787         if ((rctx->total_sent == rctx->total_todo) && rctx->is_final)
 788                 hash_parms.pad_len = spu->spu_hash_pad_len(hash_parms.alg,
 789                                                            hash_parms.mode,
 790                                                            chunksize,
 791                                                            blocksize);
 792
 793         /*
 794          * If a non-first chunk, then include the digest returned from the
 795          * previous chunk so that hw can add to it (except for AES types).
 796          */
 797         if ((hash_parms.type == HASH_TYPE_UPDT) &&
 798             (hash_parms.alg != HASH_ALG_AES)) {
 799                 hash_parms.key_buf = rctx->incr_hash;
 800                 hash_parms.key_len = digestsize;
 801         }
 802
 803         atomic64_add(chunksize, &iproc_priv.bytes_out);
 804
 805         flow_log("%s() final: %u nbuf: %u ",
 806                  __func__, rctx->is_final, local_nbuf);
 807
 808         if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
 809                 flow_log("max_payload infinite\n");
 810         else
 811                 flow_log("max_payload %u\n", ctx->max_payload);
 812
 813         flow_log("chunk_start: %u chunk_size: %u\n", chunk_start, chunksize);
 814
 815         /* Prepend SPU header with type 3 BCM header */
 816         memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN);
 817
 818         hash_parms.prebuf_len = local_nbuf;
 819         spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr +
 820                                               BCM_HDR_LEN,
 821                                               &req_opts, &cipher_parms,
 822                                               &hash_parms, &aead_parms,
 823                                               new_data_len);
 824
 825         if (spu_hdr_len == 0) {
 826                 pr_err("Failed to create SPU request header\n");
 827                 return -EFAULT;
 828         }
 829
 830         /*
 831          * Determine total length of padding required. Put all padding in one
 832          * buffer.
 833          */
 834         data_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode, chunksize);
 835         db_size = spu_real_db_size(0, 0, local_nbuf, new_data_len,
 836                                    0, 0, hash_parms.pad_len);
 837         if (spu->spu_tx_status_len())
 838                 stat_pad_len = spu->spu_wordalign_padlen(db_size);
 839         if (stat_pad_len)
 840                 rx_frag_num++;
 841         pad_len = hash_parms.pad_len + data_pad_len + stat_pad_len;
 842         if (pad_len) {
 843                 tx_frag_num++;
 844                 spu->spu_request_pad(rctx->msg_buf.spu_req_pad, data_pad_len,
 845                                      hash_parms.pad_len, ctx->auth.alg,
 846                                      ctx->auth.mode, rctx->total_sent,
 847                                      stat_pad_len);
 848         }
 849
 850         spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
 851                               spu_hdr_len);
 852         packet_dump("    prebuf: ", rctx->hash_carry, local_nbuf);
 853         flow_log("Data:\n");
 854         dump_sg(rctx->src_sg, rctx->src_skip, new_data_len);
 855         packet_dump("   pad: ", rctx->msg_buf.spu_req_pad, pad_len);
 856
 857         /*
 858          * Build mailbox message containing SPU request msg and rx buffers
 859          * to catch response message
 860          */
 861         memset(mssg, 0, sizeof(*mssg));
 862         mssg->type = BRCM_MESSAGE_SPU;
 863         mssg->ctx = rctx;       /* Will be returned in response */
 864
 865         /* Create rx scatterlist to catch result */
 866         err = spu_ahash_rx_sg_create(mssg, rctx, rx_frag_num, digestsize,
 867                                      stat_pad_len);
 868         if (err)
 869                 return err;
 870
 871         /* Create tx scatterlist containing SPU request message */
 872         tx_frag_num += rctx->src_nents;
 873         if (spu->spu_tx_status_len())
 874                 tx_frag_num++;
 875         err = spu_ahash_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len,
 876                                      local_nbuf, new_data_len, pad_len);
 877         if (err)
 878                 return err;
 879
 880         err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx);
 881         if (unlikely(err < 0))
 882                 return err;
 883
 884         return -EINPROGRESS;
 885 }
 886
 887 /**
 888  * spu_hmac_outer_hash() - Request synchonous software compute of the outer hash
 889  * for an HMAC request.
 890  * @req:  The HMAC request from the crypto API
 891  * @ctx:  The session context
 892  *
 893  * Return: 0 if synchronous hash operation successful
 894  *         -EINVAL if the hash algo is unrecognized
 895  *         any other value indicates an error
 896  */
 897 static int spu_hmac_outer_hash(struct ahash_request *req,
 898                                struct iproc_ctx_s *ctx)
 899 {
 900         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 901         unsigned int blocksize =
 902                 crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash));
 903         int rc;
 904
 905         switch (ctx->auth.alg) {
 906         case HASH_ALG_MD5:
 907                 rc = do_shash("md5", req->result, ctx->opad, blocksize,
 908                               req->result, ctx->digestsize, NULL, 0);
 909                 break;
 910         case HASH_ALG_SHA1:
 911                 rc = do_shash("sha1", req->result, ctx->opad, blocksize,
 912                               req->result, ctx->digestsize, NULL, 0);
 913                 break;
 914         case HASH_ALG_SHA224:
 915                 rc = do_shash("sha224", req->result, ctx->opad, blocksize,
 916                               req->result, ctx->digestsize, NULL, 0);
 917                 break;
 918         case HASH_ALG_SHA256:
 919                 rc = do_shash("sha256", req->result, ctx->opad, blocksize,
 920                               req->result, ctx->digestsize, NULL, 0);
 921                 break;
 922         case HASH_ALG_SHA384:
 923                 rc = do_shash("sha384", req->result, ctx->opad, blocksize,
 924                               req->result, ctx->digestsize, NULL, 0);
 925                 break;
 926         case HASH_ALG_SHA512:
 927                 rc = do_shash("sha512", req->result, ctx->opad, blocksize,
 928                               req->result, ctx->digestsize, NULL, 0);
 929                 break;
 930         default:
 931                 pr_err("%s() Error : unknown hmac type\n", __func__);
 932                 rc = -EINVAL;
 933         }
 934         return rc;
 935 }
 936
 937 /**
 938  * ahash_req_done() - Process a hash result from the SPU hardware.
 939  * @rctx: Crypto request context
 940  *
 941  * Return: 0 if successful
 942  *         < 0 if an error
 943  */
 944 static int ahash_req_done(struct iproc_reqctx_s *rctx)
 945 {
 946         struct spu_hw *spu = &iproc_priv.spu;
 947         struct crypto_async_request *areq = rctx->parent;
 948         struct ahash_request *req = ahash_request_cast(areq);
 949         struct iproc_ctx_s *ctx = rctx->ctx;
 950         int err;
 951
 952         memcpy(req->result, rctx->msg_buf.digest, ctx->digestsize);
 953
 954         if (spu->spu_type == SPU_TYPE_SPUM) {
 955                 /* byte swap the output from the UPDT function to network byte
 956                  * order
 957                  */
 958                 if (ctx->auth.alg == HASH_ALG_MD5) {
 959                         __swab32s((u32 *)req->result);
 960                         __swab32s(((u32 *)req->result) + 1);
 961                         __swab32s(((u32 *)req->result) + 2);
 962                         __swab32s(((u32 *)req->result) + 3);
 963                         __swab32s(((u32 *)req->result) + 4);
 964                 }
 965         }
 966
 967         flow_dump("  digest ", req->result, ctx->digestsize);
 968
 969         /* if this an HMAC then do the outer hash */
 970         if (rctx->is_sw_hmac) {
 971                 err = spu_hmac_outer_hash(req, ctx);
 972                 if (err < 0)
 973                         return err;
 974                 flow_dump("  hmac: ", req->result, ctx->digestsize);
 975         }
 976
 977         if (rctx->is_sw_hmac || ctx->auth.mode == HASH_MODE_HMAC) {
 978                 atomic_inc(&iproc_priv.op_counts[SPU_OP_HMAC]);
 979                 atomic_inc(&iproc_priv.hmac_cnt[ctx->auth.alg]);
 980         } else {
 981                 atomic_inc(&iproc_priv.op_counts[SPU_OP_HASH]);
 982                 atomic_inc(&iproc_priv.hash_cnt[ctx->auth.alg]);
 983         }
 984
 985         return 0;
 986 }
 987
 988 /**
 989  * handle_ahash_resp() - Process a SPU response message for a hash request.
 990  * Checks if the entire crypto API request has been processed, and if so,
 991  * invokes post processing on the result.
 992  * @rctx: Crypto request context
 993  */
 994 static void handle_ahash_resp(struct iproc_reqctx_s *rctx)
 995 {
 996         struct iproc_ctx_s *ctx = rctx->ctx;
 997         struct crypto_async_request *areq = rctx->parent;
 998         struct ahash_request *req = ahash_request_cast(areq);
 999         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1000         unsigned int blocksize =
1001                 crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash));
1002         /*
1003          * Save hash to use as input to next op if incremental. Might be copying
1004          * too much, but that's easier than figuring out actual digest size here
1005          */
1006         memcpy(rctx->incr_hash, rctx->msg_buf.digest, MAX_DIGEST_SIZE);
1007
1008         flow_log("%s() blocksize:%u digestsize:%u\n",
1009                  __func__, blocksize, ctx->digestsize);
1010
1011         atomic64_add(ctx->digestsize, &iproc_priv.bytes_in);
1012
1013         if (rctx->is_final && (rctx->total_sent == rctx->total_todo))
1014                 ahash_req_done(rctx);
1015 }
1016
1017 /**
1018  * spu_aead_rx_sg_create() - Build up the scatterlist of buffers used to receive
1019  * a SPU response message for an AEAD request. Includes buffers to catch SPU
1020  * message headers and the response data.
1021  * @mssg:       mailbox message containing the receive sg
1022  * @req:        Crypto API request
1023  * @rctx:       crypto request context
1024  * @rx_frag_num: number of scatterlist elements required to hold the
1025  *              SPU response message
1026  * @assoc_len:  Length of associated data included in the crypto request
1027  * @ret_iv_len: Length of IV returned in response
1028  * @resp_len:   Number of bytes of response data expected to be written to
1029  *              dst buffer from crypto API
1030  * @digestsize: Length of hash digest, in bytes
1031  * @stat_pad_len: Number of bytes required to pad the STAT field to
1032  *              a 4-byte boundary
1033  *
1034  * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
1035  * when the request completes, whether the request is handled successfully or
1036  * there is an error.
1037  *
1038  * Returns:
1039  *   0 if successful
1040  *   < 0 if an error
1041  */
1042 static int spu_aead_rx_sg_create(struct brcm_message *mssg,
1043                                  struct aead_request *req,
1044                                  struct iproc_reqctx_s *rctx,
1045                                  u8 rx_frag_num,
1046                                  unsigned int assoc_len,
1047                                  u32 ret_iv_len, unsigned int resp_len,
1048                                  unsigned int digestsize, u32 stat_pad_len)
1049 {
1050         struct spu_hw *spu = &iproc_priv.spu;
1051         struct scatterlist *sg; /* used to build sgs in mbox message */
1052         struct iproc_ctx_s *ctx = rctx->ctx;
1053         u32 datalen;            /* Number of bytes of response data expected */
1054         u32 assoc_buf_len;
1055         u8 data_padlen = 0;
1056
1057         if (ctx->is_rfc4543) {
1058                 /* RFC4543: only pad after data, not after AAD */
1059                 data_padlen = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1060                                                           assoc_len + resp_len);
1061                 assoc_buf_len = assoc_len;
1062         } else {
1063                 data_padlen = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1064                                                           resp_len);
1065                 assoc_buf_len = spu->spu_assoc_resp_len(ctx->cipher.mode,
1066                                                 assoc_len, ret_iv_len,
1067                                                 rctx->is_encrypt);
1068         }
1069
1070         if (ctx->cipher.mode == CIPHER_MODE_CCM)
1071                 /* ICV (after data) must be in the next 32-bit word for CCM */
1072                 data_padlen += spu->spu_wordalign_padlen(assoc_buf_len +
1073                                                          resp_len +
1074                                                          data_padlen);
1075
1076         if (data_padlen)
1077                 /* have to catch gcm pad in separate buffer */
1078                 rx_frag_num++;
1079
1080         mssg->spu.dst = kcalloc(rx_frag_num, sizeof(struct scatterlist),
1081                                 rctx->gfp);
1082         if (!mssg->spu.dst)
1083                 return -ENOMEM;
1084
1085         sg = mssg->spu.dst;
1086         sg_init_table(sg, rx_frag_num);
1087
1088         /* Space for SPU message header */
1089         sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len);
1090
1091         if (assoc_buf_len) {
1092                 /*
1093                  * Don't write directly to req->dst, because SPU may pad the
1094                  * assoc data in the response
1095                  */
1096                 memset(rctx->msg_buf.a.resp_aad, 0, assoc_buf_len);
1097                 sg_set_buf(sg++, rctx->msg_buf.a.resp_aad, assoc_buf_len);
1098         }
1099
1100         if (resp_len) {
1101                 /*
1102                  * Copy in each dst sg entry from request, up to chunksize.
1103                  * dst sg catches just the data. digest caught in separate buf.
1104                  */
1105                 datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip,
1106                                          rctx->dst_nents, resp_len);
1107                 if (datalen < (resp_len)) {
1108                         pr_err("%s(): failed to copy dst sg to mbox msg. expected len %u, datalen %u",
1109                                __func__, resp_len, datalen);
1110                         return -EFAULT;
1111                 }
1112         }
1113
1114         /* If GCM/CCM data is padded, catch padding in separate buffer */
1115         if (data_padlen) {
1116                 memset(rctx->msg_buf.a.gcmpad, 0, data_padlen);
1117                 sg_set_buf(sg++, rctx->msg_buf.a.gcmpad, data_padlen);
1118         }
1119
1120         /* Always catch ICV in separate buffer */
1121         sg_set_buf(sg++, rctx->msg_buf.digest, digestsize);
1122
1123         flow_log("stat_pad_len %u\n", stat_pad_len);
1124         if (stat_pad_len) {
1125                 memset(rctx->msg_buf.rx_stat_pad, 0, stat_pad_len);
1126                 sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len);
1127         }
1128
1129         memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN);
1130         sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len());
1131
1132         return 0;
1133 }
1134
1135 /**
1136  * spu_aead_tx_sg_create() - Build up the scatterlist of buffers used to send a
1137  * SPU request message for an AEAD request. Includes SPU message headers and the
1138  * request data.
1139  * @mssg:       mailbox message containing the transmit sg
1140  * @rctx:       crypto request context
1141  * @tx_frag_num: number of scatterlist elements required to construct the
1142  *              SPU request message
1143  * @spu_hdr_len: length of SPU message header in bytes
1144  * @assoc:      crypto API associated data scatterlist
1145  * @assoc_len:  length of associated data
1146  * @assoc_nents: number of scatterlist entries containing assoc data
1147  * @aead_iv_len: length of AEAD IV, if included
1148  * @chunksize:  Number of bytes of request data
1149  * @aad_pad_len: Number of bytes of padding at end of AAD. For GCM/CCM.
1150  * @pad_len:    Number of pad bytes
1151  * @incl_icv:   If true, write separate ICV buffer after data and
1152  *              any padding
1153  *
1154  * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
1155  * when the request completes, whether the request is handled successfully or
1156  * there is an error.
1157  *
1158  * Return:
1159  *   0 if successful
1160  *   < 0 if an error
1161  */
1162 static int spu_aead_tx_sg_create(struct brcm_message *mssg,
1163                                  struct iproc_reqctx_s *rctx,
1164                                  u8 tx_frag_num,
1165                                  u32 spu_hdr_len,
1166                                  struct scatterlist *assoc,
1167                                  unsigned int assoc_len,
1168                                  int assoc_nents,
1169                                  unsigned int aead_iv_len,
1170                                  unsigned int chunksize,
1171                                  u32 aad_pad_len, u32 pad_len, bool incl_icv)
1172 {
1173         struct spu_hw *spu = &iproc_priv.spu;
1174         struct scatterlist *sg; /* used to build sgs in mbox message */
1175         struct scatterlist *assoc_sg = assoc;
1176         struct iproc_ctx_s *ctx = rctx->ctx;
1177         u32 datalen;            /* Number of bytes of data to write */
1178         u32 written;            /* Number of bytes of data written */
1179         u32 assoc_offset = 0;
1180         u32 stat_len;
1181
1182         mssg->spu.src = kcalloc(tx_frag_num, sizeof(struct scatterlist),
1183                                 rctx->gfp);
1184         if (!mssg->spu.src)
1185                 return -ENOMEM;
1186
1187         sg = mssg->spu.src;
1188         sg_init_table(sg, tx_frag_num);
1189
1190         sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr,
1191                    BCM_HDR_LEN + spu_hdr_len);
1192
1193         if (assoc_len) {
1194                 /* Copy in each associated data sg entry from request */
1195                 written = spu_msg_sg_add(&sg, &assoc_sg, &assoc_offset,
1196                                          assoc_nents, assoc_len);
1197                 if (written < assoc_len) {
1198                         pr_err("%s(): failed to copy assoc sg to mbox msg",
1199                                __func__);
1200                         return -EFAULT;
1201                 }
1202         }
1203
1204         if (aead_iv_len)
1205                 sg_set_buf(sg++, rctx->msg_buf.iv_ctr, aead_iv_len);
1206
1207         if (aad_pad_len) {
1208                 memset(rctx->msg_buf.a.req_aad_pad, 0, aad_pad_len);
1209                 sg_set_buf(sg++, rctx->msg_buf.a.req_aad_pad, aad_pad_len);
1210         }
1211
1212         datalen = chunksize;
1213         if ((chunksize > ctx->digestsize) && incl_icv)
1214                 datalen -= ctx->digestsize;
1215         if (datalen) {
1216                 /* For aead, a single msg should consume the entire src sg */
1217                 written = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip,
1218                                          rctx->src_nents, datalen);
1219                 if (written < datalen) {
1220                         pr_err("%s(): failed to copy src sg to mbox msg",
1221                                __func__);
1222                         return -EFAULT;
1223                 }
1224         }
1225
1226         if (pad_len) {
1227                 memset(rctx->msg_buf.spu_req_pad, 0, pad_len);
1228                 sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len);
1229         }
1230
1231         if (incl_icv)
1232                 sg_set_buf(sg++, rctx->msg_buf.digest, ctx->digestsize);
1233
1234         stat_len = spu->spu_tx_status_len();
1235         if (stat_len) {
1236                 memset(rctx->msg_buf.tx_stat, 0, stat_len);
1237                 sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len);
1238         }
1239         return 0;
1240 }
1241
1242 /**
1243  * handle_aead_req() - Submit a SPU request message for the next chunk of the
1244  * current AEAD request.
1245  * @rctx:  Crypto request context
1246  *
1247  * Unlike other operation types, we assume the length of the request fits in
1248  * a single SPU request message. aead_enqueue() makes sure this is true.
1249  * Comments for other op types regarding threads applies here as well.
1250  *
1251  * Unlike incremental hash ops, where the spu returns the entire hash for
1252  * truncated algs like sha-224, the SPU returns just the truncated hash in
1253  * response to aead requests. So digestsize is always ctx->digestsize here.
1254  *
1255  * Return: -EINPROGRESS: crypto request has been accepted and result will be
1256  *                       returned asynchronously
1257  *         Any other value indicates an error
1258  */
1259 static int handle_aead_req(struct iproc_reqctx_s *rctx)
1260 {
1261         struct spu_hw *spu = &iproc_priv.spu;
1262         struct crypto_async_request *areq = rctx->parent;
1263         struct aead_request *req = container_of(areq,
1264                                                 struct aead_request, base);
1265         struct iproc_ctx_s *ctx = rctx->ctx;
1266         int err;
1267         unsigned int chunksize;
1268         unsigned int resp_len;
1269         u32 spu_hdr_len;
1270         u32 db_size;
1271         u32 stat_pad_len;
1272         u32 pad_len;
1273         struct brcm_message *mssg;      /* mailbox message */
1274         struct spu_request_opts req_opts;
1275         struct spu_cipher_parms cipher_parms;
1276         struct spu_hash_parms hash_parms;
1277         struct spu_aead_parms aead_parms;
1278         int assoc_nents = 0;
1279         bool incl_icv = false;
1280         unsigned int digestsize = ctx->digestsize;
1281
1282         /* number of entries in src and dst sg. Always includes SPU msg header.
1283          */
1284         u8 rx_frag_num = 2;     /* and STATUS */
1285         u8 tx_frag_num = 1;
1286
1287         /* doing the whole thing at once */
1288         chunksize = rctx->total_todo;
1289
1290         flow_log("%s: chunksize %u\n", __func__, chunksize);
1291
1292         memset(&req_opts, 0, sizeof(req_opts));
1293         memset(&hash_parms, 0, sizeof(hash_parms));
1294         memset(&aead_parms, 0, sizeof(aead_parms));
1295
1296         req_opts.is_inbound = !(rctx->is_encrypt);
1297         req_opts.auth_first = ctx->auth_first;
1298         req_opts.is_aead = true;
1299         req_opts.is_esp = ctx->is_esp;
1300
1301         cipher_parms.alg = ctx->cipher.alg;
1302         cipher_parms.mode = ctx->cipher.mode;
1303         cipher_parms.type = ctx->cipher_type;
1304         cipher_parms.key_buf = ctx->enckey;
1305         cipher_parms.key_len = ctx->enckeylen;
1306         cipher_parms.iv_buf = rctx->msg_buf.iv_ctr;
1307         cipher_parms.iv_len = rctx->iv_ctr_len;
1308
1309         hash_parms.alg = ctx->auth.alg;
1310         hash_parms.mode = ctx->auth.mode;
1311         hash_parms.type = HASH_TYPE_NONE;
1312         hash_parms.key_buf = (u8 *)ctx->authkey;
1313         hash_parms.key_len = ctx->authkeylen;
1314         hash_parms.digestsize = digestsize;
1315
1316         if ((ctx->auth.alg == HASH_ALG_SHA224) &&
1317             (ctx->authkeylen < SHA224_DIGEST_SIZE))
1318                 hash_parms.key_len = SHA224_DIGEST_SIZE;
1319
1320         aead_parms.assoc_size = req->assoclen;
1321         if (ctx->is_esp && !ctx->is_rfc4543) {
1322                 /*
1323                  * 8-byte IV is included assoc data in request. SPU2
1324                  * expects AAD to include just SPI and seqno. So
1325                  * subtract off the IV len.
1326                  */
1327                 aead_parms.assoc_size -= GCM_RFC4106_IV_SIZE;
1328
1329                 if (rctx->is_encrypt) {
1330                         aead_parms.return_iv = true;
1331                         aead_parms.ret_iv_len = GCM_RFC4106_IV_SIZE;
1332                         aead_parms.ret_iv_off = GCM_ESP_SALT_SIZE;
1333                 }
1334         } else {
1335                 aead_parms.ret_iv_len = 0;
1336         }
1337
1338         /*
1339          * Count number of sg entries from the crypto API request that are to
1340          * be included in this mailbox message. For dst sg, don't count space
1341          * for digest. Digest gets caught in a separate buffer and copied back
1342          * to dst sg when processing response.
1343          */
1344         rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize);
1345         rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize);
1346         if (aead_parms.assoc_size)
1347                 assoc_nents = spu_sg_count(rctx->assoc, 0,
1348                                            aead_parms.assoc_size);
1349
1350         mssg = &rctx->mb_mssg;
1351
1352         rctx->total_sent = chunksize;
1353         rctx->src_sent = chunksize;
1354         if (spu->spu_assoc_resp_len(ctx->cipher.mode,
1355                                     aead_parms.assoc_size,
1356                                     aead_parms.ret_iv_len,
1357                                     rctx->is_encrypt))
1358                 rx_frag_num++;
1359
1360         aead_parms.iv_len = spu->spu_aead_ivlen(ctx->cipher.mode,
1361                                                 rctx->iv_ctr_len);
1362
1363         if (ctx->auth.alg == HASH_ALG_AES)
1364                 hash_parms.type = (enum hash_type)ctx->cipher_type;
1365
1366         /* General case AAD padding (CCM and RFC4543 special cases below) */
1367         aead_parms.aad_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1368                                                  aead_parms.assoc_size);
1369
1370         /* General case data padding (CCM decrypt special case below) */
1371         aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1372                                                            chunksize);
1373
1374         if (ctx->cipher.mode == CIPHER_MODE_CCM) {
1375                 /*
1376                  * for CCM, AAD len + 2 (rather than AAD len) needs to be
1377                  * 128-bit aligned
1378                  */
1379                 aead_parms.aad_pad_len = spu->spu_gcm_ccm_pad_len(
1380                                          ctx->cipher.mode,
1381                                          aead_parms.assoc_size + 2);
1382
1383                 /*
1384                  * And when decrypting CCM, need to pad without including
1385                  * size of ICV which is tacked on to end of chunk
1386                  */
1387                 if (!rctx->is_encrypt)
1388                         aead_parms.data_pad_len =
1389                                 spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1390                                                         chunksize - digestsize);
1391
1392                 /* CCM also requires software to rewrite portions of IV: */
1393                 spu->spu_ccm_update_iv(digestsize, &cipher_parms, req->assoclen,
1394                                        chunksize, rctx->is_encrypt,
1395                                        ctx->is_esp);
1396         }
1397
1398         if (ctx->is_rfc4543) {
1399                 /*
1400                  * RFC4543: data is included in AAD, so don't pad after AAD
1401                  * and pad data based on both AAD + data size
1402                  */
1403                 aead_parms.aad_pad_len = 0;
1404                 if (!rctx->is_encrypt)
1405                         aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len(
1406                                         ctx->cipher.mode,
1407                                         aead_parms.assoc_size + chunksize -
1408                                         digestsize);
1409                 else
1410                         aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len(
1411                                         ctx->cipher.mode,
1412                                         aead_parms.assoc_size + chunksize);
1413
1414                 req_opts.is_rfc4543 = true;
1415         }
1416
1417         if (spu_req_incl_icv(ctx->cipher.mode, rctx->is_encrypt)) {
1418                 incl_icv = true;
1419                 tx_frag_num++;
1420                 /* Copy ICV from end of src scatterlist to digest buf */
1421                 sg_copy_part_to_buf(req->src, rctx->msg_buf.digest, digestsize,
1422                                     req->assoclen + rctx->total_sent -
1423                                     digestsize);
1424         }
1425
1426         atomic64_add(chunksize, &iproc_priv.bytes_out);
1427
1428         flow_log("%s()-sent chunksize:%u\n", __func__, chunksize);
1429
1430         /* Prepend SPU header with type 3 BCM header */
1431         memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN);
1432
1433         spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr +
1434                                               BCM_HDR_LEN, &req_opts,
1435                                               &cipher_parms, &hash_parms,
1436                                               &aead_parms, chunksize);
1437
1438         /* Determine total length of padding. Put all padding in one buffer. */
1439         db_size = spu_real_db_size(aead_parms.assoc_size, aead_parms.iv_len, 0,
1440                                    chunksize, aead_parms.aad_pad_len,
1441                                    aead_parms.data_pad_len, 0);
1442
1443         stat_pad_len = spu->spu_wordalign_padlen(db_size);
1444
1445         if (stat_pad_len)
1446                 rx_frag_num++;
1447         pad_len = aead_parms.data_pad_len + stat_pad_len;
1448         if (pad_len) {
1449                 tx_frag_num++;
1450                 spu->spu_request_pad(rctx->msg_buf.spu_req_pad,
1451                                      aead_parms.data_pad_len, 0,
1452                                      ctx->auth.alg, ctx->auth.mode,
1453                                      rctx->total_sent, stat_pad_len);
1454         }
1455
1456         spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
1457                               spu_hdr_len);
1458         dump_sg(rctx->assoc, 0, aead_parms.assoc_size);
1459         packet_dump("    aead iv: ", rctx->msg_buf.iv_ctr, aead_parms.iv_len);
1460         packet_log("BD:\n");
1461         dump_sg(rctx->src_sg, rctx->src_skip, chunksize);
1462         packet_dump("   pad: ", rctx->msg_buf.spu_req_pad, pad_len);
1463
1464         /*
1465          * Build mailbox message containing SPU request msg and rx buffers
1466          * to catch response message
1467          */
1468         memset(mssg, 0, sizeof(*mssg));
1469         mssg->type = BRCM_MESSAGE_SPU;
1470         mssg->ctx = rctx;       /* Will be returned in response */
1471
1472         /* Create rx scatterlist to catch result */
1473         rx_frag_num += rctx->dst_nents;
1474         resp_len = chunksize;
1475
1476         /*
1477          * Always catch ICV in separate buffer. Have to for GCM/CCM because of
1478          * padding. Have to for SHA-224 and other truncated SHAs because SPU
1479          * sends entire digest back.
1480          */
1481         rx_frag_num++;
1482
1483         if (((ctx->cipher.mode == CIPHER_MODE_GCM) ||
1484              (ctx->cipher.mode == CIPHER_MODE_CCM)) && !rctx->is_encrypt) {
1485                 /*
1486                  * Input is ciphertxt plus ICV, but ICV not incl
1487                  * in output.
1488                  */
1489                 resp_len -= ctx->digestsize;
1490                 if (resp_len == 0)
1491                         /* no rx frags to catch output data */
1492                         rx_frag_num -= rctx->dst_nents;
1493         }
1494
1495         err = spu_aead_rx_sg_create(mssg, req, rctx, rx_frag_num,
1496                                     aead_parms.assoc_size,
1497                                     aead_parms.ret_iv_len, resp_len, digestsize,
1498                                     stat_pad_len);
1499         if (err)
1500                 return err;
1501
1502         /* Create tx scatterlist containing SPU request message */
1503         tx_frag_num += rctx->src_nents;
1504         tx_frag_num += assoc_nents;
1505         if (aead_parms.aad_pad_len)
1506                 tx_frag_num++;
1507         if (aead_parms.iv_len)
1508                 tx_frag_num++;
1509         if (spu->spu_tx_status_len())
1510                 tx_frag_num++;
1511         err = spu_aead_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len,
1512                                     rctx->assoc, aead_parms.assoc_size,
1513                                     assoc_nents, aead_parms.iv_len, chunksize,
1514                                     aead_parms.aad_pad_len, pad_len, incl_icv);
1515         if (err)
1516                 return err;
1517
1518         err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx);
1519         if (unlikely(err < 0))
1520                 return err;
1521
1522         return -EINPROGRESS;
1523 }
1524
1525 /**
1526  * handle_aead_resp() - Process a SPU response message for an AEAD request.
1527  * @rctx:  Crypto request context
1528  */
1529 static void handle_aead_resp(struct iproc_reqctx_s *rctx)
1530 {
1531         struct spu_hw *spu = &iproc_priv.spu;
1532         struct crypto_async_request *areq = rctx->parent;
1533         struct aead_request *req = container_of(areq,
1534                                                 struct aead_request, base);
1535         struct iproc_ctx_s *ctx = rctx->ctx;
1536         u32 payload_len;
1537         unsigned int icv_offset;
1538         u32 result_len;
1539
1540         /* See how much data was returned */
1541         payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr);
1542         flow_log("payload_len %u\n", payload_len);
1543
1544         /* only count payload */
1545         atomic64_add(payload_len, &iproc_priv.bytes_in);
1546
1547         if (req->assoclen)
1548                 packet_dump("  assoc_data ", rctx->msg_buf.a.resp_aad,
1549                             req->assoclen);
1550
1551         /*
1552          * Copy the ICV back to the destination
1553          * buffer. In decrypt case, SPU gives us back the digest, but crypto
1554          * API doesn't expect ICV in dst buffer.
1555          */
1556         result_len = req->cryptlen;
1557         if (rctx->is_encrypt) {
1558                 icv_offset = req->assoclen + rctx->total_sent;
1559                 packet_dump("  ICV: ", rctx->msg_buf.digest, ctx->digestsize);
1560                 flow_log("copying ICV to dst sg at offset %u\n", icv_offset);
1561                 sg_copy_part_from_buf(req->dst, rctx->msg_buf.digest,
1562                                       ctx->digestsize, icv_offset);
1563                 result_len += ctx->digestsize;
1564         }
1565
1566         packet_log("response data:  ");
1567         dump_sg(req->dst, req->assoclen, result_len);
1568
1569         atomic_inc(&iproc_priv.op_counts[SPU_OP_AEAD]);
1570         if (ctx->cipher.alg == CIPHER_ALG_AES) {
1571                 if (ctx->cipher.mode == CIPHER_MODE_CCM)
1572                         atomic_inc(&iproc_priv.aead_cnt[AES_CCM]);
1573                 else if (ctx->cipher.mode == CIPHER_MODE_GCM)
1574                         atomic_inc(&iproc_priv.aead_cnt[AES_GCM]);
1575                 else
1576                         atomic_inc(&iproc_priv.aead_cnt[AUTHENC]);
1577         } else {
1578                 atomic_inc(&iproc_priv.aead_cnt[AUTHENC]);
1579         }
1580 }
1581
1582 /**
1583  * spu_chunk_cleanup() - Do cleanup after processing one chunk of a request
1584  * @rctx:  request context
1585  *
1586  * Mailbox scatterlists are allocated for each chunk. So free them after
1587  * processing each chunk.
1588  */
1589 static void spu_chunk_cleanup(struct iproc_reqctx_s *rctx)
1590 {
1591         /* mailbox message used to tx request */
1592         struct brcm_message *mssg = &rctx->mb_mssg;
1593
1594         kfree(mssg->spu.src);
1595         kfree(mssg->spu.dst);
1596         memset(mssg, 0, sizeof(struct brcm_message));
1597 }
1598
1599 /**
1600  * finish_req() - Used to invoke the complete callback from the requester when
1601  * a request has been handled asynchronously.
1602  * @rctx:  Request context
1603  * @err:   Indicates whether the request was successful or not
1604  *
1605  * Ensures that cleanup has been done for request
1606  */
1607 static void finish_req(struct iproc_reqctx_s *rctx, int err)
1608 {
1609         struct crypto_async_request *areq = rctx->parent;
1610
1611         flow_log("%s() err:%d\n\n", __func__, err);
1612
1613         /* No harm done if already called */
1614         spu_chunk_cleanup(rctx);
1615
1616         if (areq)
1617                 areq->complete(areq, err);
1618 }
1619
1620 /**
1621  * spu_rx_callback() - Callback from mailbox framework with a SPU response.
1622  * @cl:         mailbox client structure for SPU driver
1623  * @msg:        mailbox message containing SPU response
1624  */
1625 static void spu_rx_callback(struct mbox_client *cl, void *msg)
1626 {
1627         struct spu_hw *spu = &iproc_priv.spu;
1628         struct brcm_message *mssg = msg;
1629         struct iproc_reqctx_s *rctx;
1630         int err;
1631
1632         rctx = mssg->ctx;
1633         if (unlikely(!rctx)) {
1634                 /* This is fatal */
1635                 pr_err("%s(): no request context", __func__);
1636                 err = -EFAULT;
1637                 goto cb_finish;
1638         }
1639
1640         /* process the SPU status */
1641         err = spu->spu_status_process(rctx->msg_buf.rx_stat);
1642         if (err != 0) {
1643                 if (err == SPU_INVALID_ICV)
1644                         atomic_inc(&iproc_priv.bad_icv);
1645                 err = -EBADMSG;
1646                 goto cb_finish;
1647         }
1648
1649         /* Process the SPU response message */
1650         switch (rctx->ctx->alg->type) {
1651         case CRYPTO_ALG_TYPE_SKCIPHER:
1652                 handle_skcipher_resp(rctx);
1653                 break;
1654         case CRYPTO_ALG_TYPE_AHASH:
1655                 handle_ahash_resp(rctx);
1656                 break;
1657         case CRYPTO_ALG_TYPE_AEAD:
1658                 handle_aead_resp(rctx);
1659                 break;
1660         default:
1661                 err = -EINVAL;
1662                 goto cb_finish;
1663         }
1664
1665         /*
1666          * If this response does not complete the request, then send the next
1667          * request chunk.
1668          */
1669         if (rctx->total_sent < rctx->total_todo) {
1670                 /* Deallocate anything specific to previous chunk */
1671                 spu_chunk_cleanup(rctx);
1672
1673                 switch (rctx->ctx->alg->type) {
1674                 case CRYPTO_ALG_TYPE_SKCIPHER:
1675                         err = handle_skcipher_req(rctx);
1676                         break;
1677                 case CRYPTO_ALG_TYPE_AHASH:
1678                         err = handle_ahash_req(rctx);
1679                         if (err == -EAGAIN)
1680                                 /*
1681                                  * we saved data in hash carry, but tell crypto
1682                                  * API we successfully completed request.
1683                                  */
1684                                 err = 0;
1685                         break;
1686                 case CRYPTO_ALG_TYPE_AEAD:
1687                         err = handle_aead_req(rctx);
1688                         break;
1689                 default:
1690                         err = -EINVAL;
1691                 }
1692
1693                 if (err == -EINPROGRESS)
1694                         /* Successfully submitted request for next chunk */
1695                         return;
1696         }
1697
1698 cb_finish:
1699         finish_req(rctx, err);
1700 }
1701
1702 /* ==================== Kernel Cryptographic API ==================== */
1703
1704 /**
1705  * skcipher_enqueue() - Handle skcipher encrypt or decrypt request.
1706  * @req:        Crypto API request
1707  * @encrypt:    true if encrypting; false if decrypting
1708  *
1709  * Return: -EINPROGRESS if request accepted and result will be returned
1710  *                      asynchronously
1711  *         < 0 if an error
1712  */
1713 static int skcipher_enqueue(struct skcipher_request *req, bool encrypt)
1714 {
1715         struct iproc_reqctx_s *rctx = skcipher_request_ctx(req);
1716         struct iproc_ctx_s *ctx =
1717             crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
1718         int err;
1719
1720         flow_log("%s() enc:%u\n", __func__, encrypt);
1721
1722         rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1723                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1724         rctx->parent = &req->base;
1725         rctx->is_encrypt = encrypt;
1726         rctx->bd_suppress = false;
1727         rctx->total_todo = req->cryptlen;
1728         rctx->src_sent = 0;
1729         rctx->total_sent = 0;
1730         rctx->total_received = 0;
1731         rctx->ctx = ctx;
1732
1733         /* Initialize current position in src and dst scatterlists */
1734         rctx->src_sg = req->src;
1735         rctx->src_nents = 0;
1736         rctx->src_skip = 0;
1737         rctx->dst_sg = req->dst;
1738         rctx->dst_nents = 0;
1739         rctx->dst_skip = 0;
1740
1741         if (ctx->cipher.mode == CIPHER_MODE_CBC ||
1742             ctx->cipher.mode == CIPHER_MODE_CTR ||
1743             ctx->cipher.mode == CIPHER_MODE_OFB ||
1744             ctx->cipher.mode == CIPHER_MODE_XTS ||
1745             ctx->cipher.mode == CIPHER_MODE_GCM ||
1746             ctx->cipher.mode == CIPHER_MODE_CCM) {
1747                 rctx->iv_ctr_len =
1748                     crypto_skcipher_ivsize(crypto_skcipher_reqtfm(req));
1749                 memcpy(rctx->msg_buf.iv_ctr, req->iv, rctx->iv_ctr_len);
1750         } else {
1751                 rctx->iv_ctr_len = 0;
1752         }
1753
1754         /* Choose a SPU to process this request */
1755         rctx->chan_idx = select_channel();
1756         err = handle_skcipher_req(rctx);
1757         if (err != -EINPROGRESS)
1758                 /* synchronous result */
1759                 spu_chunk_cleanup(rctx);
1760
1761         return err;
1762 }
1763
1764 static int des_setkey(struct crypto_skcipher *cipher, const u8 *key,
1765                       unsigned int keylen)
1766 {
1767         struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1768         int err;
1769
1770         err = verify_skcipher_des_key(cipher, key);
1771         if (err)
1772                 return err;
1773
1774         ctx->cipher_type = CIPHER_TYPE_DES;
1775         return 0;
1776 }
1777
1778 static int threedes_setkey(struct crypto_skcipher *cipher, const u8 *key,
1779                            unsigned int keylen)
1780 {
1781         struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1782         int err;
1783
1784         err = verify_skcipher_des3_key(cipher, key);
1785         if (err)
1786                 return err;
1787
1788         ctx->cipher_type = CIPHER_TYPE_3DES;
1789         return 0;
1790 }
1791
1792 static int aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
1793                       unsigned int keylen)
1794 {
1795         struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1796
1797         if (ctx->cipher.mode == CIPHER_MODE_XTS)
1798                 /* XTS includes two keys of equal length */
1799                 keylen = keylen / 2;
1800
1801         switch (keylen) {
1802         case AES_KEYSIZE_128:
1803                 ctx->cipher_type = CIPHER_TYPE_AES128;
1804                 break;
1805         case AES_KEYSIZE_192:
1806                 ctx->cipher_type = CIPHER_TYPE_AES192;
1807                 break;
1808         case AES_KEYSIZE_256:
1809                 ctx->cipher_type = CIPHER_TYPE_AES256;
1810                 break;
1811         default:
1812                 return -EINVAL;
1813         }
1814         WARN_ON((ctx->max_payload != SPU_MAX_PAYLOAD_INF) &&
1815                 ((ctx->max_payload % AES_BLOCK_SIZE) != 0));
1816         return 0;
1817 }
1818
1819 static int skcipher_setkey(struct crypto_skcipher *cipher, const u8 *key,
1820                              unsigned int keylen)
1821 {
1822         struct spu_hw *spu = &iproc_priv.spu;
1823         struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1824         struct spu_cipher_parms cipher_parms;
1825         u32 alloc_len = 0;
1826         int err;
1827
1828         flow_log("skcipher_setkey() keylen: %d\n", keylen);
1829         flow_dump("  key: ", key, keylen);
1830
1831         switch (ctx->cipher.alg) {
1832         case CIPHER_ALG_DES:
1833                 err = des_setkey(cipher, key, keylen);
1834                 break;
1835         case CIPHER_ALG_3DES:
1836                 err = threedes_setkey(cipher, key, keylen);
1837                 break;
1838         case CIPHER_ALG_AES:
1839                 err = aes_setkey(cipher, key, keylen);
1840                 break;
1841         default:
1842                 pr_err("%s() Error: unknown cipher alg\n", __func__);
1843                 err = -EINVAL;
1844         }
1845         if (err)
1846                 return err;
1847
1848         memcpy(ctx->enckey, key, keylen);
1849         ctx->enckeylen = keylen;
1850
1851         /* SPU needs XTS keys in the reverse order the crypto API presents */
1852         if ((ctx->cipher.alg == CIPHER_ALG_AES) &&
1853             (ctx->cipher.mode == CIPHER_MODE_XTS)) {
1854                 unsigned int xts_keylen = keylen / 2;
1855
1856                 memcpy(ctx->enckey, key + xts_keylen, xts_keylen);
1857                 memcpy(ctx->enckey + xts_keylen, key, xts_keylen);
1858         }
1859
1860         if (spu->spu_type == SPU_TYPE_SPUM)
1861                 alloc_len = BCM_HDR_LEN + SPU_HEADER_ALLOC_LEN;
1862         else if (spu->spu_type == SPU_TYPE_SPU2)
1863                 alloc_len = BCM_HDR_LEN + SPU2_HEADER_ALLOC_LEN;
1864         memset(ctx->bcm_spu_req_hdr, 0, alloc_len);
1865         cipher_parms.iv_buf = NULL;
1866         cipher_parms.iv_len = crypto_skcipher_ivsize(cipher);
1867         flow_log("%s: iv_len %u\n", __func__, cipher_parms.iv_len);
1868
1869         cipher_parms.alg = ctx->cipher.alg;
1870         cipher_parms.mode = ctx->cipher.mode;
1871         cipher_parms.type = ctx->cipher_type;
1872         cipher_parms.key_buf = ctx->enckey;
1873         cipher_parms.key_len = ctx->enckeylen;
1874
1875         /* Prepend SPU request message with BCM header */
1876         memcpy(ctx->bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN);
1877         ctx->spu_req_hdr_len =
1878             spu->spu_cipher_req_init(ctx->bcm_spu_req_hdr + BCM_HDR_LEN,
1879                                      &cipher_parms);
1880
1881         ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen,
1882                                                           ctx->enckeylen,
1883                                                           false);
1884
1885         atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_CIPHER]);
1886
1887         return 0;
1888 }
1889
1890 static int skcipher_encrypt(struct skcipher_request *req)
1891 {
1892         flow_log("skcipher_encrypt() nbytes:%u\n", req->cryptlen);
1893
1894         return skcipher_enqueue(req, true);
1895 }
1896
1897 static int skcipher_decrypt(struct skcipher_request *req)
1898 {
1899         flow_log("skcipher_decrypt() nbytes:%u\n", req->cryptlen);
1900         return skcipher_enqueue(req, false);
1901 }
1902
1903 static int ahash_enqueue(struct ahash_request *req)
1904 {
1905         struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
1906         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1907         struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
1908         int err;
1909         const char *alg_name;
1910
1911         flow_log("ahash_enqueue() nbytes:%u\n", req->nbytes);
1912
1913         rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1914                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1915         rctx->parent = &req->base;
1916         rctx->ctx = ctx;
1917         rctx->bd_suppress = true;
1918         memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message));
1919
1920         /* Initialize position in src scatterlist */
1921         rctx->src_sg = req->src;
1922         rctx->src_skip = 0;
1923         rctx->src_nents = 0;
1924         rctx->dst_sg = NULL;
1925         rctx->dst_skip = 0;
1926         rctx->dst_nents = 0;
1927
1928         /* SPU2 hardware does not compute hash of zero length data */
1929         if ((rctx->is_final == 1) && (rctx->total_todo == 0) &&
1930             (iproc_priv.spu.spu_type == SPU_TYPE_SPU2)) {
1931                 alg_name = crypto_tfm_alg_name(crypto_ahash_tfm(tfm));
1932                 flow_log("Doing %sfinal %s zero-len hash request in software\n",
1933                          rctx->is_final ? "" : "non-", alg_name);
1934                 err = do_shash((unsigned char *)alg_name, req->result,
1935                                NULL, 0, NULL, 0, ctx->authkey,
1936                                ctx->authkeylen);
1937                 if (err < 0)
1938                         flow_log("Hash request failed with error %d\n", err);
1939                 return err;
1940         }
1941         /* Choose a SPU to process this request */
1942         rctx->chan_idx = select_channel();
1943
1944         err = handle_ahash_req(rctx);
1945         if (err != -EINPROGRESS)
1946                 /* synchronous result */
1947                 spu_chunk_cleanup(rctx);
1948
1949         if (err == -EAGAIN)
1950                 /*
1951                  * we saved data in hash carry, but tell crypto API
1952                  * we successfully completed request.
1953                  */
1954                 err = 0;
1955
1956         return err;
1957 }
1958
1959 static int __ahash_init(struct ahash_request *req)
1960 {
1961         struct spu_hw *spu = &iproc_priv.spu;
1962         struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
1963         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1964         struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
1965
1966         flow_log("%s()\n", __func__);
1967
1968         /* Initialize the context */
1969         rctx->hash_carry_len = 0;
1970         rctx->is_final = 0;
1971
1972         rctx->total_todo = 0;
1973         rctx->src_sent = 0;
1974         rctx->total_sent = 0;
1975         rctx->total_received = 0;
1976
1977         ctx->digestsize = crypto_ahash_digestsize(tfm);
1978         /* If we add a hash whose digest is larger, catch it here. */
1979         WARN_ON(ctx->digestsize > MAX_DIGEST_SIZE);
1980
1981         rctx->is_sw_hmac = false;
1982
1983         ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen, 0,
1984                                                           true);
1985
1986         return 0;
1987 }
1988
1989 /**
1990  * spu_no_incr_hash() - Determine whether incremental hashing is supported.
1991  * @ctx:  Crypto session context
1992  *
1993  * SPU-2 does not support incremental hashing (we'll have to revisit and
1994  * condition based on chip revision or device tree entry if future versions do
1995  * support incremental hash)
1996  *
1997  * SPU-M also doesn't support incremental hashing of AES-XCBC
1998  *
1999  * Return: true if incremental hashing is not supported
2000  *         false otherwise
2001  */
2002 static bool spu_no_incr_hash(struct iproc_ctx_s *ctx)
2003 {
2004         struct spu_hw *spu = &iproc_priv.spu;
2005
2006         if (spu->spu_type == SPU_TYPE_SPU2)
2007                 return true;
2008
2009         if ((ctx->auth.alg == HASH_ALG_AES) &&
2010             (ctx->auth.mode == HASH_MODE_XCBC))
2011                 return true;
2012
2013         /* Otherwise, incremental hashing is supported */
2014         return false;
2015 }
2016
2017 static int ahash_init(struct ahash_request *req)
2018 {
2019         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2020         struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2021         const char *alg_name;
2022         struct crypto_shash *hash;
2023         int ret;
2024         gfp_t gfp;
2025
2026         if (spu_no_incr_hash(ctx)) {
2027                 /*
2028                  * If we get an incremental hashing request and it's not
2029                  * supported by the hardware, we need to handle it in software
2030                  * by calling synchronous hash functions.
2031                  */
2032                 alg_name = crypto_tfm_alg_name(crypto_ahash_tfm(tfm));
2033                 hash = crypto_alloc_shash(alg_name, 0, 0);
2034                 if (IS_ERR(hash)) {
2035                         ret = PTR_ERR(hash);
2036                         goto err;
2037                 }
2038
2039                 gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2040                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2041                 ctx->shash = kmalloc(sizeof(*ctx->shash) +
2042                                      crypto_shash_descsize(hash), gfp);
2043                 if (!ctx->shash) {
2044                         ret = -ENOMEM;
2045                         goto err_hash;
2046                 }
2047                 ctx->shash->tfm = hash;
2048
2049                 /* Set the key using data we already have from setkey */
2050                 if (ctx->authkeylen > 0) {
2051                         ret = crypto_shash_setkey(hash, ctx->authkey,
2052                                                   ctx->authkeylen);
2053                         if (ret)
2054                                 goto err_shash;
2055                 }
2056
2057                 /* Initialize hash w/ this key and other params */
2058                 ret = crypto_shash_init(ctx->shash);
2059                 if (ret)
2060                         goto err_shash;
2061         } else {
2062                 /* Otherwise call the internal function which uses SPU hw */
2063                 ret = __ahash_init(req);
2064         }
2065
2066         return ret;
2067
2068 err_shash:
2069         kfree(ctx->shash);
2070 err_hash:
2071         crypto_free_shash(hash);
2072 err:
2073         return ret;
2074 }
2075
2076 static int __ahash_update(struct ahash_request *req)
2077 {
2078         struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2079
2080         flow_log("ahash_update() nbytes:%u\n", req->nbytes);
2081
2082         if (!req->nbytes)
2083                 return 0;
2084         rctx->total_todo += req->nbytes;
2085         rctx->src_sent = 0;
2086
2087         return ahash_enqueue(req);
2088 }
2089
2090 static int ahash_update(struct ahash_request *req)
2091 {
2092         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2093         struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2094         u8 *tmpbuf;
2095         int ret;
2096         int nents;
2097         gfp_t gfp;
2098
2099         if (spu_no_incr_hash(ctx)) {
2100                 /*
2101                  * If we get an incremental hashing request and it's not
2102                  * supported by the hardware, we need to handle it in software
2103                  * by calling synchronous hash functions.
2104                  */
2105                 if (req->src)
2106                         nents = sg_nents(req->src);
2107                 else
2108                         return -EINVAL;
2109
2110                 /* Copy data from req scatterlist to tmp buffer */
2111                 gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2112                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2113                 tmpbuf = kmalloc(req->nbytes, gfp);
2114                 if (!tmpbuf)
2115                         return -ENOMEM;
2116
2117                 if (sg_copy_to_buffer(req->src, nents, tmpbuf, req->nbytes) !=
2118                                 req->nbytes) {
2119                         kfree(tmpbuf);
2120                         return -EINVAL;
2121                 }
2122
2123                 /* Call synchronous update */
2124                 ret = crypto_shash_update(ctx->shash, tmpbuf, req->nbytes);
2125                 kfree(tmpbuf);
2126         } else {
2127                 /* Otherwise call the internal function which uses SPU hw */
2128                 ret = __ahash_update(req);
2129         }
2130
2131         return ret;
2132 }
2133
2134 static int __ahash_final(struct ahash_request *req)
2135 {
2136         struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2137
2138         flow_log("ahash_final() nbytes:%u\n", req->nbytes);
2139
2140         rctx->is_final = 1;
2141
2142         return ahash_enqueue(req);
2143 }
2144
2145 static int ahash_final(struct ahash_request *req)
2146 {
2147         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2148         struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2149         int ret;
2150
2151         if (spu_no_incr_hash(ctx)) {
2152                 /*
2153                  * If we get an incremental hashing request and it's not
2154                  * supported by the hardware, we need to handle it in software
2155                  * by calling synchronous hash functions.
2156                  */
2157                 ret = crypto_shash_final(ctx->shash, req->result);
2158
2159                 /* Done with hash, can deallocate it now */
2160                 crypto_free_shash(ctx->shash->tfm);
2161                 kfree(ctx->shash);
2162
2163         } else {
2164                 /* Otherwise call the internal function which uses SPU hw */
2165                 ret = __ahash_final(req);
2166         }
2167
2168         return ret;
2169 }
2170
2171 static int __ahash_finup(struct ahash_request *req)
2172 {
2173         struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2174
2175         flow_log("ahash_finup() nbytes:%u\n", req->nbytes);
2176
2177         rctx->total_todo += req->nbytes;
2178         rctx->src_sent = 0;
2179         rctx->is_final = 1;
2180
2181         return ahash_enqueue(req);
2182 }
2183
2184 static int ahash_finup(struct ahash_request *req)
2185 {
2186         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2187         struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2188         u8 *tmpbuf;
2189         int ret;
2190         int nents;
2191         gfp_t gfp;
2192
2193         if (spu_no_incr_hash(ctx)) {
2194                 /*
2195                  * If we get an incremental hashing request and it's not
2196                  * supported by the hardware, we need to handle it in software
2197                  * by calling synchronous hash functions.
2198                  */
2199                 if (req->src) {
2200                         nents = sg_nents(req->src);
2201                 } else {
2202                         ret = -EINVAL;
2203                         goto ahash_finup_exit;
2204                 }
2205
2206                 /* Copy data from req scatterlist to tmp buffer */
2207                 gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2208                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2209                 tmpbuf = kmalloc(req->nbytes, gfp);
2210                 if (!tmpbuf) {
2211                         ret = -ENOMEM;
2212                         goto ahash_finup_exit;
2213                 }
2214
2215                 if (sg_copy_to_buffer(req->src, nents, tmpbuf, req->nbytes) !=
2216                                 req->nbytes) {
2217                         ret = -EINVAL;
2218                         goto ahash_finup_free;
2219                 }
2220
2221                 /* Call synchronous update */
2222                 ret = crypto_shash_finup(ctx->shash, tmpbuf, req->nbytes,
2223                                          req->result);
2224         } else {
2225                 /* Otherwise call the internal function which uses SPU hw */
2226                 return __ahash_finup(req);
2227         }
2228 ahash_finup_free:
2229         kfree(tmpbuf);
2230
2231 ahash_finup_exit:
2232         /* Done with hash, can deallocate it now */
2233         crypto_free_shash(ctx->shash->tfm);
2234         kfree(ctx->shash);
2235         return ret;
2236 }
2237
2238 static int ahash_digest(struct ahash_request *req)
2239 {
2240         int err;
2241
2242         flow_log("ahash_digest() nbytes:%u\n", req->nbytes);
2243
2244         /* whole thing at once */
2245         err = __ahash_init(req);
2246         if (!err)
2247                 err = __ahash_finup(req);
2248
2249         return err;
2250 }
2251
2252 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
2253                         unsigned int keylen)
2254 {
2255         struct iproc_ctx_s *ctx = crypto_ahash_ctx(ahash);
2256
2257         flow_log("%s() ahash:%p key:%p keylen:%u\n",
2258                  __func__, ahash, key, keylen);
2259         flow_dump("  key: ", key, keylen);
2260
2261         if (ctx->auth.alg == HASH_ALG_AES) {
2262                 switch (keylen) {
2263                 case AES_KEYSIZE_128:
2264                         ctx->cipher_type = CIPHER_TYPE_AES128;
2265                         break;
2266                 case AES_KEYSIZE_192:
2267                         ctx->cipher_type = CIPHER_TYPE_AES192;
2268                         break;
2269                 case AES_KEYSIZE_256:
2270                         ctx->cipher_type = CIPHER_TYPE_AES256;
2271                         break;
2272                 default:
2273                         pr_err("%s() Error: Invalid key length\n", __func__);
2274                         return -EINVAL;
2275                 }
2276         } else {
2277                 pr_err("%s() Error: unknown hash alg\n", __func__);
2278                 return -EINVAL;
2279         }
2280         memcpy(ctx->authkey, key, keylen);
2281         ctx->authkeylen = keylen;
2282
2283         return 0;
2284 }
2285
2286 static int ahash_export(struct ahash_request *req, void *out)
2287 {
2288         const struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2289         struct spu_hash_export_s *spu_exp = (struct spu_hash_export_s *)out;
2290
2291         spu_exp->total_todo = rctx->total_todo;
2292         spu_exp->total_sent = rctx->total_sent;
2293         spu_exp->is_sw_hmac = rctx->is_sw_hmac;
2294         memcpy(spu_exp->hash_carry, rctx->hash_carry, sizeof(rctx->hash_carry));
2295         spu_exp->hash_carry_len = rctx->hash_carry_len;
2296         memcpy(spu_exp->incr_hash, rctx->incr_hash, sizeof(rctx->incr_hash));
2297
2298         return 0;
2299 }
2300
2301 static int ahash_import(struct ahash_request *req, const void *in)
2302 {
2303         struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2304         struct spu_hash_export_s *spu_exp = (struct spu_hash_export_s *)in;
2305
2306         rctx->total_todo = spu_exp->total_todo;
2307         rctx->total_sent = spu_exp->total_sent;
2308         rctx->is_sw_hmac = spu_exp->is_sw_hmac;
2309         memcpy(rctx->hash_carry, spu_exp->hash_carry, sizeof(rctx->hash_carry));
2310         rctx->hash_carry_len = spu_exp->hash_carry_len;
2311         memcpy(rctx->incr_hash, spu_exp->incr_hash, sizeof(rctx->incr_hash));
2312
2313         return 0;
2314 }
2315
2316 static int ahash_hmac_setkey(struct crypto_ahash *ahash, const u8 *key,
2317                              unsigned int keylen)
2318 {
2319         struct iproc_ctx_s *ctx = crypto_ahash_ctx(ahash);
2320         unsigned int blocksize =
2321                 crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash));
2322         unsigned int digestsize = crypto_ahash_digestsize(ahash);
2323         unsigned int index;
2324         int rc;
2325
2326         flow_log("%s() ahash:%p key:%p keylen:%u blksz:%u digestsz:%u\n",
2327                  __func__, ahash, key, keylen, blocksize, digestsize);
2328         flow_dump("  key: ", key, keylen);
2329
2330         if (keylen > blocksize) {
2331                 switch (ctx->auth.alg) {
2332                 case HASH_ALG_MD5:
2333                         rc = do_shash("md5", ctx->authkey, key, keylen, NULL,
2334                                       0, NULL, 0);
2335                         break;
2336                 case HASH_ALG_SHA1:
2337                         rc = do_shash("sha1", ctx->authkey, key, keylen, NULL,
2338                                       0, NULL, 0);
2339                         break;
2340                 case HASH_ALG_SHA224:
2341                         rc = do_shash("sha224", ctx->authkey, key, keylen, NULL,
2342                                       0, NULL, 0);
2343                         break;
2344                 case HASH_ALG_SHA256:
2345                         rc = do_shash("sha256", ctx->authkey, key, keylen, NULL,
2346                                       0, NULL, 0);
2347                         break;
2348                 case HASH_ALG_SHA384:
2349                         rc = do_shash("sha384", ctx->authkey, key, keylen, NULL,
2350                                       0, NULL, 0);
2351                         break;
2352                 case HASH_ALG_SHA512:
2353                         rc = do_shash("sha512", ctx->authkey, key, keylen, NULL,
2354                                       0, NULL, 0);
2355                         break;
2356                 case HASH_ALG_SHA3_224:
2357                         rc = do_shash("sha3-224", ctx->authkey, key, keylen,
2358                                       NULL, 0, NULL, 0);
2359                         break;
2360                 case HASH_ALG_SHA3_256:
2361                         rc = do_shash("sha3-256", ctx->authkey, key, keylen,
2362                                       NULL, 0, NULL, 0);
2363                         break;
2364                 case HASH_ALG_SHA3_384:
2365                         rc = do_shash("sha3-384", ctx->authkey, key, keylen,
2366                                       NULL, 0, NULL, 0);
2367                         break;
2368                 case HASH_ALG_SHA3_512:
2369                         rc = do_shash("sha3-512", ctx->authkey, key, keylen,
2370                                       NULL, 0, NULL, 0);
2371                         break;
2372                 default:
2373                         pr_err("%s() Error: unknown hash alg\n", __func__);
2374                         return -EINVAL;
2375                 }
2376                 if (rc < 0) {
2377                         pr_err("%s() Error %d computing shash for %s\n",
2378                                __func__, rc, hash_alg_name[ctx->auth.alg]);
2379                         return rc;
2380                 }
2381                 ctx->authkeylen = digestsize;
2382
2383                 flow_log("  keylen > digestsize... hashed\n");
2384                 flow_dump("  newkey: ", ctx->authkey, ctx->authkeylen);
2385         } else {
2386                 memcpy(ctx->authkey, key, keylen);
2387                 ctx->authkeylen = keylen;
2388         }
2389
2390         /*
2391          * Full HMAC operation in SPUM is not verified,
2392          * So keeping the generation of IPAD, OPAD and
2393          * outer hashing in software.
2394          */
2395         if (iproc_priv.spu.spu_type == SPU_TYPE_SPUM) {
2396                 memcpy(ctx->ipad, ctx->authkey, ctx->authkeylen);
2397                 memset(ctx->ipad + ctx->authkeylen, 0,
2398                        blocksize - ctx->authkeylen);
2399                 ctx->authkeylen = 0;
2400                 memcpy(ctx->opad, ctx->ipad, blocksize);
2401
2402                 for (index = 0; index < blocksize; index++) {
2403                         ctx->ipad[index] ^= HMAC_IPAD_VALUE;
2404                         ctx->opad[index] ^= HMAC_OPAD_VALUE;
2405                 }
2406
2407                 flow_dump("  ipad: ", ctx->ipad, blocksize);
2408                 flow_dump("  opad: ", ctx->opad, blocksize);
2409         }
2410         ctx->digestsize = digestsize;
2411         atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_HMAC]);
2412
2413         return 0;
2414 }
2415
2416 static int ahash_hmac_init(struct ahash_request *req)
2417 {
2418         struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2419         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2420         struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2421         unsigned int blocksize =
2422                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2423
2424         flow_log("ahash_hmac_init()\n");
2425
2426         /* init the context as a hash */
2427         ahash_init(req);
2428
2429         if (!spu_no_incr_hash(ctx)) {
2430                 /* SPU-M can do incr hashing but needs sw for outer HMAC */
2431                 rctx->is_sw_hmac = true;
2432                 ctx->auth.mode = HASH_MODE_HASH;
2433                 /* start with a prepended ipad */
2434                 memcpy(rctx->hash_carry, ctx->ipad, blocksize);
2435                 rctx->hash_carry_len = blocksize;
2436                 rctx->total_todo += blocksize;
2437         }
2438
2439         return 0;
2440 }
2441
2442 static int ahash_hmac_update(struct ahash_request *req)
2443 {
2444         flow_log("ahash_hmac_update() nbytes:%u\n", req->nbytes);
2445
2446         if (!req->nbytes)
2447                 return 0;
2448
2449         return ahash_update(req);
2450 }
2451
2452 static int ahash_hmac_final(struct ahash_request *req)
2453 {
2454         flow_log("ahash_hmac_final() nbytes:%u\n", req->nbytes);
2455
2456         return ahash_final(req);
2457 }
2458
2459 static int ahash_hmac_finup(struct ahash_request *req)
2460 {
2461         flow_log("ahash_hmac_finupl() nbytes:%u\n", req->nbytes);
2462
2463         return ahash_finup(req);
2464 }
2465
2466 static int ahash_hmac_digest(struct ahash_request *req)
2467 {
2468         struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2469         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2470         struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2471         unsigned int blocksize =
2472                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2473
2474         flow_log("ahash_hmac_digest() nbytes:%u\n", req->nbytes);
2475
2476         /* Perform initialization and then call finup */
2477         __ahash_init(req);
2478
2479         if (iproc_priv.spu.spu_type == SPU_TYPE_SPU2) {
2480                 /*
2481                  * SPU2 supports full HMAC implementation in the
2482                  * hardware, need not to generate IPAD, OPAD and
2483                  * outer hash in software.
2484                  * Only for hash key len > hash block size, SPU2
2485                  * expects to perform hashing on the key, shorten
2486                  * it to digest size and feed it as hash key.
2487                  */
2488                 rctx->is_sw_hmac = false;
2489                 ctx->auth.mode = HASH_MODE_HMAC;
2490         } else {
2491                 rctx->is_sw_hmac = true;
2492                 ctx->auth.mode = HASH_MODE_HASH;
2493                 /* start with a prepended ipad */
2494                 memcpy(rctx->hash_carry, ctx->ipad, blocksize);
2495                 rctx->hash_carry_len = blocksize;
2496                 rctx->total_todo += blocksize;
2497         }
2498
2499         return __ahash_finup(req);
2500 }
2501
2502 /* aead helpers */
2503
2504 static int aead_need_fallback(struct aead_request *req)
2505 {
2506         struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2507         struct spu_hw *spu = &iproc_priv.spu;
2508         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2509         struct iproc_ctx_s *ctx = crypto_aead_ctx(aead);
2510         u32 payload_len;
2511
2512         /*
2513          * SPU hardware cannot handle the AES-GCM/CCM case where plaintext
2514          * and AAD are both 0 bytes long. So use fallback in this case.
2515          */
2516         if (((ctx->cipher.mode == CIPHER_MODE_GCM) ||
2517              (ctx->cipher.mode == CIPHER_MODE_CCM)) &&
2518             (req->assoclen == 0)) {
2519                 if ((rctx->is_encrypt && (req->cryptlen == 0)) ||
2520                     (!rctx->is_encrypt && (req->cryptlen == ctx->digestsize))) {
2521                         flow_log("AES GCM/CCM needs fallback for 0 len req\n");
2522                         return 1;
2523                 }
2524         }
2525
2526         /* SPU-M hardware only supports CCM digest size of 8, 12, or 16 bytes */
2527         if ((ctx->cipher.mode == CIPHER_MODE_CCM) &&
2528             (spu->spu_type == SPU_TYPE_SPUM) &&
2529             (ctx->digestsize != 8) && (ctx->digestsize != 12) &&
2530             (ctx->digestsize != 16)) {
2531                 flow_log("%s() AES CCM needs fallback for digest size %d\n",
2532                          __func__, ctx->digestsize);
2533                 return 1;
2534         }
2535
2536         /*
2537          * SPU-M on NSP has an issue where AES-CCM hash is not correct
2538          * when AAD size is 0
2539          */
2540         if ((ctx->cipher.mode == CIPHER_MODE_CCM) &&
2541             (spu->spu_subtype == SPU_SUBTYPE_SPUM_NSP) &&
2542             (req->assoclen == 0)) {
2543                 flow_log("%s() AES_CCM needs fallback for 0 len AAD on NSP\n",
2544                          __func__);
2545                 return 1;
2546         }
2547
2548         /*
2549          * RFC4106 and RFC4543 cannot handle the case where AAD is other than
2550          * 16 or 20 bytes long. So use fallback in this case.
2551          */
2552         if (ctx->cipher.mode == CIPHER_MODE_GCM &&
2553             ctx->cipher.alg == CIPHER_ALG_AES &&
2554             rctx->iv_ctr_len == GCM_RFC4106_IV_SIZE &&
2555             req->assoclen != 16 && req->assoclen != 20) {
2556                 flow_log("RFC4106/RFC4543 needs fallback for assoclen"
2557                          " other than 16 or 20 bytes\n");
2558                 return 1;
2559         }
2560
2561         payload_len = req->cryptlen;
2562         if (spu->spu_type == SPU_TYPE_SPUM)
2563                 payload_len += req->assoclen;
2564
2565         flow_log("%s() payload len: %u\n", __func__, payload_len);
2566
2567         if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
2568                 return 0;
2569         else
2570                 return payload_len > ctx->max_payload;
2571 }
2572
2573 static void aead_complete(struct crypto_async_request *areq, int err)
2574 {
2575         struct aead_request *req =
2576             container_of(areq, struct aead_request, base);
2577         struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2578         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2579
2580         flow_log("%s() err:%d\n", __func__, err);
2581
2582         areq->tfm = crypto_aead_tfm(aead);
2583
2584         areq->complete = rctx->old_complete;
2585         areq->data = rctx->old_data;
2586
2587         areq->complete(areq, err);
2588 }
2589
2590 static int aead_do_fallback(struct aead_request *req, bool is_encrypt)
2591 {
2592         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2593         struct crypto_tfm *tfm = crypto_aead_tfm(aead);
2594         struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2595         struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
2596         int err;
2597         u32 req_flags;
2598
2599         flow_log("%s() enc:%u\n", __func__, is_encrypt);
2600
2601         if (ctx->fallback_cipher) {
2602                 /* Store the cipher tfm and then use the fallback tfm */
2603                 rctx->old_tfm = tfm;
2604                 aead_request_set_tfm(req, ctx->fallback_cipher);
2605                 /*
2606                  * Save the callback and chain ourselves in, so we can restore
2607                  * the tfm
2608                  */
2609                 rctx->old_complete = req->base.complete;
2610                 rctx->old_data = req->base.data;
2611                 req_flags = aead_request_flags(req);
2612                 aead_request_set_callback(req, req_flags, aead_complete, req);
2613                 err = is_encrypt ? crypto_aead_encrypt(req) :
2614                     crypto_aead_decrypt(req);
2615
2616                 if (err == 0) {
2617                         /*
2618                          * fallback was synchronous (did not return
2619                          * -EINPROGRESS). So restore request state here.
2620                          */
2621                         aead_request_set_callback(req, req_flags,
2622                                                   rctx->old_complete, req);
2623                         req->base.data = rctx->old_data;
2624                         aead_request_set_tfm(req, aead);
2625                         flow_log("%s() fallback completed successfully\n\n",
2626                                  __func__);
2627                 }
2628         } else {
2629                 err = -EINVAL;
2630         }
2631
2632         return err;
2633 }
2634
2635 static int aead_enqueue(struct aead_request *req, bool is_encrypt)
2636 {
2637         struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2638         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2639         struct iproc_ctx_s *ctx = crypto_aead_ctx(aead);
2640         int err;
2641
2642         flow_log("%s() enc:%u\n", __func__, is_encrypt);
2643
2644         if (req->assoclen > MAX_ASSOC_SIZE) {
2645                 pr_err
2646                     ("%s() Error: associated data too long. (%u > %u bytes)\n",
2647                      __func__, req->assoclen, MAX_ASSOC_SIZE);
2648                 return -EINVAL;
2649         }
2650
2651         rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2652                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2653         rctx->parent = &req->base;
2654         rctx->is_encrypt = is_encrypt;
2655         rctx->bd_suppress = false;
2656         rctx->total_todo = req->cryptlen;
2657         rctx->src_sent = 0;
2658         rctx->total_sent = 0;
2659         rctx->total_received = 0;
2660         rctx->is_sw_hmac = false;
2661         rctx->ctx = ctx;
2662         memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message));
2663
2664         /* assoc data is at start of src sg */
2665         rctx->assoc = req->src;
2666
2667         /*
2668          * Init current position in src scatterlist to be after assoc data.
2669          * src_skip set to buffer offset where data begins. (Assoc data could
2670          * end in the middle of a buffer.)
2671          */
2672         if (spu_sg_at_offset(req->src, req->assoclen, &rctx->src_sg,
2673                              &rctx->src_skip) < 0) {
2674                 pr_err("%s() Error: Unable to find start of src data\n",
2675                        __func__);
2676                 return -EINVAL;
2677         }
2678
2679         rctx->src_nents = 0;
2680         rctx->dst_nents = 0;
2681         if (req->dst == req->src) {
2682                 rctx->dst_sg = rctx->src_sg;
2683                 rctx->dst_skip = rctx->src_skip;
2684         } else {
2685                 /*
2686                  * Expect req->dst to have room for assoc data followed by
2687                  * output data and ICV, if encrypt. So initialize dst_sg
2688                  * to point beyond assoc len offset.
2689                  */
2690                 if (spu_sg_at_offset(req->dst, req->assoclen, &rctx->dst_sg,
2691                                      &rctx->dst_skip) < 0) {
2692                         pr_err("%s() Error: Unable to find start of dst data\n",
2693                                __func__);
2694                         return -EINVAL;
2695                 }
2696         }
2697
2698         if (ctx->cipher.mode == CIPHER_MODE_CBC ||
2699             ctx->cipher.mode == CIPHER_MODE_CTR ||
2700             ctx->cipher.mode == CIPHER_MODE_OFB ||
2701             ctx->cipher.mode == CIPHER_MODE_XTS ||
2702             ctx->cipher.mode == CIPHER_MODE_GCM) {
2703                 rctx->iv_ctr_len =
2704                         ctx->salt_len +
2705                         crypto_aead_ivsize(crypto_aead_reqtfm(req));
2706         } else if (ctx->cipher.mode == CIPHER_MODE_CCM) {
2707                 rctx->iv_ctr_len = CCM_AES_IV_SIZE;
2708         } else {
2709                 rctx->iv_ctr_len = 0;
2710         }
2711
2712         rctx->hash_carry_len = 0;
2713
2714         flow_log("  src sg: %p\n", req->src);
2715         flow_log("  rctx->src_sg: %p, src_skip %u\n",
2716                  rctx->src_sg, rctx->src_skip);
2717         flow_log("  assoc:  %p, assoclen %u\n", rctx->assoc, req->assoclen);
2718         flow_log("  dst sg: %p\n", req->dst);
2719         flow_log("  rctx->dst_sg: %p, dst_skip %u\n",
2720                  rctx->dst_sg, rctx->dst_skip);
2721         flow_log("  iv_ctr_len:%u\n", rctx->iv_ctr_len);
2722         flow_dump("  iv: ", req->iv, rctx->iv_ctr_len);
2723         flow_log("  authkeylen:%u\n", ctx->authkeylen);
2724         flow_log("  is_esp: %s\n", ctx->is_esp ? "yes" : "no");
2725
2726         if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
2727                 flow_log("  max_payload infinite");
2728         else
2729                 flow_log("  max_payload: %u\n", ctx->max_payload);
2730
2731         if (unlikely(aead_need_fallback(req)))
2732                 return aead_do_fallback(req, is_encrypt);
2733
2734         /*
2735          * Do memory allocations for request after fallback check, because if we
2736          * do fallback, we won't call finish_req() to dealloc.
2737          */
2738         if (rctx->iv_ctr_len) {
2739                 if (ctx->salt_len)
2740                         memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset,
2741                                ctx->salt, ctx->salt_len);
2742                 memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset + ctx->salt_len,
2743                        req->iv,
2744                        rctx->iv_ctr_len - ctx->salt_len - ctx->salt_offset);
2745         }
2746
2747         rctx->chan_idx = select_channel();
2748         err = handle_aead_req(rctx);
2749         if (err != -EINPROGRESS)
2750                 /* synchronous result */
2751                 spu_chunk_cleanup(rctx);
2752
2753         return err;
2754 }
2755
2756 static int aead_authenc_setkey(struct crypto_aead *cipher,
2757                                const u8 *key, unsigned int keylen)
2758 {
2759         struct spu_hw *spu = &iproc_priv.spu;
2760         struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2761         struct crypto_tfm *tfm = crypto_aead_tfm(cipher);
2762         struct crypto_authenc_keys keys;
2763         int ret;
2764
2765         flow_log("%s() aead:%p key:%p keylen:%u\n", __func__, cipher, key,
2766                  keylen);
2767         flow_dump("  key: ", key, keylen);
2768
2769         ret = crypto_authenc_extractkeys(&keys, key, keylen);
2770         if (ret)
2771                 goto badkey;
2772
2773         if (keys.enckeylen > MAX_KEY_SIZE ||
2774             keys.authkeylen > MAX_KEY_SIZE)
2775                 goto badkey;
2776
2777         ctx->enckeylen = keys.enckeylen;
2778         ctx->authkeylen = keys.authkeylen;
2779
2780         memcpy(ctx->enckey, keys.enckey, keys.enckeylen);
2781         /* May end up padding auth key. So make sure it's zeroed. */
2782         memset(ctx->authkey, 0, sizeof(ctx->authkey));
2783         memcpy(ctx->authkey, keys.authkey, keys.authkeylen);
2784
2785         switch (ctx->alg->cipher_info.alg) {
2786         case CIPHER_ALG_DES:
2787                 if (verify_aead_des_key(cipher, keys.enckey, keys.enckeylen))
2788                         return -EINVAL;
2789
2790                 ctx->cipher_type = CIPHER_TYPE_DES;
2791                 break;
2792         case CIPHER_ALG_3DES:
2793                 if (verify_aead_des3_key(cipher, keys.enckey, keys.enckeylen))
2794                         return -EINVAL;
2795
2796                 ctx->cipher_type = CIPHER_TYPE_3DES;
2797                 break;
2798         case CIPHER_ALG_AES:
2799                 switch (ctx->enckeylen) {
2800                 case AES_KEYSIZE_128:
2801                         ctx->cipher_type = CIPHER_TYPE_AES128;
2802                         break;
2803                 case AES_KEYSIZE_192:
2804                         ctx->cipher_type = CIPHER_TYPE_AES192;
2805                         break;
2806                 case AES_KEYSIZE_256:
2807                         ctx->cipher_type = CIPHER_TYPE_AES256;
2808                         break;
2809                 default:
2810                         goto badkey;
2811                 }
2812                 break;
2813         default:
2814                 pr_err("%s() Error: Unknown cipher alg\n", __func__);
2815                 return -EINVAL;
2816         }
2817
2818         flow_log("  enckeylen:%u authkeylen:%u\n", ctx->enckeylen,
2819                  ctx->authkeylen);
2820         flow_dump("  enc: ", ctx->enckey, ctx->enckeylen);
2821         flow_dump("  auth: ", ctx->authkey, ctx->authkeylen);
2822
2823         /* setkey the fallback just in case we needto use it */
2824         if (ctx->fallback_cipher) {
2825                 flow_log("  running fallback setkey()\n");
2826
2827                 ctx->fallback_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
2828                 ctx->fallback_cipher->base.crt_flags |=
2829                     tfm->crt_flags & CRYPTO_TFM_REQ_MASK;
2830                 ret = crypto_aead_setkey(ctx->fallback_cipher, key, keylen);
2831                 if (ret)
2832                         flow_log("  fallback setkey() returned:%d\n", ret);
2833         }
2834
2835         ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen,
2836                                                           ctx->enckeylen,
2837                                                           false);
2838
2839         atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_AEAD]);
2840
2841         return ret;
2842
2843 badkey:
2844         ctx->enckeylen = 0;
2845         ctx->authkeylen = 0;
2846         ctx->digestsize = 0;
2847
2848         return -EINVAL;
2849 }
2850
2851 static int aead_gcm_ccm_setkey(struct crypto_aead *cipher,
2852                                const u8 *key, unsigned int keylen)
2853 {
2854         struct spu_hw *spu = &iproc_priv.spu;
2855         struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2856         struct crypto_tfm *tfm = crypto_aead_tfm(cipher);
2857
2858         int ret = 0;
2859
2860         flow_log("%s() keylen:%u\n", __func__, keylen);
2861         flow_dump("  key: ", key, keylen);
2862
2863         if (!ctx->is_esp)
2864                 ctx->digestsize = keylen;
2865
2866         ctx->enckeylen = keylen;
2867         ctx->authkeylen = 0;
2868
2869         switch (ctx->enckeylen) {
2870         case AES_KEYSIZE_128:
2871                 ctx->cipher_type = CIPHER_TYPE_AES128;
2872                 break;
2873         case AES_KEYSIZE_192:
2874                 ctx->cipher_type = CIPHER_TYPE_AES192;
2875                 break;
2876         case AES_KEYSIZE_256:
2877                 ctx->cipher_type = CIPHER_TYPE_AES256;
2878                 break;
2879         default:
2880                 goto badkey;
2881         }
2882
2883         memcpy(ctx->enckey, key, ctx->enckeylen);
2884
2885         flow_log("  enckeylen:%u authkeylen:%u\n", ctx->enckeylen,
2886                  ctx->authkeylen);
2887         flow_dump("  enc: ", ctx->enckey, ctx->enckeylen);
2888         flow_dump("  auth: ", ctx->authkey, ctx->authkeylen);
2889
2890         /* setkey the fallback just in case we need to use it */
2891         if (ctx->fallback_cipher) {
2892                 flow_log("  running fallback setkey()\n");
2893
2894                 ctx->fallback_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
2895                 ctx->fallback_cipher->base.crt_flags |=
2896                     tfm->crt_flags & CRYPTO_TFM_REQ_MASK;
2897                 ret = crypto_aead_setkey(ctx->fallback_cipher, key,
2898                                          keylen + ctx->salt_len);
2899                 if (ret)
2900                         flow_log("  fallback setkey() returned:%d\n", ret);
2901         }
2902
2903         ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen,
2904                                                           ctx->enckeylen,
2905                                                           false);
2906
2907         atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_AEAD]);
2908
2909         flow_log("  enckeylen:%u authkeylen:%u\n", ctx->enckeylen,
2910                  ctx->authkeylen);
2911
2912         return ret;
2913
2914 badkey:
2915         ctx->enckeylen = 0;
2916         ctx->authkeylen = 0;
2917         ctx->digestsize = 0;
2918
2919         return -EINVAL;
2920 }
2921
2922 /**
2923  * aead_gcm_esp_setkey() - setkey() operation for ESP variant of GCM AES.
2924  * @cipher: AEAD structure
2925  * @key:    Key followed by 4 bytes of salt
2926  * @keylen: Length of key plus salt, in bytes
2927  *
2928  * Extracts salt from key and stores it to be prepended to IV on each request.
2929  * Digest is always 16 bytes
2930  *
2931  * Return: Value from generic gcm setkey.
2932  */
2933 static int aead_gcm_esp_setkey(struct crypto_aead *cipher,
2934                                const u8 *key, unsigned int keylen)
2935 {
2936         struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2937
2938         flow_log("%s\n", __func__);
2939
2940         if (keylen < GCM_ESP_SALT_SIZE)
2941                 return -EINVAL;
2942
2943         ctx->salt_len = GCM_ESP_SALT_SIZE;
2944         ctx->salt_offset = GCM_ESP_SALT_OFFSET;
2945         memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE);
2946         keylen -= GCM_ESP_SALT_SIZE;
2947         ctx->digestsize = GCM_ESP_DIGESTSIZE;
2948         ctx->is_esp = true;
2949         flow_dump("salt: ", ctx->salt, GCM_ESP_SALT_SIZE);
2950
2951         return aead_gcm_ccm_setkey(cipher, key, keylen);
2952 }
2953
2954 /**
2955  * rfc4543_gcm_esp_setkey() - setkey operation for RFC4543 variant of GCM/GMAC.
2956  * @cipher: AEAD structure
2957  * @key:    Key followed by 4 bytes of salt
2958  * @keylen: Length of key plus salt, in bytes
2959  *
2960  * Extracts salt from key and stores it to be prepended to IV on each request.
2961  * Digest is always 16 bytes
2962  *
2963  * Return: Value from generic gcm setkey.
2964  */
2965 static int rfc4543_gcm_esp_setkey(struct crypto_aead *cipher,
2966                                   const u8 *key, unsigned int keylen)
2967 {
2968         struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2969
2970         flow_log("%s\n", __func__);
2971
2972         if (keylen < GCM_ESP_SALT_SIZE)
2973                 return -EINVAL;
2974
2975         ctx->salt_len = GCM_ESP_SALT_SIZE;
2976         ctx->salt_offset = GCM_ESP_SALT_OFFSET;
2977         memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE);
2978         keylen -= GCM_ESP_SALT_SIZE;
2979         ctx->digestsize = GCM_ESP_DIGESTSIZE;
2980         ctx->is_esp = true;
2981         ctx->is_rfc4543 = true;
2982         flow_dump("salt: ", ctx->salt, GCM_ESP_SALT_SIZE);
2983
2984         return aead_gcm_ccm_setkey(cipher, key, keylen);
2985 }
2986
2987 /**
2988  * aead_ccm_esp_setkey() - setkey() operation for ESP variant of CCM AES.
2989  * @cipher: AEAD structure
2990  * @key:    Key followed by 4 bytes of salt
2991  * @keylen: Length of key plus salt, in bytes
2992  *
2993  * Extracts salt from key and stores it to be prepended to IV on each request.
2994  * Digest is always 16 bytes
2995  *
2996  * Return: Value from generic ccm setkey.
2997  */
2998 static int aead_ccm_esp_setkey(struct crypto_aead *cipher,
2999                                const u8 *key, unsigned int keylen)
3000 {
3001         struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
3002
3003         flow_log("%s\n", __func__);
3004
3005         if (keylen < CCM_ESP_SALT_SIZE)
3006                 return -EINVAL;
3007
3008         ctx->salt_len = CCM_ESP_SALT_SIZE;
3009         ctx->salt_offset = CCM_ESP_SALT_OFFSET;
3010         memcpy(ctx->salt, key + keylen - CCM_ESP_SALT_SIZE, CCM_ESP_SALT_SIZE);
3011         keylen -= CCM_ESP_SALT_SIZE;
3012         ctx->is_esp = true;
3013         flow_dump("salt: ", ctx->salt, CCM_ESP_SALT_SIZE);
3014
3015         return aead_gcm_ccm_setkey(cipher, key, keylen);
3016 }
3017
3018 static int aead_setauthsize(struct crypto_aead *cipher, unsigned int authsize)
3019 {
3020         struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
3021         int ret = 0;
3022
3023         flow_log("%s() authkeylen:%u authsize:%u\n",
3024                  __func__, ctx->authkeylen, authsize);
3025
3026         ctx->digestsize = authsize;
3027
3028         /* setkey the fallback just in case we needto use it */
3029         if (ctx->fallback_cipher) {
3030                 flow_log("  running fallback setauth()\n");
3031
3032                 ret = crypto_aead_setauthsize(ctx->fallback_cipher, authsize);
3033                 if (ret)
3034                         flow_log("  fallback setauth() returned:%d\n", ret);
3035         }
3036
3037         return ret;
3038 }
3039
3040 static int aead_encrypt(struct aead_request *req)
3041 {
3042         flow_log("%s() cryptlen:%u %08x\n", __func__, req->cryptlen,
3043                  req->cryptlen);
3044         dump_sg(req->src, 0, req->cryptlen + req->assoclen);
3045         flow_log("  assoc_len:%u\n", req->assoclen);
3046
3047         return aead_enqueue(req, true);
3048 }
3049
3050 static int aead_decrypt(struct aead_request *req)
3051 {
3052         flow_log("%s() cryptlen:%u\n", __func__, req->cryptlen);
3053         dump_sg(req->src, 0, req->cryptlen + req->assoclen);
3054         flow_log("  assoc_len:%u\n", req->assoclen);
3055
3056         return aead_enqueue(req, false);
3057 }
3058
3059 /* ==================== Supported Cipher Algorithms ==================== */
3060
3061 static struct iproc_alg_s driver_algs[] = {
3062         {
3063          .type = CRYPTO_ALG_TYPE_AEAD,
3064          .alg.aead = {
3065                  .base = {
3066                         .cra_name = "gcm(aes)",
3067                         .cra_driver_name = "gcm-aes-iproc",
3068                         .cra_blocksize = AES_BLOCK_SIZE,
3069                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3070                  },
3071                  .setkey = aead_gcm_ccm_setkey,
3072                  .ivsize = GCM_AES_IV_SIZE,
3073                 .maxauthsize = AES_BLOCK_SIZE,
3074          },
3075          .cipher_info = {
3076                          .alg = CIPHER_ALG_AES,
3077                          .mode = CIPHER_MODE_GCM,
3078                          },
3079          .auth_info = {
3080                        .alg = HASH_ALG_AES,
3081                        .mode = HASH_MODE_GCM,
3082                        },
3083          .auth_first = 0,
3084          },
3085         {
3086          .type = CRYPTO_ALG_TYPE_AEAD,
3087          .alg.aead = {
3088                  .base = {
3089                         .cra_name = "ccm(aes)",
3090                         .cra_driver_name = "ccm-aes-iproc",
3091                         .cra_blocksize = AES_BLOCK_SIZE,
3092                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3093                  },
3094                  .setkey = aead_gcm_ccm_setkey,
3095                  .ivsize = CCM_AES_IV_SIZE,
3096                 .maxauthsize = AES_BLOCK_SIZE,
3097          },
3098          .cipher_info = {
3099                          .alg = CIPHER_ALG_AES,
3100                          .mode = CIPHER_MODE_CCM,
3101                          },
3102          .auth_info = {
3103                        .alg = HASH_ALG_AES,
3104                        .mode = HASH_MODE_CCM,
3105                        },
3106          .auth_first = 0,
3107          },
3108         {
3109          .type = CRYPTO_ALG_TYPE_AEAD,
3110          .alg.aead = {
3111                  .base = {
3112                         .cra_name = "rfc4106(gcm(aes))",
3113                         .cra_driver_name = "gcm-aes-esp-iproc",
3114                         .cra_blocksize = AES_BLOCK_SIZE,
3115                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3116                  },
3117                  .setkey = aead_gcm_esp_setkey,
3118                  .ivsize = GCM_RFC4106_IV_SIZE,
3119                  .maxauthsize = AES_BLOCK_SIZE,
3120          },
3121          .cipher_info = {
3122                          .alg = CIPHER_ALG_AES,
3123                          .mode = CIPHER_MODE_GCM,
3124                          },
3125          .auth_info = {
3126                        .alg = HASH_ALG_AES,
3127                        .mode = HASH_MODE_GCM,
3128                        },
3129          .auth_first = 0,
3130          },
3131         {
3132          .type = CRYPTO_ALG_TYPE_AEAD,
3133          .alg.aead = {
3134                  .base = {
3135                         .cra_name = "rfc4309(ccm(aes))",
3136                         .cra_driver_name = "ccm-aes-esp-iproc",
3137                         .cra_blocksize = AES_BLOCK_SIZE,
3138                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3139                  },
3140                  .setkey = aead_ccm_esp_setkey,
3141                  .ivsize = CCM_AES_IV_SIZE,
3142                  .maxauthsize = AES_BLOCK_SIZE,
3143          },
3144          .cipher_info = {
3145                          .alg = CIPHER_ALG_AES,
3146                          .mode = CIPHER_MODE_CCM,
3147                          },
3148          .auth_info = {
3149                        .alg = HASH_ALG_AES,
3150                        .mode = HASH_MODE_CCM,
3151                        },
3152          .auth_first = 0,
3153          },
3154         {
3155          .type = CRYPTO_ALG_TYPE_AEAD,
3156          .alg.aead = {
3157                  .base = {
3158                         .cra_name = "rfc4543(gcm(aes))",
3159                         .cra_driver_name = "gmac-aes-esp-iproc",
3160                         .cra_blocksize = AES_BLOCK_SIZE,
3161                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3162                  },
3163                  .setkey = rfc4543_gcm_esp_setkey,
3164                  .ivsize = GCM_RFC4106_IV_SIZE,
3165                  .maxauthsize = AES_BLOCK_SIZE,
3166          },
3167          .cipher_info = {
3168                          .alg = CIPHER_ALG_AES,
3169                          .mode = CIPHER_MODE_GCM,
3170                          },
3171          .auth_info = {
3172                        .alg = HASH_ALG_AES,
3173                        .mode = HASH_MODE_GCM,
3174                        },
3175          .auth_first = 0,
3176          },
3177         {
3178          .type = CRYPTO_ALG_TYPE_AEAD,
3179          .alg.aead = {
3180                  .base = {
3181                         .cra_name = "authenc(hmac(md5),cbc(aes))",
3182                         .cra_driver_name = "authenc-hmac-md5-cbc-aes-iproc",
3183                         .cra_blocksize = AES_BLOCK_SIZE,
3184                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3185                                      CRYPTO_ALG_ASYNC |
3186                                      CRYPTO_ALG_ALLOCATES_MEMORY
3187                  },
3188                  .setkey = aead_authenc_setkey,
3189                 .ivsize = AES_BLOCK_SIZE,
3190                 .maxauthsize = MD5_DIGEST_SIZE,
3191          },
3192          .cipher_info = {
3193                          .alg = CIPHER_ALG_AES,
3194                          .mode = CIPHER_MODE_CBC,
3195                          },
3196          .auth_info = {
3197                        .alg = HASH_ALG_MD5,
3198                        .mode = HASH_MODE_HMAC,
3199                        },
3200          .auth_first = 0,
3201          },
3202         {
3203          .type = CRYPTO_ALG_TYPE_AEAD,
3204          .alg.aead = {
3205                  .base = {
3206                         .cra_name = "authenc(hmac(sha1),cbc(aes))",
3207                         .cra_driver_name = "authenc-hmac-sha1-cbc-aes-iproc",
3208                         .cra_blocksize = AES_BLOCK_SIZE,
3209                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3210                                      CRYPTO_ALG_ASYNC |
3211                                      CRYPTO_ALG_ALLOCATES_MEMORY
3212                  },
3213                  .setkey = aead_authenc_setkey,
3214                  .ivsize = AES_BLOCK_SIZE,
3215                  .maxauthsize = SHA1_DIGEST_SIZE,
3216          },
3217          .cipher_info = {
3218                          .alg = CIPHER_ALG_AES,
3219                          .mode = CIPHER_MODE_CBC,
3220                          },
3221          .auth_info = {
3222                        .alg = HASH_ALG_SHA1,
3223                        .mode = HASH_MODE_HMAC,
3224                        },
3225          .auth_first = 0,
3226          },
3227         {
3228          .type = CRYPTO_ALG_TYPE_AEAD,
3229          .alg.aead = {
3230                  .base = {
3231                         .cra_name = "authenc(hmac(sha256),cbc(aes))",
3232                         .cra_driver_name = "authenc-hmac-sha256-cbc-aes-iproc",
3233                         .cra_blocksize = AES_BLOCK_SIZE,
3234                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3235                                      CRYPTO_ALG_ASYNC |
3236                                      CRYPTO_ALG_ALLOCATES_MEMORY
3237                  },
3238                  .setkey = aead_authenc_setkey,
3239                  .ivsize = AES_BLOCK_SIZE,
3240                  .maxauthsize = SHA256_DIGEST_SIZE,
3241          },
3242          .cipher_info = {
3243                          .alg = CIPHER_ALG_AES,
3244                          .mode = CIPHER_MODE_CBC,
3245                          },
3246          .auth_info = {
3247                        .alg = HASH_ALG_SHA256,
3248                        .mode = HASH_MODE_HMAC,
3249                        },
3250          .auth_first = 0,
3251          },
3252         {
3253          .type = CRYPTO_ALG_TYPE_AEAD,
3254          .alg.aead = {
3255                  .base = {
3256                         .cra_name = "authenc(hmac(md5),cbc(des))",
3257                         .cra_driver_name = "authenc-hmac-md5-cbc-des-iproc",
3258                         .cra_blocksize = DES_BLOCK_SIZE,
3259                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3260                                      CRYPTO_ALG_ASYNC |
3261                                      CRYPTO_ALG_ALLOCATES_MEMORY
3262                  },
3263                  .setkey = aead_authenc_setkey,
3264                  .ivsize = DES_BLOCK_SIZE,
3265                  .maxauthsize = MD5_DIGEST_SIZE,
3266          },
3267          .cipher_info = {
3268                          .alg = CIPHER_ALG_DES,
3269                          .mode = CIPHER_MODE_CBC,
3270                          },
3271          .auth_info = {
3272                        .alg = HASH_ALG_MD5,
3273                        .mode = HASH_MODE_HMAC,
3274                        },
3275          .auth_first = 0,
3276          },
3277         {
3278          .type = CRYPTO_ALG_TYPE_AEAD,
3279          .alg.aead = {
3280                  .base = {
3281                         .cra_name = "authenc(hmac(sha1),cbc(des))",
3282                         .cra_driver_name = "authenc-hmac-sha1-cbc-des-iproc",
3283                         .cra_blocksize = DES_BLOCK_SIZE,
3284                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3285                                      CRYPTO_ALG_ASYNC |
3286                                      CRYPTO_ALG_ALLOCATES_MEMORY
3287                  },
3288                  .setkey = aead_authenc_setkey,
3289                  .ivsize = DES_BLOCK_SIZE,
3290                  .maxauthsize = SHA1_DIGEST_SIZE,
3291          },
3292          .cipher_info = {
3293                          .alg = CIPHER_ALG_DES,
3294                          .mode = CIPHER_MODE_CBC,
3295                          },
3296          .auth_info = {
3297                        .alg = HASH_ALG_SHA1,
3298                        .mode = HASH_MODE_HMAC,
3299                        },
3300          .auth_first = 0,
3301          },
3302         {
3303          .type = CRYPTO_ALG_TYPE_AEAD,
3304          .alg.aead = {
3305                  .base = {
3306                         .cra_name = "authenc(hmac(sha224),cbc(des))",
3307                         .cra_driver_name = "authenc-hmac-sha224-cbc-des-iproc",
3308                         .cra_blocksize = DES_BLOCK_SIZE,
3309                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3310                                      CRYPTO_ALG_ASYNC |
3311                                      CRYPTO_ALG_ALLOCATES_MEMORY
3312                  },
3313                  .setkey = aead_authenc_setkey,
3314                  .ivsize = DES_BLOCK_SIZE,
3315                  .maxauthsize = SHA224_DIGEST_SIZE,
3316          },
3317          .cipher_info = {
3318                          .alg = CIPHER_ALG_DES,
3319                          .mode = CIPHER_MODE_CBC,
3320                          },
3321          .auth_info = {
3322                        .alg = HASH_ALG_SHA224,
3323                        .mode = HASH_MODE_HMAC,
3324                        },
3325          .auth_first = 0,
3326          },
3327         {
3328          .type = CRYPTO_ALG_TYPE_AEAD,
3329          .alg.aead = {
3330                  .base = {
3331                         .cra_name = "authenc(hmac(sha256),cbc(des))",
3332                         .cra_driver_name = "authenc-hmac-sha256-cbc-des-iproc",
3333                         .cra_blocksize = DES_BLOCK_SIZE,
3334                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3335                                      CRYPTO_ALG_ASYNC |
3336                                      CRYPTO_ALG_ALLOCATES_MEMORY
3337                  },
3338                  .setkey = aead_authenc_setkey,
3339                  .ivsize = DES_BLOCK_SIZE,
3340                  .maxauthsize = SHA256_DIGEST_SIZE,
3341          },
3342          .cipher_info = {
3343                          .alg = CIPHER_ALG_DES,
3344                          .mode = CIPHER_MODE_CBC,
3345                          },
3346          .auth_info = {
3347                        .alg = HASH_ALG_SHA256,
3348                        .mode = HASH_MODE_HMAC,
3349                        },
3350          .auth_first = 0,
3351          },
3352         {
3353          .type = CRYPTO_ALG_TYPE_AEAD,
3354          .alg.aead = {
3355                  .base = {
3356                         .cra_name = "authenc(hmac(sha384),cbc(des))",
3357                         .cra_driver_name = "authenc-hmac-sha384-cbc-des-iproc",
3358                         .cra_blocksize = DES_BLOCK_SIZE,
3359                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3360                                      CRYPTO_ALG_ASYNC |
3361                                      CRYPTO_ALG_ALLOCATES_MEMORY
3362                  },
3363                  .setkey = aead_authenc_setkey,
3364                  .ivsize = DES_BLOCK_SIZE,
3365                  .maxauthsize = SHA384_DIGEST_SIZE,
3366          },
3367          .cipher_info = {
3368                          .alg = CIPHER_ALG_DES,
3369                          .mode = CIPHER_MODE_CBC,
3370                          },
3371          .auth_info = {
3372                        .alg = HASH_ALG_SHA384,
3373                        .mode = HASH_MODE_HMAC,
3374                        },
3375          .auth_first = 0,
3376          },
3377         {
3378          .type = CRYPTO_ALG_TYPE_AEAD,
3379          .alg.aead = {
3380                  .base = {
3381                         .cra_name = "authenc(hmac(sha512),cbc(des))",
3382                         .cra_driver_name = "authenc-hmac-sha512-cbc-des-iproc",
3383                         .cra_blocksize = DES_BLOCK_SIZE,
3384                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3385                                      CRYPTO_ALG_ASYNC |
3386                                      CRYPTO_ALG_ALLOCATES_MEMORY
3387                  },
3388                  .setkey = aead_authenc_setkey,
3389                  .ivsize = DES_BLOCK_SIZE,
3390                  .maxauthsize = SHA512_DIGEST_SIZE,
3391          },
3392          .cipher_info = {
3393                          .alg = CIPHER_ALG_DES,
3394                          .mode = CIPHER_MODE_CBC,
3395                          },
3396          .auth_info = {
3397                        .alg = HASH_ALG_SHA512,
3398                        .mode = HASH_MODE_HMAC,
3399                        },
3400          .auth_first = 0,
3401          },
3402         {
3403          .type = CRYPTO_ALG_TYPE_AEAD,
3404          .alg.aead = {
3405                  .base = {
3406                         .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
3407                         .cra_driver_name = "authenc-hmac-md5-cbc-des3-iproc",
3408                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3409                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3410                                      CRYPTO_ALG_ASYNC |
3411                                      CRYPTO_ALG_ALLOCATES_MEMORY
3412                  },
3413                  .setkey = aead_authenc_setkey,
3414                  .ivsize = DES3_EDE_BLOCK_SIZE,
3415                  .maxauthsize = MD5_DIGEST_SIZE,
3416          },
3417          .cipher_info = {
3418                          .alg = CIPHER_ALG_3DES,
3419                          .mode = CIPHER_MODE_CBC,
3420                          },
3421          .auth_info = {
3422                        .alg = HASH_ALG_MD5,
3423                        .mode = HASH_MODE_HMAC,
3424                        },
3425          .auth_first = 0,
3426          },
3427         {
3428          .type = CRYPTO_ALG_TYPE_AEAD,
3429          .alg.aead = {
3430                  .base = {
3431                         .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
3432                         .cra_driver_name = "authenc-hmac-sha1-cbc-des3-iproc",
3433                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3434                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3435                                      CRYPTO_ALG_ASYNC |
3436                                      CRYPTO_ALG_ALLOCATES_MEMORY
3437                  },
3438                  .setkey = aead_authenc_setkey,
3439                  .ivsize = DES3_EDE_BLOCK_SIZE,
3440                  .maxauthsize = SHA1_DIGEST_SIZE,
3441          },
3442          .cipher_info = {
3443                          .alg = CIPHER_ALG_3DES,
3444                          .mode = CIPHER_MODE_CBC,
3445                          },
3446          .auth_info = {
3447                        .alg = HASH_ALG_SHA1,
3448                        .mode = HASH_MODE_HMAC,
3449                        },
3450          .auth_first = 0,
3451          },
3452         {
3453          .type = CRYPTO_ALG_TYPE_AEAD,
3454          .alg.aead = {
3455                  .base = {
3456                         .cra_name = "authenc(hmac(sha224),cbc(des3_ede))",
3457                         .cra_driver_name = "authenc-hmac-sha224-cbc-des3-iproc",
3458                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3459                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3460                                      CRYPTO_ALG_ASYNC |
3461                                      CRYPTO_ALG_ALLOCATES_MEMORY
3462                  },
3463                  .setkey = aead_authenc_setkey,
3464                  .ivsize = DES3_EDE_BLOCK_SIZE,
3465                  .maxauthsize = SHA224_DIGEST_SIZE,
3466          },
3467          .cipher_info = {
3468                          .alg = CIPHER_ALG_3DES,
3469                          .mode = CIPHER_MODE_CBC,
3470                          },
3471          .auth_info = {
3472                        .alg = HASH_ALG_SHA224,
3473                        .mode = HASH_MODE_HMAC,
3474                        },
3475          .auth_first = 0,
3476          },
3477         {
3478          .type = CRYPTO_ALG_TYPE_AEAD,
3479          .alg.aead = {
3480                  .base = {
3481                         .cra_name = "authenc(hmac(sha256),cbc(des3_ede))",
3482                         .cra_driver_name = "authenc-hmac-sha256-cbc-des3-iproc",
3483                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3484                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3485                                      CRYPTO_ALG_ASYNC |
3486                                      CRYPTO_ALG_ALLOCATES_MEMORY
3487                  },
3488                  .setkey = aead_authenc_setkey,
3489                  .ivsize = DES3_EDE_BLOCK_SIZE,
3490                  .maxauthsize = SHA256_DIGEST_SIZE,
3491          },
3492          .cipher_info = {
3493                          .alg = CIPHER_ALG_3DES,
3494                          .mode = CIPHER_MODE_CBC,
3495                          },
3496          .auth_info = {
3497                        .alg = HASH_ALG_SHA256,
3498                        .mode = HASH_MODE_HMAC,
3499                        },
3500          .auth_first = 0,
3501          },
3502         {
3503          .type = CRYPTO_ALG_TYPE_AEAD,
3504          .alg.aead = {
3505                  .base = {
3506                         .cra_name = "authenc(hmac(sha384),cbc(des3_ede))",
3507                         .cra_driver_name = "authenc-hmac-sha384-cbc-des3-iproc",
3508                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3509                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3510                                      CRYPTO_ALG_ASYNC |
3511                                      CRYPTO_ALG_ALLOCATES_MEMORY
3512                  },
3513                  .setkey = aead_authenc_setkey,
3514                  .ivsize = DES3_EDE_BLOCK_SIZE,
3515                  .maxauthsize = SHA384_DIGEST_SIZE,
3516          },
3517          .cipher_info = {
3518                          .alg = CIPHER_ALG_3DES,
3519                          .mode = CIPHER_MODE_CBC,
3520                          },
3521          .auth_info = {
3522                        .alg = HASH_ALG_SHA384,
3523                        .mode = HASH_MODE_HMAC,
3524                        },
3525          .auth_first = 0,
3526          },
3527         {
3528          .type = CRYPTO_ALG_TYPE_AEAD,
3529          .alg.aead = {
3530                  .base = {
3531                         .cra_name = "authenc(hmac(sha512),cbc(des3_ede))",
3532                         .cra_driver_name = "authenc-hmac-sha512-cbc-des3-iproc",
3533                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3534                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3535                                      CRYPTO_ALG_ASYNC |
3536                                      CRYPTO_ALG_ALLOCATES_MEMORY
3537                  },
3538                  .setkey = aead_authenc_setkey,
3539                  .ivsize = DES3_EDE_BLOCK_SIZE,
3540                  .maxauthsize = SHA512_DIGEST_SIZE,
3541          },
3542          .cipher_info = {
3543                          .alg = CIPHER_ALG_3DES,
3544                          .mode = CIPHER_MODE_CBC,
3545                          },
3546          .auth_info = {
3547                        .alg = HASH_ALG_SHA512,
3548                        .mode = HASH_MODE_HMAC,
3549                        },
3550          .auth_first = 0,
3551          },
3552
3553 /* SKCIPHER algorithms. */
3554         {
3555          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3556          .alg.skcipher = {
3557                         .base.cra_name = "ofb(des)",
3558                         .base.cra_driver_name = "ofb-des-iproc",
3559                         .base.cra_blocksize = DES_BLOCK_SIZE,
3560                         .min_keysize = DES_KEY_SIZE,
3561                         .max_keysize = DES_KEY_SIZE,
3562                         .ivsize = DES_BLOCK_SIZE,
3563                         },
3564          .cipher_info = {
3565                          .alg = CIPHER_ALG_DES,
3566                          .mode = CIPHER_MODE_OFB,
3567                          },
3568          .auth_info = {
3569                        .alg = HASH_ALG_NONE,
3570                        .mode = HASH_MODE_NONE,
3571                        },
3572          },
3573         {
3574          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3575          .alg.skcipher = {
3576                         .base.cra_name = "cbc(des)",
3577                         .base.cra_driver_name = "cbc-des-iproc",
3578                         .base.cra_blocksize = DES_BLOCK_SIZE,
3579                         .min_keysize = DES_KEY_SIZE,
3580                         .max_keysize = DES_KEY_SIZE,
3581                         .ivsize = DES_BLOCK_SIZE,
3582                         },
3583          .cipher_info = {
3584                          .alg = CIPHER_ALG_DES,
3585                          .mode = CIPHER_MODE_CBC,
3586                          },
3587          .auth_info = {
3588                        .alg = HASH_ALG_NONE,
3589                        .mode = HASH_MODE_NONE,
3590                        },
3591          },
3592         {
3593          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3594          .alg.skcipher = {
3595                         .base.cra_name = "ecb(des)",
3596                         .base.cra_driver_name = "ecb-des-iproc",
3597                         .base.cra_blocksize = DES_BLOCK_SIZE,
3598                         .min_keysize = DES_KEY_SIZE,
3599                         .max_keysize = DES_KEY_SIZE,
3600                         .ivsize = 0,
3601                         },
3602          .cipher_info = {
3603                          .alg = CIPHER_ALG_DES,
3604                          .mode = CIPHER_MODE_ECB,
3605                          },
3606          .auth_info = {
3607                        .alg = HASH_ALG_NONE,
3608                        .mode = HASH_MODE_NONE,
3609                        },
3610          },
3611         {
3612          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3613          .alg.skcipher = {
3614                         .base.cra_name = "ofb(des3_ede)",
3615                         .base.cra_driver_name = "ofb-des3-iproc",
3616                         .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3617                         .min_keysize = DES3_EDE_KEY_SIZE,
3618                         .max_keysize = DES3_EDE_KEY_SIZE,
3619                         .ivsize = DES3_EDE_BLOCK_SIZE,
3620                         },
3621          .cipher_info = {
3622                          .alg = CIPHER_ALG_3DES,
3623                          .mode = CIPHER_MODE_OFB,
3624                          },
3625          .auth_info = {
3626                        .alg = HASH_ALG_NONE,
3627                        .mode = HASH_MODE_NONE,
3628                        },
3629          },
3630         {
3631          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3632          .alg.skcipher = {
3633                         .base.cra_name = "cbc(des3_ede)",
3634                         .base.cra_driver_name = "cbc-des3-iproc",
3635                         .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3636                         .min_keysize = DES3_EDE_KEY_SIZE,
3637                         .max_keysize = DES3_EDE_KEY_SIZE,
3638                         .ivsize = DES3_EDE_BLOCK_SIZE,
3639                         },
3640          .cipher_info = {
3641                          .alg = CIPHER_ALG_3DES,
3642                          .mode = CIPHER_MODE_CBC,
3643                          },
3644          .auth_info = {
3645                        .alg = HASH_ALG_NONE,
3646                        .mode = HASH_MODE_NONE,
3647                        },
3648          },
3649         {
3650          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3651          .alg.skcipher = {
3652                         .base.cra_name = "ecb(des3_ede)",
3653                         .base.cra_driver_name = "ecb-des3-iproc",
3654                         .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3655                         .min_keysize = DES3_EDE_KEY_SIZE,
3656                         .max_keysize = DES3_EDE_KEY_SIZE,
3657                         .ivsize = 0,
3658                         },
3659          .cipher_info = {
3660                          .alg = CIPHER_ALG_3DES,
3661                          .mode = CIPHER_MODE_ECB,
3662                          },
3663          .auth_info = {
3664                        .alg = HASH_ALG_NONE,
3665                        .mode = HASH_MODE_NONE,
3666                        },
3667          },
3668         {
3669          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3670          .alg.skcipher = {
3671                         .base.cra_name = "ofb(aes)",
3672                         .base.cra_driver_name = "ofb-aes-iproc",
3673                         .base.cra_blocksize = AES_BLOCK_SIZE,
3674                         .min_keysize = AES_MIN_KEY_SIZE,
3675                         .max_keysize = AES_MAX_KEY_SIZE,
3676                         .ivsize = AES_BLOCK_SIZE,
3677                         },
3678          .cipher_info = {
3679                          .alg = CIPHER_ALG_AES,
3680                          .mode = CIPHER_MODE_OFB,
3681                          },
3682          .auth_info = {
3683                        .alg = HASH_ALG_NONE,
3684                        .mode = HASH_MODE_NONE,
3685                        },
3686          },
3687         {
3688          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3689          .alg.skcipher = {
3690                         .base.cra_name = "cbc(aes)",
3691                         .base.cra_driver_name = "cbc-aes-iproc",
3692                         .base.cra_blocksize = AES_BLOCK_SIZE,
3693                         .min_keysize = AES_MIN_KEY_SIZE,
3694                         .max_keysize = AES_MAX_KEY_SIZE,
3695                         .ivsize = AES_BLOCK_SIZE,
3696                         },
3697          .cipher_info = {
3698                          .alg = CIPHER_ALG_AES,
3699                          .mode = CIPHER_MODE_CBC,
3700                          },
3701          .auth_info = {
3702                        .alg = HASH_ALG_NONE,
3703                        .mode = HASH_MODE_NONE,
3704                        },
3705          },
3706         {
3707          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3708          .alg.skcipher = {
3709                         .base.cra_name = "ecb(aes)",
3710                         .base.cra_driver_name = "ecb-aes-iproc",
3711                         .base.cra_blocksize = AES_BLOCK_SIZE,
3712                         .min_keysize = AES_MIN_KEY_SIZE,
3713                         .max_keysize = AES_MAX_KEY_SIZE,
3714                         .ivsize = 0,
3715                         },
3716          .cipher_info = {
3717                          .alg = CIPHER_ALG_AES,
3718                          .mode = CIPHER_MODE_ECB,
3719                          },
3720          .auth_info = {
3721                        .alg = HASH_ALG_NONE,
3722                        .mode = HASH_MODE_NONE,
3723                        },
3724          },
3725         {
3726          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3727          .alg.skcipher = {
3728                         .base.cra_name = "ctr(aes)",
3729                         .base.cra_driver_name = "ctr-aes-iproc",
3730                         .base.cra_blocksize = AES_BLOCK_SIZE,
3731                         .min_keysize = AES_MIN_KEY_SIZE,
3732                         .max_keysize = AES_MAX_KEY_SIZE,
3733                         .ivsize = AES_BLOCK_SIZE,
3734                         },
3735          .cipher_info = {
3736                          .alg = CIPHER_ALG_AES,
3737                          .mode = CIPHER_MODE_CTR,
3738                          },
3739          .auth_info = {
3740                        .alg = HASH_ALG_NONE,
3741                        .mode = HASH_MODE_NONE,
3742                        },
3743          },
3744 {
3745          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3746          .alg.skcipher = {
3747                         .base.cra_name = "xts(aes)",
3748                         .base.cra_driver_name = "xts-aes-iproc",
3749                         .base.cra_blocksize = AES_BLOCK_SIZE,
3750                         .min_keysize = 2 * AES_MIN_KEY_SIZE,
3751                         .max_keysize = 2 * AES_MAX_KEY_SIZE,
3752                         .ivsize = AES_BLOCK_SIZE,
3753                         },
3754          .cipher_info = {
3755                          .alg = CIPHER_ALG_AES,
3756                          .mode = CIPHER_MODE_XTS,
3757                          },
3758          .auth_info = {
3759                        .alg = HASH_ALG_NONE,
3760                        .mode = HASH_MODE_NONE,
3761                        },
3762          },
3763
3764 /* AHASH algorithms. */
3765         {
3766          .type = CRYPTO_ALG_TYPE_AHASH,
3767          .alg.hash = {
3768                       .halg.digestsize = MD5_DIGEST_SIZE,
3769                       .halg.base = {
3770                                     .cra_name = "md5",
3771                                     .cra_driver_name = "md5-iproc",
3772                                     .cra_blocksize = MD5_BLOCK_WORDS * 4,
3773                                     .cra_flags = CRYPTO_ALG_ASYNC |
3774                                                  CRYPTO_ALG_ALLOCATES_MEMORY,
3775                                 }
3776                       },
3777          .cipher_info = {
3778                          .alg = CIPHER_ALG_NONE,
3779                          .mode = CIPHER_MODE_NONE,
3780                          },
3781          .auth_info = {
3782                        .alg = HASH_ALG_MD5,
3783                        .mode = HASH_MODE_HASH,
3784                        },
3785          },
3786         {
3787          .type = CRYPTO_ALG_TYPE_AHASH,
3788          .alg.hash = {
3789                       .halg.digestsize = MD5_DIGEST_SIZE,
3790                       .halg.base = {
3791                                     .cra_name = "hmac(md5)",
3792                                     .cra_driver_name = "hmac-md5-iproc",
3793                                     .cra_blocksize = MD5_BLOCK_WORDS * 4,
3794                                 }
3795                       },
3796          .cipher_info = {
3797                          .alg = CIPHER_ALG_NONE,
3798                          .mode = CIPHER_MODE_NONE,
3799                          },
3800          .auth_info = {
3801                        .alg = HASH_ALG_MD5,
3802                        .mode = HASH_MODE_HMAC,
3803                        },
3804          },
3805         {.type = CRYPTO_ALG_TYPE_AHASH,
3806          .alg.hash = {
3807                       .halg.digestsize = SHA1_DIGEST_SIZE,
3808                       .halg.base = {
3809                                     .cra_name = "sha1",
3810                                     .cra_driver_name = "sha1-iproc",
3811                                     .cra_blocksize = SHA1_BLOCK_SIZE,
3812                                 }
3813                       },
3814          .cipher_info = {
3815                          .alg = CIPHER_ALG_NONE,
3816                          .mode = CIPHER_MODE_NONE,
3817                          },
3818          .auth_info = {
3819                        .alg = HASH_ALG_SHA1,
3820                        .mode = HASH_MODE_HASH,
3821                        },
3822          },
3823         {.type = CRYPTO_ALG_TYPE_AHASH,
3824          .alg.hash = {
3825                       .halg.digestsize = SHA1_DIGEST_SIZE,
3826                       .halg.base = {
3827                                     .cra_name = "hmac(sha1)",
3828                                     .cra_driver_name = "hmac-sha1-iproc",
3829                                     .cra_blocksize = SHA1_BLOCK_SIZE,
3830                                 }
3831                       },
3832          .cipher_info = {
3833                          .alg = CIPHER_ALG_NONE,
3834                          .mode = CIPHER_MODE_NONE,
3835                          },
3836          .auth_info = {
3837                        .alg = HASH_ALG_SHA1,
3838                        .mode = HASH_MODE_HMAC,
3839                        },
3840          },
3841         {.type = CRYPTO_ALG_TYPE_AHASH,
3842          .alg.hash = {
3843                         .halg.digestsize = SHA224_DIGEST_SIZE,
3844                         .halg.base = {
3845                                     .cra_name = "sha224",
3846                                     .cra_driver_name = "sha224-iproc",
3847                                     .cra_blocksize = SHA224_BLOCK_SIZE,
3848                         }
3849                       },
3850          .cipher_info = {
3851                          .alg = CIPHER_ALG_NONE,
3852                          .mode = CIPHER_MODE_NONE,
3853                          },
3854          .auth_info = {
3855                        .alg = HASH_ALG_SHA224,
3856                        .mode = HASH_MODE_HASH,
3857                        },
3858          },
3859         {.type = CRYPTO_ALG_TYPE_AHASH,
3860          .alg.hash = {
3861                       .halg.digestsize = SHA224_DIGEST_SIZE,
3862                       .halg.base = {
3863                                     .cra_name = "hmac(sha224)",
3864                                     .cra_driver_name = "hmac-sha224-iproc",
3865                                     .cra_blocksize = SHA224_BLOCK_SIZE,
3866                                 }
3867                       },
3868          .cipher_info = {
3869                          .alg = CIPHER_ALG_NONE,
3870                          .mode = CIPHER_MODE_NONE,
3871                          },
3872          .auth_info = {
3873                        .alg = HASH_ALG_SHA224,
3874                        .mode = HASH_MODE_HMAC,
3875                        },
3876          },
3877         {.type = CRYPTO_ALG_TYPE_AHASH,
3878          .alg.hash = {
3879                       .halg.digestsize = SHA256_DIGEST_SIZE,
3880                       .halg.base = {
3881                                     .cra_name = "sha256",
3882                                     .cra_driver_name = "sha256-iproc",
3883                                     .cra_blocksize = SHA256_BLOCK_SIZE,
3884                                 }
3885                       },
3886          .cipher_info = {
3887                          .alg = CIPHER_ALG_NONE,
3888                          .mode = CIPHER_MODE_NONE,
3889                          },
3890          .auth_info = {
3891                        .alg = HASH_ALG_SHA256,
3892                        .mode = HASH_MODE_HASH,
3893                        },
3894          },
3895         {.type = CRYPTO_ALG_TYPE_AHASH,
3896          .alg.hash = {
3897                       .halg.digestsize = SHA256_DIGEST_SIZE,
3898                       .halg.base = {
3899                                     .cra_name = "hmac(sha256)",
3900                                     .cra_driver_name = "hmac-sha256-iproc",
3901                                     .cra_blocksize = SHA256_BLOCK_SIZE,
3902                                 }
3903                       },
3904          .cipher_info = {
3905                          .alg = CIPHER_ALG_NONE,
3906                          .mode = CIPHER_MODE_NONE,
3907                          },
3908          .auth_info = {
3909                        .alg = HASH_ALG_SHA256,
3910                        .mode = HASH_MODE_HMAC,
3911                        },
3912          },
3913         {
3914         .type = CRYPTO_ALG_TYPE_AHASH,
3915          .alg.hash = {
3916                       .halg.digestsize = SHA384_DIGEST_SIZE,
3917                       .halg.base = {
3918                                     .cra_name = "sha384",
3919                                     .cra_driver_name = "sha384-iproc",
3920                                     .cra_blocksize = SHA384_BLOCK_SIZE,
3921                                 }
3922                       },
3923          .cipher_info = {
3924                          .alg = CIPHER_ALG_NONE,
3925                          .mode = CIPHER_MODE_NONE,
3926                          },
3927          .auth_info = {
3928                        .alg = HASH_ALG_SHA384,
3929                        .mode = HASH_MODE_HASH,
3930                        },
3931          },
3932         {
3933          .type = CRYPTO_ALG_TYPE_AHASH,
3934          .alg.hash = {
3935                       .halg.digestsize = SHA384_DIGEST_SIZE,
3936                       .halg.base = {
3937                                     .cra_name = "hmac(sha384)",
3938                                     .cra_driver_name = "hmac-sha384-iproc",
3939                                     .cra_blocksize = SHA384_BLOCK_SIZE,
3940                                 }
3941                       },
3942          .cipher_info = {
3943                          .alg = CIPHER_ALG_NONE,
3944                          .mode = CIPHER_MODE_NONE,
3945                          },
3946          .auth_info = {
3947                        .alg = HASH_ALG_SHA384,
3948                        .mode = HASH_MODE_HMAC,
3949                        },
3950          },
3951         {
3952          .type = CRYPTO_ALG_TYPE_AHASH,
3953          .alg.hash = {
3954                       .halg.digestsize = SHA512_DIGEST_SIZE,
3955                       .halg.base = {
3956                                     .cra_name = "sha512",
3957                                     .cra_driver_name = "sha512-iproc",
3958                                     .cra_blocksize = SHA512_BLOCK_SIZE,
3959                                 }
3960                       },
3961          .cipher_info = {
3962                          .alg = CIPHER_ALG_NONE,
3963                          .mode = CIPHER_MODE_NONE,
3964                          },
3965          .auth_info = {
3966                        .alg = HASH_ALG_SHA512,
3967                        .mode = HASH_MODE_HASH,
3968                        },
3969          },
3970         {
3971          .type = CRYPTO_ALG_TYPE_AHASH,
3972          .alg.hash = {
3973                       .halg.digestsize = SHA512_DIGEST_SIZE,
3974                       .halg.base = {
3975                                     .cra_name = "hmac(sha512)",
3976                                     .cra_driver_name = "hmac-sha512-iproc",
3977                                     .cra_blocksize = SHA512_BLOCK_SIZE,
3978                                 }
3979                       },
3980          .cipher_info = {
3981                          .alg = CIPHER_ALG_NONE,
3982                          .mode = CIPHER_MODE_NONE,
3983                          },
3984          .auth_info = {
3985                        .alg = HASH_ALG_SHA512,
3986                        .mode = HASH_MODE_HMAC,
3987                        },
3988          },
3989         {
3990          .type = CRYPTO_ALG_TYPE_AHASH,
3991          .alg.hash = {
3992                       .halg.digestsize = SHA3_224_DIGEST_SIZE,
3993                       .halg.base = {
3994                                     .cra_name = "sha3-224",
3995                                     .cra_driver_name = "sha3-224-iproc",
3996                                     .cra_blocksize = SHA3_224_BLOCK_SIZE,
3997                                 }
3998                       },
3999          .cipher_info = {
4000                          .alg = CIPHER_ALG_NONE,
4001                          .mode = CIPHER_MODE_NONE,
4002                          },
4003          .auth_info = {
4004                        .alg = HASH_ALG_SHA3_224,
4005                        .mode = HASH_MODE_HASH,
4006                        },
4007          },
4008         {
4009          .type = CRYPTO_ALG_TYPE_AHASH,
4010          .alg.hash = {
4011                       .halg.digestsize = SHA3_224_DIGEST_SIZE,
4012                       .halg.base = {
4013                                     .cra_name = "hmac(sha3-224)",
4014                                     .cra_driver_name = "hmac-sha3-224-iproc",
4015                                     .cra_blocksize = SHA3_224_BLOCK_SIZE,
4016                                 }
4017                       },
4018          .cipher_info = {
4019                          .alg = CIPHER_ALG_NONE,
4020                          .mode = CIPHER_MODE_NONE,
4021                          },
4022          .auth_info = {
4023                        .alg = HASH_ALG_SHA3_224,
4024                        .mode = HASH_MODE_HMAC
4025                        },
4026          },
4027         {
4028          .type = CRYPTO_ALG_TYPE_AHASH,
4029          .alg.hash = {
4030                       .halg.digestsize = SHA3_256_DIGEST_SIZE,
4031                       .halg.base = {
4032                                     .cra_name = "sha3-256",
4033                                     .cra_driver_name = "sha3-256-iproc",
4034                                     .cra_blocksize = SHA3_256_BLOCK_SIZE,
4035                                 }
4036                       },
4037          .cipher_info = {
4038                          .alg = CIPHER_ALG_NONE,
4039                          .mode = CIPHER_MODE_NONE,
4040                          },
4041          .auth_info = {
4042                        .alg = HASH_ALG_SHA3_256,
4043                        .mode = HASH_MODE_HASH,
4044                        },
4045          },
4046         {
4047          .type = CRYPTO_ALG_TYPE_AHASH,
4048          .alg.hash = {
4049                       .halg.digestsize = SHA3_256_DIGEST_SIZE,
4050                       .halg.base = {
4051                                     .cra_name = "hmac(sha3-256)",
4052                                     .cra_driver_name = "hmac-sha3-256-iproc",
4053                                     .cra_blocksize = SHA3_256_BLOCK_SIZE,
4054                                 }
4055                       },
4056          .cipher_info = {
4057                          .alg = CIPHER_ALG_NONE,
4058                          .mode = CIPHER_MODE_NONE,
4059                          },
4060          .auth_info = {
4061                        .alg = HASH_ALG_SHA3_256,
4062                        .mode = HASH_MODE_HMAC,
4063                        },
4064          },
4065         {
4066          .type = CRYPTO_ALG_TYPE_AHASH,
4067          .alg.hash = {
4068                       .halg.digestsize = SHA3_384_DIGEST_SIZE,
4069                       .halg.base = {
4070                                     .cra_name = "sha3-384",
4071                                     .cra_driver_name = "sha3-384-iproc",
4072                                     .cra_blocksize = SHA3_224_BLOCK_SIZE,
4073                                 }
4074                       },
4075          .cipher_info = {
4076                          .alg = CIPHER_ALG_NONE,
4077                          .mode = CIPHER_MODE_NONE,
4078                          },
4079          .auth_info = {
4080                        .alg = HASH_ALG_SHA3_384,
4081                        .mode = HASH_MODE_HASH,
4082                        },
4083          },
4084         {
4085          .type = CRYPTO_ALG_TYPE_AHASH,
4086          .alg.hash = {
4087                       .halg.digestsize = SHA3_384_DIGEST_SIZE,
4088                       .halg.base = {
4089                                     .cra_name = "hmac(sha3-384)",
4090                                     .cra_driver_name = "hmac-sha3-384-iproc",
4091                                     .cra_blocksize = SHA3_384_BLOCK_SIZE,
4092                                 }
4093                       },
4094          .cipher_info = {
4095                          .alg = CIPHER_ALG_NONE,
4096                          .mode = CIPHER_MODE_NONE,
4097                          },
4098          .auth_info = {
4099                        .alg = HASH_ALG_SHA3_384,
4100                        .mode = HASH_MODE_HMAC,
4101                        },
4102          },
4103         {
4104          .type = CRYPTO_ALG_TYPE_AHASH,
4105          .alg.hash = {
4106                       .halg.digestsize = SHA3_512_DIGEST_SIZE,
4107                       .halg.base = {
4108                                     .cra_name = "sha3-512",
4109                                     .cra_driver_name = "sha3-512-iproc",
4110                                     .cra_blocksize = SHA3_512_BLOCK_SIZE,
4111                                 }
4112                       },
4113          .cipher_info = {
4114                          .alg = CIPHER_ALG_NONE,
4115                          .mode = CIPHER_MODE_NONE,
4116                          },
4117          .auth_info = {
4118                        .alg = HASH_ALG_SHA3_512,
4119                        .mode = HASH_MODE_HASH,
4120                        },
4121          },
4122         {
4123          .type = CRYPTO_ALG_TYPE_AHASH,
4124          .alg.hash = {
4125                       .halg.digestsize = SHA3_512_DIGEST_SIZE,
4126                       .halg.base = {
4127                                     .cra_name = "hmac(sha3-512)",
4128                                     .cra_driver_name = "hmac-sha3-512-iproc",
4129                                     .cra_blocksize = SHA3_512_BLOCK_SIZE,
4130                                 }
4131                       },
4132          .cipher_info = {
4133                          .alg = CIPHER_ALG_NONE,
4134                          .mode = CIPHER_MODE_NONE,
4135                          },
4136          .auth_info = {
4137                        .alg = HASH_ALG_SHA3_512,
4138                        .mode = HASH_MODE_HMAC,
4139                        },
4140          },
4141         {
4142          .type = CRYPTO_ALG_TYPE_AHASH,
4143          .alg.hash = {
4144                       .halg.digestsize = AES_BLOCK_SIZE,
4145                       .halg.base = {
4146                                     .cra_name = "xcbc(aes)",
4147                                     .cra_driver_name = "xcbc-aes-iproc",
4148                                     .cra_blocksize = AES_BLOCK_SIZE,
4149                                 }
4150                       },
4151          .cipher_info = {
4152                          .alg = CIPHER_ALG_NONE,
4153                          .mode = CIPHER_MODE_NONE,
4154                          },
4155          .auth_info = {
4156                        .alg = HASH_ALG_AES,
4157                        .mode = HASH_MODE_XCBC,
4158                        },
4159          },
4160         {
4161          .type = CRYPTO_ALG_TYPE_AHASH,
4162          .alg.hash = {
4163                       .halg.digestsize = AES_BLOCK_SIZE,
4164                       .halg.base = {
4165                                     .cra_name = "cmac(aes)",
4166                                     .cra_driver_name = "cmac-aes-iproc",
4167                                     .cra_blocksize = AES_BLOCK_SIZE,
4168                                 }
4169                       },
4170          .cipher_info = {
4171                          .alg = CIPHER_ALG_NONE,
4172                          .mode = CIPHER_MODE_NONE,
4173                          },
4174          .auth_info = {
4175                        .alg = HASH_ALG_AES,
4176                        .mode = HASH_MODE_CMAC,
4177                        },
4178          },
4179 };
4180
4181 static int generic_cra_init(struct crypto_tfm *tfm,
4182                             struct iproc_alg_s *cipher_alg)
4183 {
4184         struct spu_hw *spu = &iproc_priv.spu;
4185         struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
4186         unsigned int blocksize = crypto_tfm_alg_blocksize(tfm);
4187
4188         flow_log("%s()\n", __func__);
4189
4190         ctx->alg = cipher_alg;
4191         ctx->cipher = cipher_alg->cipher_info;
4192         ctx->auth = cipher_alg->auth_info;
4193         ctx->auth_first = cipher_alg->auth_first;
4194         ctx->max_payload = spu->spu_ctx_max_payload(ctx->cipher.alg,
4195                                                     ctx->cipher.mode,
4196                                                     blocksize);
4197         ctx->fallback_cipher = NULL;
4198
4199         ctx->enckeylen = 0;
4200         ctx->authkeylen = 0;
4201
4202         atomic_inc(&iproc_priv.stream_count);
4203         atomic_inc(&iproc_priv.session_count);
4204
4205         return 0;
4206 }
4207
4208 static int skcipher_init_tfm(struct crypto_skcipher *skcipher)
4209 {
4210         struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher);
4211         struct skcipher_alg *alg = crypto_skcipher_alg(skcipher);
4212         struct iproc_alg_s *cipher_alg;
4213
4214         flow_log("%s()\n", __func__);
4215
4216         crypto_skcipher_set_reqsize(skcipher, sizeof(struct iproc_reqctx_s));
4217
4218         cipher_alg = container_of(alg, struct iproc_alg_s, alg.skcipher);
4219         return generic_cra_init(tfm, cipher_alg);
4220 }
4221
4222 static int ahash_cra_init(struct crypto_tfm *tfm)
4223 {
4224         int err;
4225         struct crypto_alg *alg = tfm->__crt_alg;
4226         struct iproc_alg_s *cipher_alg;
4227
4228         cipher_alg = container_of(__crypto_ahash_alg(alg), struct iproc_alg_s,
4229                                   alg.hash);
4230
4231         err = generic_cra_init(tfm, cipher_alg);
4232         flow_log("%s()\n", __func__);
4233
4234         /*
4235          * export state size has to be < 512 bytes. So don't include msg bufs
4236          * in state size.
4237          */
4238         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
4239                                  sizeof(struct iproc_reqctx_s));
4240
4241         return err;
4242 }
4243
4244 static int aead_cra_init(struct crypto_aead *aead)
4245 {
4246         struct crypto_tfm *tfm = crypto_aead_tfm(aead);
4247         struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
4248         struct crypto_alg *alg = tfm->__crt_alg;
4249         struct aead_alg *aalg = container_of(alg, struct aead_alg, base);
4250         struct iproc_alg_s *cipher_alg = container_of(aalg, struct iproc_alg_s,
4251                                                       alg.aead);
4252
4253         int err = generic_cra_init(tfm, cipher_alg);
4254
4255         flow_log("%s()\n", __func__);
4256
4257         crypto_aead_set_reqsize(aead, sizeof(struct iproc_reqctx_s));
4258         ctx->is_esp = false;
4259         ctx->salt_len = 0;
4260         ctx->salt_offset = 0;
4261
4262         /* random first IV */
4263         get_random_bytes(ctx->iv, MAX_IV_SIZE);
4264         flow_dump("  iv: ", ctx->iv, MAX_IV_SIZE);
4265
4266         if (!err) {
4267                 if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
4268                         flow_log("%s() creating fallback cipher\n", __func__);
4269
4270                         ctx->fallback_cipher =
4271                             crypto_alloc_aead(alg->cra_name, 0,
4272                                               CRYPTO_ALG_ASYNC |
4273                                               CRYPTO_ALG_NEED_FALLBACK);
4274                         if (IS_ERR(ctx->fallback_cipher)) {
4275                                 pr_err("%s() Error: failed to allocate fallback for %s\n",
4276                                        __func__, alg->cra_name);
4277                                 return PTR_ERR(ctx->fallback_cipher);
4278                         }
4279                 }
4280         }
4281
4282         return err;
4283 }
4284
4285 static void generic_cra_exit(struct crypto_tfm *tfm)
4286 {
4287         atomic_dec(&iproc_priv.session_count);
4288 }
4289
4290 static void skcipher_exit_tfm(struct crypto_skcipher *tfm)
4291 {
4292         generic_cra_exit(crypto_skcipher_tfm(tfm));
4293 }
4294
4295 static void aead_cra_exit(struct crypto_aead *aead)
4296 {
4297         struct crypto_tfm *tfm = crypto_aead_tfm(aead);
4298         struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
4299
4300         generic_cra_exit(tfm);
4301
4302         if (ctx->fallback_cipher) {
4303                 crypto_free_aead(ctx->fallback_cipher);
4304                 ctx->fallback_cipher = NULL;
4305         }
4306 }
4307
4308 /**
4309  * spu_functions_register() - Specify hardware-specific SPU functions based on
4310  * SPU type read from device tree.
4311  * @dev:        device structure
4312  * @spu_type:   SPU hardware generation
4313  * @spu_subtype: SPU hardware version
4314  */
4315 static void spu_functions_register(struct device *dev,
4316                                    enum spu_spu_type spu_type,
4317                                    enum spu_spu_subtype spu_subtype)
4318 {
4319         struct spu_hw *spu = &iproc_priv.spu;
4320
4321         if (spu_type == SPU_TYPE_SPUM) {
4322                 dev_dbg(dev, "Registering SPUM functions");
4323                 spu->spu_dump_msg_hdr = spum_dump_msg_hdr;
4324                 spu->spu_payload_length = spum_payload_length;
4325                 spu->spu_response_hdr_len = spum_response_hdr_len;
4326                 spu->spu_hash_pad_len = spum_hash_pad_len;
4327                 spu->spu_gcm_ccm_pad_len = spum_gcm_ccm_pad_len;
4328                 spu->spu_assoc_resp_len = spum_assoc_resp_len;
4329                 spu->spu_aead_ivlen = spum_aead_ivlen;
4330                 spu->spu_hash_type = spum_hash_type;
4331                 spu->spu_digest_size = spum_digest_size;
4332                 spu->spu_create_request = spum_create_request;
4333                 spu->spu_cipher_req_init = spum_cipher_req_init;
4334                 spu->spu_cipher_req_finish = spum_cipher_req_finish;
4335                 spu->spu_request_pad = spum_request_pad;
4336                 spu->spu_tx_status_len = spum_tx_status_len;
4337                 spu->spu_rx_status_len = spum_rx_status_len;
4338                 spu->spu_status_process = spum_status_process;
4339                 spu->spu_xts_tweak_in_payload = spum_xts_tweak_in_payload;
4340                 spu->spu_ccm_update_iv = spum_ccm_update_iv;
4341                 spu->spu_wordalign_padlen = spum_wordalign_padlen;
4342                 if (spu_subtype == SPU_SUBTYPE_SPUM_NS2)
4343                         spu->spu_ctx_max_payload = spum_ns2_ctx_max_payload;
4344                 else
4345                         spu->spu_ctx_max_payload = spum_nsp_ctx_max_payload;
4346         } else {
4347                 dev_dbg(dev, "Registering SPU2 functions");
4348                 spu->spu_dump_msg_hdr = spu2_dump_msg_hdr;
4349                 spu->spu_ctx_max_payload = spu2_ctx_max_payload;
4350                 spu->spu_payload_length = spu2_payload_length;
4351                 spu->spu_response_hdr_len = spu2_response_hdr_len;
4352                 spu->spu_hash_pad_len = spu2_hash_pad_len;
4353                 spu->spu_gcm_ccm_pad_len = spu2_gcm_ccm_pad_len;
4354                 spu->spu_assoc_resp_len = spu2_assoc_resp_len;
4355                 spu->spu_aead_ivlen = spu2_aead_ivlen;
4356                 spu->spu_hash_type = spu2_hash_type;
4357                 spu->spu_digest_size = spu2_digest_size;
4358                 spu->spu_create_request = spu2_create_request;
4359                 spu->spu_cipher_req_init = spu2_cipher_req_init;
4360                 spu->spu_cipher_req_finish = spu2_cipher_req_finish;
4361                 spu->spu_request_pad = spu2_request_pad;
4362                 spu->spu_tx_status_len = spu2_tx_status_len;
4363                 spu->spu_rx_status_len = spu2_rx_status_len;
4364                 spu->spu_status_process = spu2_status_process;
4365                 spu->spu_xts_tweak_in_payload = spu2_xts_tweak_in_payload;
4366                 spu->spu_ccm_update_iv = spu2_ccm_update_iv;
4367                 spu->spu_wordalign_padlen = spu2_wordalign_padlen;
4368         }
4369 }
4370
4371 /**
4372  * spu_mb_init() - Initialize mailbox client. Request ownership of a mailbox
4373  * channel for the SPU being probed.
4374  * @dev:  SPU driver device structure
4375  *
4376  * Return: 0 if successful
4377  *         < 0 otherwise
4378  */
4379 static int spu_mb_init(struct device *dev)
4380 {
4381         struct mbox_client *mcl = &iproc_priv.mcl;
4382         int err, i;
4383
4384         iproc_priv.mbox = devm_kcalloc(dev, iproc_priv.spu.num_chan,
4385                                   sizeof(struct mbox_chan *), GFP_KERNEL);
4386         if (!iproc_priv.mbox)
4387                 return -ENOMEM;
4388
4389         mcl->dev = dev;
4390         mcl->tx_block = false;
4391         mcl->tx_tout = 0;
4392         mcl->knows_txdone = true;
4393         mcl->rx_callback = spu_rx_callback;
4394         mcl->tx_done = NULL;
4395
4396         for (i = 0; i < iproc_priv.spu.num_chan; i++) {
4397                 iproc_priv.mbox[i] = mbox_request_channel(mcl, i);
4398                 if (IS_ERR(iproc_priv.mbox[i])) {
4399                         err = PTR_ERR(iproc_priv.mbox[i]);
4400                         dev_err(dev,
4401                                 "Mbox channel %d request failed with err %d",
4402                                 i, err);
4403                         iproc_priv.mbox[i] = NULL;
4404                         goto free_channels;
4405                 }
4406         }
4407
4408         return 0;
4409 free_channels:
4410         for (i = 0; i < iproc_priv.spu.num_chan; i++) {
4411                 if (iproc_priv.mbox[i])
4412                         mbox_free_channel(iproc_priv.mbox[i]);
4413         }
4414
4415         return err;
4416 }
4417
4418 static void spu_mb_release(struct platform_device *pdev)
4419 {
4420         int i;
4421
4422         for (i = 0; i < iproc_priv.spu.num_chan; i++)
4423                 mbox_free_channel(iproc_priv.mbox[i]);
4424 }
4425
4426 static void spu_counters_init(void)
4427 {
4428         int i;
4429         int j;
4430
4431         atomic_set(&iproc_priv.session_count, 0);
4432         atomic_set(&iproc_priv.stream_count, 0);
4433         atomic_set(&iproc_priv.next_chan, (int)iproc_priv.spu.num_chan);
4434         atomic64_set(&iproc_priv.bytes_in, 0);
4435         atomic64_set(&iproc_priv.bytes_out, 0);
4436         for (i = 0; i < SPU_OP_NUM; i++) {
4437                 atomic_set(&iproc_priv.op_counts[i], 0);
4438                 atomic_set(&iproc_priv.setkey_cnt[i], 0);
4439         }
4440         for (i = 0; i < CIPHER_ALG_LAST; i++)
4441                 for (j = 0; j < CIPHER_MODE_LAST; j++)
4442                         atomic_set(&iproc_priv.cipher_cnt[i][j], 0);
4443
4444         for (i = 0; i < HASH_ALG_LAST; i++) {
4445                 atomic_set(&iproc_priv.hash_cnt[i], 0);
4446                 atomic_set(&iproc_priv.hmac_cnt[i], 0);
4447         }
4448         for (i = 0; i < AEAD_TYPE_LAST; i++)
4449                 atomic_set(&iproc_priv.aead_cnt[i], 0);
4450
4451         atomic_set(&iproc_priv.mb_no_spc, 0);
4452         atomic_set(&iproc_priv.mb_send_fail, 0);
4453         atomic_set(&iproc_priv.bad_icv, 0);
4454 }
4455
4456 static int spu_register_skcipher(struct iproc_alg_s *driver_alg)
4457 {
4458         struct skcipher_alg *crypto = &driver_alg->alg.skcipher;
4459         int err;
4460
4461         crypto->base.cra_module = THIS_MODULE;
4462         crypto->base.cra_priority = cipher_pri;
4463         crypto->base.cra_alignmask = 0;
4464         crypto->base.cra_ctxsize = sizeof(struct iproc_ctx_s);
4465         crypto->base.cra_flags = CRYPTO_ALG_ASYNC |
4466                                  CRYPTO_ALG_ALLOCATES_MEMORY |
4467                                  CRYPTO_ALG_KERN_DRIVER_ONLY;
4468
4469         crypto->init = skcipher_init_tfm;
4470         crypto->exit = skcipher_exit_tfm;
4471         crypto->setkey = skcipher_setkey;
4472         crypto->encrypt = skcipher_encrypt;
4473         crypto->decrypt = skcipher_decrypt;
4474
4475         err = crypto_register_skcipher(crypto);
4476         /* Mark alg as having been registered, if successful */
4477         if (err == 0)
4478                 driver_alg->registered = true;
4479         pr_debug("  registered skcipher %s\n", crypto->base.cra_driver_name);
4480         return err;
4481 }
4482
4483 static int spu_register_ahash(struct iproc_alg_s *driver_alg)
4484 {
4485         struct spu_hw *spu = &iproc_priv.spu;
4486         struct ahash_alg *hash = &driver_alg->alg.hash;
4487         int err;
4488
4489         /* AES-XCBC is the only AES hash type currently supported on SPU-M */
4490         if ((driver_alg->auth_info.alg == HASH_ALG_AES) &&
4491             (driver_alg->auth_info.mode != HASH_MODE_XCBC) &&
4492             (spu->spu_type == SPU_TYPE_SPUM))
4493                 return 0;
4494
4495         /* SHA3 algorithm variants are not registered for SPU-M or SPU2. */
4496         if ((driver_alg->auth_info.alg >= HASH_ALG_SHA3_224) &&
4497             (spu->spu_subtype != SPU_SUBTYPE_SPU2_V2))
4498                 return 0;
4499
4500         hash->halg.base.cra_module = THIS_MODULE;
4501         hash->halg.base.cra_priority = hash_pri;
4502         hash->halg.base.cra_alignmask = 0;
4503         hash->halg.base.cra_ctxsize = sizeof(struct iproc_ctx_s);
4504         hash->halg.base.cra_init = ahash_cra_init;
4505         hash->halg.base.cra_exit = generic_cra_exit;
4506         hash->halg.base.cra_flags = CRYPTO_ALG_ASYNC |
4507                                     CRYPTO_ALG_ALLOCATES_MEMORY;
4508         hash->halg.statesize = sizeof(struct spu_hash_export_s);
4509
4510         if (driver_alg->auth_info.mode != HASH_MODE_HMAC) {
4511                 hash->init = ahash_init;
4512                 hash->update = ahash_update;
4513                 hash->final = ahash_final;
4514                 hash->finup = ahash_finup;
4515                 hash->digest = ahash_digest;
4516                 if ((driver_alg->auth_info.alg == HASH_ALG_AES) &&
4517                     ((driver_alg->auth_info.mode == HASH_MODE_XCBC) ||
4518                     (driver_alg->auth_info.mode == HASH_MODE_CMAC))) {
4519                         hash->setkey = ahash_setkey;
4520                 }
4521         } else {
4522                 hash->setkey = ahash_hmac_setkey;
4523                 hash->init = ahash_hmac_init;
4524                 hash->update = ahash_hmac_update;
4525                 hash->final = ahash_hmac_final;
4526                 hash->finup = ahash_hmac_finup;
4527                 hash->digest = ahash_hmac_digest;
4528         }
4529         hash->export = ahash_export;
4530         hash->import = ahash_import;
4531
4532         err = crypto_register_ahash(hash);
4533         /* Mark alg as having been registered, if successful */
4534         if (err == 0)
4535                 driver_alg->registered = true;
4536         pr_debug("  registered ahash %s\n",
4537                  hash->halg.base.cra_driver_name);
4538         return err;
4539 }
4540
4541 static int spu_register_aead(struct iproc_alg_s *driver_alg)
4542 {
4543         struct aead_alg *aead = &driver_alg->alg.aead;
4544         int err;
4545
4546         aead->base.cra_module = THIS_MODULE;
4547         aead->base.cra_priority = aead_pri;
4548         aead->base.cra_alignmask = 0;
4549         aead->base.cra_ctxsize = sizeof(struct iproc_ctx_s);
4550
4551         aead->base.cra_flags |= CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY;
4552         /* setkey set in alg initialization */
4553         aead->setauthsize = aead_setauthsize;
4554         aead->encrypt = aead_encrypt;
4555         aead->decrypt = aead_decrypt;
4556         aead->init = aead_cra_init;
4557         aead->exit = aead_cra_exit;
4558
4559         err = crypto_register_aead(aead);
4560         /* Mark alg as having been registered, if successful */
4561         if (err == 0)
4562                 driver_alg->registered = true;
4563         pr_debug("  registered aead %s\n", aead->base.cra_driver_name);
4564         return err;
4565 }
4566
4567 /* register crypto algorithms the device supports */
4568 static int spu_algs_register(struct device *dev)
4569 {
4570         int i, j;
4571         int err;
4572
4573         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4574                 switch (driver_algs[i].type) {
4575                 case CRYPTO_ALG_TYPE_SKCIPHER:
4576                         err = spu_register_skcipher(&driver_algs[i]);
4577                         break;
4578                 case CRYPTO_ALG_TYPE_AHASH:
4579                         err = spu_register_ahash(&driver_algs[i]);
4580                         break;
4581                 case CRYPTO_ALG_TYPE_AEAD:
4582                         err = spu_register_aead(&driver_algs[i]);
4583                         break;
4584                 default:
4585                         dev_err(dev,
4586                                 "iproc-crypto: unknown alg type: %d",
4587                                 driver_algs[i].type);
4588                         err = -EINVAL;
4589                 }
4590
4591                 if (err) {
4592                         dev_err(dev, "alg registration failed with error %d\n",
4593                                 err);
4594                         goto err_algs;
4595                 }
4596         }
4597
4598         return 0;
4599
4600 err_algs:
4601         for (j = 0; j < i; j++) {
4602                 /* Skip any algorithm not registered */
4603                 if (!driver_algs[j].registered)
4604                         continue;
4605                 switch (driver_algs[j].type) {
4606                 case CRYPTO_ALG_TYPE_SKCIPHER:
4607                         crypto_unregister_skcipher(&driver_algs[j].alg.skcipher);
4608                         driver_algs[j].registered = false;
4609                         break;
4610                 case CRYPTO_ALG_TYPE_AHASH:
4611                         crypto_unregister_ahash(&driver_algs[j].alg.hash);
4612                         driver_algs[j].registered = false;
4613                         break;
4614                 case CRYPTO_ALG_TYPE_AEAD:
4615                         crypto_unregister_aead(&driver_algs[j].alg.aead);
4616                         driver_algs[j].registered = false;
4617                         break;
4618                 }
4619         }
4620         return err;
4621 }
4622
4623 /* ==================== Kernel Platform API ==================== */
4624
4625 static struct spu_type_subtype spum_ns2_types = {
4626         SPU_TYPE_SPUM, SPU_SUBTYPE_SPUM_NS2
4627 };
4628
4629 static struct spu_type_subtype spum_nsp_types = {
4630         SPU_TYPE_SPUM, SPU_SUBTYPE_SPUM_NSP
4631 };
4632
4633 static struct spu_type_subtype spu2_types = {
4634         SPU_TYPE_SPU2, SPU_SUBTYPE_SPU2_V1
4635 };
4636
4637 static struct spu_type_subtype spu2_v2_types = {
4638         SPU_TYPE_SPU2, SPU_SUBTYPE_SPU2_V2
4639 };
4640
4641 static const struct of_device_id bcm_spu_dt_ids[] = {
4642         {
4643                 .compatible = "brcm,spum-crypto",
4644                 .data = &spum_ns2_types,
4645         },
4646         {
4647                 .compatible = "brcm,spum-nsp-crypto",
4648                 .data = &spum_nsp_types,
4649         },
4650         {
4651                 .compatible = "brcm,spu2-crypto",
4652                 .data = &spu2_types,
4653         },
4654         {
4655                 .compatible = "brcm,spu2-v2-crypto",
4656                 .data = &spu2_v2_types,
4657         },
4658         { /* sentinel */ }
4659 };
4660
4661 MODULE_DEVICE_TABLE(of, bcm_spu_dt_ids);
4662
4663 static int spu_dt_read(struct platform_device *pdev)
4664 {
4665         struct device *dev = &pdev->dev;
4666         struct spu_hw *spu = &iproc_priv.spu;
4667         struct resource *spu_ctrl_regs;
4668         const struct spu_type_subtype *matched_spu_type;
4669         struct device_node *dn = pdev->dev.of_node;
4670         int err, i;
4671
4672         /* Count number of mailbox channels */
4673         spu->num_chan = of_count_phandle_with_args(dn, "mboxes", "#mbox-cells");
4674
4675         matched_spu_type = of_device_get_match_data(dev);
4676         if (!matched_spu_type) {
4677                 dev_err(dev, "Failed to match device\n");
4678                 return -ENODEV;
4679         }
4680
4681         spu->spu_type = matched_spu_type->type;
4682         spu->spu_subtype = matched_spu_type->subtype;
4683
4684         for (i = 0; (i < MAX_SPUS) && ((spu_ctrl_regs =
4685                 platform_get_resource(pdev, IORESOURCE_MEM, i)) != NULL); i++) {
4686
4687                 spu->reg_vbase[i] = devm_ioremap_resource(dev, spu_ctrl_regs);
4688                 if (IS_ERR(spu->reg_vbase[i])) {
4689                         err = PTR_ERR(spu->reg_vbase[i]);
4690                         dev_err(dev, "Failed to map registers: %d\n",
4691                                 err);
4692                         spu->reg_vbase[i] = NULL;
4693                         return err;
4694                 }
4695         }
4696         spu->num_spu = i;
4697         dev_dbg(dev, "Device has %d SPUs", spu->num_spu);
4698
4699         return 0;
4700 }
4701
4702 static int bcm_spu_probe(struct platform_device *pdev)
4703 {
4704         struct device *dev = &pdev->dev;
4705         struct spu_hw *spu = &iproc_priv.spu;
4706         int err;
4707
4708         iproc_priv.pdev  = pdev;
4709         platform_set_drvdata(iproc_priv.pdev,
4710                              &iproc_priv);
4711
4712         err = spu_dt_read(pdev);
4713         if (err < 0)
4714                 goto failure;
4715
4716         err = spu_mb_init(dev);
4717         if (err < 0)
4718                 goto failure;
4719
4720         if (spu->spu_type == SPU_TYPE_SPUM)
4721                 iproc_priv.bcm_hdr_len = 8;
4722         else if (spu->spu_type == SPU_TYPE_SPU2)
4723                 iproc_priv.bcm_hdr_len = 0;
4724
4725         spu_functions_register(dev, spu->spu_type, spu->spu_subtype);
4726
4727         spu_counters_init();
4728
4729         spu_setup_debugfs();
4730
4731         err = spu_algs_register(dev);
4732         if (err < 0)
4733                 goto fail_reg;
4734
4735         return 0;
4736
4737 fail_reg:
4738         spu_free_debugfs();
4739 failure:
4740         spu_mb_release(pdev);
4741         dev_err(dev, "%s failed with error %d.\n", __func__, err);
4742
4743         return err;
4744 }
4745
4746 static int bcm_spu_remove(struct platform_device *pdev)
4747 {
4748         int i;
4749         struct device *dev = &pdev->dev;
4750         char *cdn;
4751
4752         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4753                 /*
4754                  * Not all algorithms were registered, depending on whether
4755                  * hardware is SPU or SPU2.  So here we make sure to skip
4756                  * those algorithms that were not previously registered.
4757                  */
4758                 if (!driver_algs[i].registered)
4759                         continue;
4760
4761                 switch (driver_algs[i].type) {
4762                 case CRYPTO_ALG_TYPE_SKCIPHER:
4763                         crypto_unregister_skcipher(&driver_algs[i].alg.skcipher);
4764                         dev_dbg(dev, "  unregistered cipher %s\n",
4765                                 driver_algs[i].alg.skcipher.base.cra_driver_name);
4766                         driver_algs[i].registered = false;
4767                         break;
4768                 case CRYPTO_ALG_TYPE_AHASH:
4769                         crypto_unregister_ahash(&driver_algs[i].alg.hash);
4770                         cdn = driver_algs[i].alg.hash.halg.base.cra_driver_name;
4771                         dev_dbg(dev, "  unregistered hash %s\n", cdn);
4772                         driver_algs[i].registered = false;
4773                         break;
4774                 case CRYPTO_ALG_TYPE_AEAD:
4775                         crypto_unregister_aead(&driver_algs[i].alg.aead);
4776                         dev_dbg(dev, "  unregistered aead %s\n",
4777                                 driver_algs[i].alg.aead.base.cra_driver_name);
4778                         driver_algs[i].registered = false;
4779                         break;
4780                 }
4781         }
4782         spu_free_debugfs();
4783         spu_mb_release(pdev);
4784         return 0;
4785 }
4786
4787 /* ===== Kernel Module API ===== */
4788
4789 static struct platform_driver bcm_spu_pdriver = {
4790         .driver = {
4791                    .name = "brcm-spu-crypto",
4792                    .of_match_table = of_match_ptr(bcm_spu_dt_ids),
4793                    },
4794         .probe = bcm_spu_probe,
4795         .remove = bcm_spu_remove,
4796 };
4797 module_platform_driver(bcm_spu_pdriver);
4798
4799 MODULE_AUTHOR("Rob Rice <rob.rice@broadcom.com>");
4800 MODULE_DESCRIPTION("Broadcom symmetric crypto offload driver");
4801 MODULE_LICENSE("GPL v2");