drivers/crypto/talitos.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * talitos - Freescale Integrated Security Engine (SEC) device driver
   4  *
   5  * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
   6  *
   7  * Scatterlist Crypto API glue code copied from files with the following:
   8  * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
   9  *
  10  * Crypto algorithm registration code copied from hifn driver:
  11  * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
  12  * All rights reserved.
  13  */
  14
  15 #include <linux/kernel.h>
  16 #include <linux/module.h>
  17 #include <linux/mod_devicetable.h>
  18 #include <linux/device.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/crypto.h>
  21 #include <linux/hw_random.h>
  22 #include <linux/of_address.h>
  23 #include <linux/of_irq.h>
  24 #include <linux/of_platform.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/io.h>
  27 #include <linux/spinlock.h>
  28 #include <linux/rtnetlink.h>
  29 #include <linux/slab.h>
  30
  31 #include <crypto/algapi.h>
  32 #include <crypto/aes.h>
  33 #include <crypto/internal/des.h>
  34 #include <crypto/sha1.h>
  35 #include <crypto/sha2.h>
  36 #include <crypto/md5.h>
  37 #include <crypto/internal/aead.h>
  38 #include <crypto/authenc.h>
  39 #include <crypto/internal/skcipher.h>
  40 #include <crypto/hash.h>
  41 #include <crypto/internal/hash.h>
  42 #include <crypto/scatterwalk.h>
  43
  44 #include "talitos.h"
  45
  46 static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
  47                            unsigned int len, bool is_sec1)
  48 {
  49         ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
  50         if (is_sec1) {
  51                 ptr->len1 = cpu_to_be16(len);
  52         } else {
  53                 ptr->len = cpu_to_be16(len);
  54                 ptr->eptr = upper_32_bits(dma_addr);
  55         }
  56 }
  57
  58 static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
  59                              struct talitos_ptr *src_ptr, bool is_sec1)
  60 {
  61         dst_ptr->ptr = src_ptr->ptr;
  62         if (is_sec1) {
  63                 dst_ptr->len1 = src_ptr->len1;
  64         } else {
  65                 dst_ptr->len = src_ptr->len;
  66                 dst_ptr->eptr = src_ptr->eptr;
  67         }
  68 }
  69
  70 static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
  71                                            bool is_sec1)
  72 {
  73         if (is_sec1)
  74                 return be16_to_cpu(ptr->len1);
  75         else
  76                 return be16_to_cpu(ptr->len);
  77 }
  78
  79 static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val,
  80                                    bool is_sec1)
  81 {
  82         if (!is_sec1)
  83                 ptr->j_extent = val;
  84 }
  85
  86 static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1)
  87 {
  88         if (!is_sec1)
  89                 ptr->j_extent |= val;
  90 }
  91
  92 /*
  93  * map virtual single (contiguous) pointer to h/w descriptor pointer
  94  */
  95 static void __map_single_talitos_ptr(struct device *dev,
  96                                      struct talitos_ptr *ptr,
  97                                      unsigned int len, void *data,
  98                                      enum dma_data_direction dir,
  99                                      unsigned long attrs)
 100 {
 101         dma_addr_t dma_addr = dma_map_single_attrs(dev, data, len, dir, attrs);
 102         struct talitos_private *priv = dev_get_drvdata(dev);
 103         bool is_sec1 = has_ftr_sec1(priv);
 104
 105         to_talitos_ptr(ptr, dma_addr, len, is_sec1);
 106 }
 107
 108 static void map_single_talitos_ptr(struct device *dev,
 109                                    struct talitos_ptr *ptr,
 110                                    unsigned int len, void *data,
 111                                    enum dma_data_direction dir)
 112 {
 113         __map_single_talitos_ptr(dev, ptr, len, data, dir, 0);
 114 }
 115
 116 static void map_single_talitos_ptr_nosync(struct device *dev,
 117                                           struct talitos_ptr *ptr,
 118                                           unsigned int len, void *data,
 119                                           enum dma_data_direction dir)
 120 {
 121         __map_single_talitos_ptr(dev, ptr, len, data, dir,
 122                                  DMA_ATTR_SKIP_CPU_SYNC);
 123 }
 124
 125 /*
 126  * unmap bus single (contiguous) h/w descriptor pointer
 127  */
 128 static void unmap_single_talitos_ptr(struct device *dev,
 129                                      struct talitos_ptr *ptr,
 130                                      enum dma_data_direction dir)
 131 {
 132         struct talitos_private *priv = dev_get_drvdata(dev);
 133         bool is_sec1 = has_ftr_sec1(priv);
 134
 135         dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
 136                          from_talitos_ptr_len(ptr, is_sec1), dir);
 137 }
 138
 139 static int reset_channel(struct device *dev, int ch)
 140 {
 141         struct talitos_private *priv = dev_get_drvdata(dev);
 142         unsigned int timeout = TALITOS_TIMEOUT;
 143         bool is_sec1 = has_ftr_sec1(priv);
 144
 145         if (is_sec1) {
 146                 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
 147                           TALITOS1_CCCR_LO_RESET);
 148
 149                 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
 150                         TALITOS1_CCCR_LO_RESET) && --timeout)
 151                         cpu_relax();
 152         } else {
 153                 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
 154                           TALITOS2_CCCR_RESET);
 155
 156                 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
 157                         TALITOS2_CCCR_RESET) && --timeout)
 158                         cpu_relax();
 159         }
 160
 161         if (timeout == 0) {
 162                 dev_err(dev, "failed to reset channel %d\n", ch);
 163                 return -EIO;
 164         }
 165
 166         /* set 36-bit addressing, done writeback enable and done IRQ enable */
 167         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
 168                   TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
 169         /* enable chaining descriptors */
 170         if (is_sec1)
 171                 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
 172                           TALITOS_CCCR_LO_NE);
 173
 174         /* and ICCR writeback, if available */
 175         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
 176                 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
 177                           TALITOS_CCCR_LO_IWSE);
 178
 179         return 0;
 180 }
 181
 182 static int reset_device(struct device *dev)
 183 {
 184         struct talitos_private *priv = dev_get_drvdata(dev);
 185         unsigned int timeout = TALITOS_TIMEOUT;
 186         bool is_sec1 = has_ftr_sec1(priv);
 187         u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
 188
 189         setbits32(priv->reg + TALITOS_MCR, mcr);
 190
 191         while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
 192                && --timeout)
 193                 cpu_relax();
 194
 195         if (priv->irq[1]) {
 196                 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
 197                 setbits32(priv->reg + TALITOS_MCR, mcr);
 198         }
 199
 200         if (timeout == 0) {
 201                 dev_err(dev, "failed to reset device\n");
 202                 return -EIO;
 203         }
 204
 205         return 0;
 206 }
 207
 208 /*
 209  * Reset and initialize the device
 210  */
 211 static int init_device(struct device *dev)
 212 {
 213         struct talitos_private *priv = dev_get_drvdata(dev);
 214         int ch, err;
 215         bool is_sec1 = has_ftr_sec1(priv);
 216
 217         /*
 218          * Master reset
 219          * errata documentation: warning: certain SEC interrupts
 220          * are not fully cleared by writing the MCR:SWR bit,
 221          * set bit twice to completely reset
 222          */
 223         err = reset_device(dev);
 224         if (err)
 225                 return err;
 226
 227         err = reset_device(dev);
 228         if (err)
 229                 return err;
 230
 231         /* reset channels */
 232         for (ch = 0; ch < priv->num_channels; ch++) {
 233                 err = reset_channel(dev, ch);
 234                 if (err)
 235                         return err;
 236         }
 237
 238         /* enable channel done and error interrupts */
 239         if (is_sec1) {
 240                 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
 241                 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
 242                 /* disable parity error check in DEU (erroneous? test vect.) */
 243                 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
 244         } else {
 245                 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
 246                 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
 247         }
 248
 249         /* disable integrity check error interrupts (use writeback instead) */
 250         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
 251                 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
 252                           TALITOS_MDEUICR_LO_ICE);
 253
 254         return 0;
 255 }
 256
 257 /**
 258  * talitos_submit - submits a descriptor to the device for processing
 259  * @dev:        the SEC device to be used
 260  * @ch:         the SEC device channel to be used
 261  * @desc:       the descriptor to be processed by the device
 262  * @callback:   whom to call when processing is complete
 263  * @context:    a handle for use by caller (optional)
 264  *
 265  * desc must contain valid dma-mapped (bus physical) address pointers.
 266  * callback must check err and feedback in descriptor header
 267  * for device processing status.
 268  */
 269 static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
 270                           void (*callback)(struct device *dev,
 271                                            struct talitos_desc *desc,
 272                                            void *context, int error),
 273                           void *context)
 274 {
 275         struct talitos_private *priv = dev_get_drvdata(dev);
 276         struct talitos_request *request;
 277         unsigned long flags;
 278         int head;
 279         bool is_sec1 = has_ftr_sec1(priv);
 280
 281         spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
 282
 283         if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
 284                 /* h/w fifo is full */
 285                 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
 286                 return -EAGAIN;
 287         }
 288
 289         head = priv->chan[ch].head;
 290         request = &priv->chan[ch].fifo[head];
 291
 292         /* map descriptor and save caller data */
 293         if (is_sec1) {
 294                 desc->hdr1 = desc->hdr;
 295                 request->dma_desc = dma_map_single(dev, &desc->hdr1,
 296                                                    TALITOS_DESC_SIZE,
 297                                                    DMA_BIDIRECTIONAL);
 298         } else {
 299                 request->dma_desc = dma_map_single(dev, desc,
 300                                                    TALITOS_DESC_SIZE,
 301                                                    DMA_BIDIRECTIONAL);
 302         }
 303         request->callback = callback;
 304         request->context = context;
 305
 306         /* increment fifo head */
 307         priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
 308
 309         smp_wmb();
 310         request->desc = desc;
 311
 312         /* GO! */
 313         wmb();
 314         out_be32(priv->chan[ch].reg + TALITOS_FF,
 315                  upper_32_bits(request->dma_desc));
 316         out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
 317                  lower_32_bits(request->dma_desc));
 318
 319         spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
 320
 321         return -EINPROGRESS;
 322 }
 323
 324 static __be32 get_request_hdr(struct talitos_request *request, bool is_sec1)
 325 {
 326         struct talitos_edesc *edesc;
 327
 328         if (!is_sec1)
 329                 return request->desc->hdr;
 330
 331         if (!request->desc->next_desc)
 332                 return request->desc->hdr1;
 333
 334         edesc = container_of(request->desc, struct talitos_edesc, desc);
 335
 336         return ((struct talitos_desc *)(edesc->buf + edesc->dma_len))->hdr1;
 337 }
 338
 339 /*
 340  * process what was done, notify callback of error if not
 341  */
 342 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
 343 {
 344         struct talitos_private *priv = dev_get_drvdata(dev);
 345         struct talitos_request *request, saved_req;
 346         unsigned long flags;
 347         int tail, status;
 348         bool is_sec1 = has_ftr_sec1(priv);
 349
 350         spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
 351
 352         tail = priv->chan[ch].tail;
 353         while (priv->chan[ch].fifo[tail].desc) {
 354                 __be32 hdr;
 355
 356                 request = &priv->chan[ch].fifo[tail];
 357
 358                 /* descriptors with their done bits set don't get the error */
 359                 rmb();
 360                 hdr = get_request_hdr(request, is_sec1);
 361
 362                 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
 363                         status = 0;
 364                 else
 365                         if (!error)
 366                                 break;
 367                         else
 368                                 status = error;
 369
 370                 dma_unmap_single(dev, request->dma_desc,
 371                                  TALITOS_DESC_SIZE,
 372                                  DMA_BIDIRECTIONAL);
 373
 374                 /* copy entries so we can call callback outside lock */
 375                 saved_req.desc = request->desc;
 376                 saved_req.callback = request->callback;
 377                 saved_req.context = request->context;
 378
 379                 /* release request entry in fifo */
 380                 smp_wmb();
 381                 request->desc = NULL;
 382
 383                 /* increment fifo tail */
 384                 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
 385
 386                 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
 387
 388                 atomic_dec(&priv->chan[ch].submit_count);
 389
 390                 saved_req.callback(dev, saved_req.desc, saved_req.context,
 391                                    status);
 392                 /* channel may resume processing in single desc error case */
 393                 if (error && !reset_ch && status == error)
 394                         return;
 395                 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
 396                 tail = priv->chan[ch].tail;
 397         }
 398
 399         spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
 400 }
 401
 402 /*
 403  * process completed requests for channels that have done status
 404  */
 405 #define DEF_TALITOS1_DONE(name, ch_done_mask)                           \
 406 static void talitos1_done_##name(unsigned long data)                    \
 407 {                                                                       \
 408         struct device *dev = (struct device *)data;                     \
 409         struct talitos_private *priv = dev_get_drvdata(dev);            \
 410         unsigned long flags;                                            \
 411                                                                         \
 412         if (ch_done_mask & 0x10000000)                                  \
 413                 flush_channel(dev, 0, 0, 0);                    \
 414         if (ch_done_mask & 0x40000000)                                  \
 415                 flush_channel(dev, 1, 0, 0);                    \
 416         if (ch_done_mask & 0x00010000)                                  \
 417                 flush_channel(dev, 2, 0, 0);                    \
 418         if (ch_done_mask & 0x00040000)                                  \
 419                 flush_channel(dev, 3, 0, 0);                    \
 420                                                                         \
 421         /* At this point, all completed channels have been processed */ \
 422         /* Unmask done interrupts for channels completed later on. */   \
 423         spin_lock_irqsave(&priv->reg_lock, flags);                      \
 424         clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);               \
 425         clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);    \
 426         spin_unlock_irqrestore(&priv->reg_lock, flags);                 \
 427 }
 428
 429 DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
 430 DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE)
 431
 432 #define DEF_TALITOS2_DONE(name, ch_done_mask)                           \
 433 static void talitos2_done_##name(unsigned long data)                    \
 434 {                                                                       \
 435         struct device *dev = (struct device *)data;                     \
 436         struct talitos_private *priv = dev_get_drvdata(dev);            \
 437         unsigned long flags;                                            \
 438                                                                         \
 439         if (ch_done_mask & 1)                                           \
 440                 flush_channel(dev, 0, 0, 0);                            \
 441         if (ch_done_mask & (1 << 2))                                    \
 442                 flush_channel(dev, 1, 0, 0);                            \
 443         if (ch_done_mask & (1 << 4))                                    \
 444                 flush_channel(dev, 2, 0, 0);                            \
 445         if (ch_done_mask & (1 << 6))                                    \
 446                 flush_channel(dev, 3, 0, 0);                            \
 447                                                                         \
 448         /* At this point, all completed channels have been processed */ \
 449         /* Unmask done interrupts for channels completed later on. */   \
 450         spin_lock_irqsave(&priv->reg_lock, flags);                      \
 451         setbits32(priv->reg + TALITOS_IMR, ch_done_mask);               \
 452         setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);    \
 453         spin_unlock_irqrestore(&priv->reg_lock, flags);                 \
 454 }
 455
 456 DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
 457 DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE)
 458 DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
 459 DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
 460
 461 /*
 462  * locate current (offending) descriptor
 463  */
 464 static __be32 current_desc_hdr(struct device *dev, int ch)
 465 {
 466         struct talitos_private *priv = dev_get_drvdata(dev);
 467         int tail, iter;
 468         dma_addr_t cur_desc;
 469
 470         cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
 471         cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
 472
 473         if (!cur_desc) {
 474                 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
 475                 return 0;
 476         }
 477
 478         tail = priv->chan[ch].tail;
 479
 480         iter = tail;
 481         while (priv->chan[ch].fifo[iter].dma_desc != cur_desc &&
 482                priv->chan[ch].fifo[iter].desc->next_desc != cpu_to_be32(cur_desc)) {
 483                 iter = (iter + 1) & (priv->fifo_len - 1);
 484                 if (iter == tail) {
 485                         dev_err(dev, "couldn't locate current descriptor\n");
 486                         return 0;
 487                 }
 488         }
 489
 490         if (priv->chan[ch].fifo[iter].desc->next_desc == cpu_to_be32(cur_desc)) {
 491                 struct talitos_edesc *edesc;
 492
 493                 edesc = container_of(priv->chan[ch].fifo[iter].desc,
 494                                      struct talitos_edesc, desc);
 495                 return ((struct talitos_desc *)
 496                         (edesc->buf + edesc->dma_len))->hdr;
 497         }
 498
 499         return priv->chan[ch].fifo[iter].desc->hdr;
 500 }
 501
 502 /*
 503  * user diagnostics; report root cause of error based on execution unit status
 504  */
 505 static void report_eu_error(struct device *dev, int ch, __be32 desc_hdr)
 506 {
 507         struct talitos_private *priv = dev_get_drvdata(dev);
 508         int i;
 509
 510         if (!desc_hdr)
 511                 desc_hdr = cpu_to_be32(in_be32(priv->chan[ch].reg + TALITOS_DESCBUF));
 512
 513         switch (desc_hdr & DESC_HDR_SEL0_MASK) {
 514         case DESC_HDR_SEL0_AFEU:
 515                 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
 516                         in_be32(priv->reg_afeu + TALITOS_EUISR),
 517                         in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
 518                 break;
 519         case DESC_HDR_SEL0_DEU:
 520                 dev_err(dev, "DEUISR 0x%08x_%08x\n",
 521                         in_be32(priv->reg_deu + TALITOS_EUISR),
 522                         in_be32(priv->reg_deu + TALITOS_EUISR_LO));
 523                 break;
 524         case DESC_HDR_SEL0_MDEUA:
 525         case DESC_HDR_SEL0_MDEUB:
 526                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
 527                         in_be32(priv->reg_mdeu + TALITOS_EUISR),
 528                         in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
 529                 break;
 530         case DESC_HDR_SEL0_RNG:
 531                 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
 532                         in_be32(priv->reg_rngu + TALITOS_ISR),
 533                         in_be32(priv->reg_rngu + TALITOS_ISR_LO));
 534                 break;
 535         case DESC_HDR_SEL0_PKEU:
 536                 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
 537                         in_be32(priv->reg_pkeu + TALITOS_EUISR),
 538                         in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
 539                 break;
 540         case DESC_HDR_SEL0_AESU:
 541                 dev_err(dev, "AESUISR 0x%08x_%08x\n",
 542                         in_be32(priv->reg_aesu + TALITOS_EUISR),
 543                         in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
 544                 break;
 545         case DESC_HDR_SEL0_CRCU:
 546                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
 547                         in_be32(priv->reg_crcu + TALITOS_EUISR),
 548                         in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
 549                 break;
 550         case DESC_HDR_SEL0_KEU:
 551                 dev_err(dev, "KEUISR 0x%08x_%08x\n",
 552                         in_be32(priv->reg_pkeu + TALITOS_EUISR),
 553                         in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
 554                 break;
 555         }
 556
 557         switch (desc_hdr & DESC_HDR_SEL1_MASK) {
 558         case DESC_HDR_SEL1_MDEUA:
 559         case DESC_HDR_SEL1_MDEUB:
 560                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
 561                         in_be32(priv->reg_mdeu + TALITOS_EUISR),
 562                         in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
 563                 break;
 564         case DESC_HDR_SEL1_CRCU:
 565                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
 566                         in_be32(priv->reg_crcu + TALITOS_EUISR),
 567                         in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
 568                 break;
 569         }
 570
 571         for (i = 0; i < 8; i++)
 572                 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
 573                         in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
 574                         in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
 575 }
 576
 577 /*
 578  * recover from error interrupts
 579  */
 580 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
 581 {
 582         struct talitos_private *priv = dev_get_drvdata(dev);
 583         unsigned int timeout = TALITOS_TIMEOUT;
 584         int ch, error, reset_dev = 0;
 585         u32 v_lo;
 586         bool is_sec1 = has_ftr_sec1(priv);
 587         int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
 588
 589         for (ch = 0; ch < priv->num_channels; ch++) {
 590                 /* skip channels without errors */
 591                 if (is_sec1) {
 592                         /* bits 29, 31, 17, 19 */
 593                         if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
 594                                 continue;
 595                 } else {
 596                         if (!(isr & (1 << (ch * 2 + 1))))
 597                                 continue;
 598                 }
 599
 600                 error = -EINVAL;
 601
 602                 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
 603
 604                 if (v_lo & TALITOS_CCPSR_LO_DOF) {
 605                         dev_err(dev, "double fetch fifo overflow error\n");
 606                         error = -EAGAIN;
 607                         reset_ch = 1;
 608                 }
 609                 if (v_lo & TALITOS_CCPSR_LO_SOF) {
 610                         /* h/w dropped descriptor */
 611                         dev_err(dev, "single fetch fifo overflow error\n");
 612                         error = -EAGAIN;
 613                 }
 614                 if (v_lo & TALITOS_CCPSR_LO_MDTE)
 615                         dev_err(dev, "master data transfer error\n");
 616                 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
 617                         dev_err(dev, is_sec1 ? "pointer not complete error\n"
 618                                              : "s/g data length zero error\n");
 619                 if (v_lo & TALITOS_CCPSR_LO_FPZ)
 620                         dev_err(dev, is_sec1 ? "parity error\n"
 621                                              : "fetch pointer zero error\n");
 622                 if (v_lo & TALITOS_CCPSR_LO_IDH)
 623                         dev_err(dev, "illegal descriptor header error\n");
 624                 if (v_lo & TALITOS_CCPSR_LO_IEU)
 625                         dev_err(dev, is_sec1 ? "static assignment error\n"
 626                                              : "invalid exec unit error\n");
 627                 if (v_lo & TALITOS_CCPSR_LO_EU)
 628                         report_eu_error(dev, ch, current_desc_hdr(dev, ch));
 629                 if (!is_sec1) {
 630                         if (v_lo & TALITOS_CCPSR_LO_GB)
 631                                 dev_err(dev, "gather boundary error\n");
 632                         if (v_lo & TALITOS_CCPSR_LO_GRL)
 633                                 dev_err(dev, "gather return/length error\n");
 634                         if (v_lo & TALITOS_CCPSR_LO_SB)
 635                                 dev_err(dev, "scatter boundary error\n");
 636                         if (v_lo & TALITOS_CCPSR_LO_SRL)
 637                                 dev_err(dev, "scatter return/length error\n");
 638                 }
 639
 640                 flush_channel(dev, ch, error, reset_ch);
 641
 642                 if (reset_ch) {
 643                         reset_channel(dev, ch);
 644                 } else {
 645                         setbits32(priv->chan[ch].reg + TALITOS_CCCR,
 646                                   TALITOS2_CCCR_CONT);
 647                         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
 648                         while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
 649                                TALITOS2_CCCR_CONT) && --timeout)
 650                                 cpu_relax();
 651                         if (timeout == 0) {
 652                                 dev_err(dev, "failed to restart channel %d\n",
 653                                         ch);
 654                                 reset_dev = 1;
 655                         }
 656                 }
 657         }
 658         if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
 659             (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
 660                 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
 661                         dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
 662                                 isr, isr_lo);
 663                 else
 664                         dev_err(dev, "done overflow, internal time out, or "
 665                                 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
 666
 667                 /* purge request queues */
 668                 for (ch = 0; ch < priv->num_channels; ch++)
 669                         flush_channel(dev, ch, -EIO, 1);
 670
 671                 /* reset and reinitialize the device */
 672                 init_device(dev);
 673         }
 674 }
 675
 676 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)          \
 677 static irqreturn_t talitos1_interrupt_##name(int irq, void *data)              \
 678 {                                                                              \
 679         struct device *dev = data;                                             \
 680         struct talitos_private *priv = dev_get_drvdata(dev);                   \
 681         u32 isr, isr_lo;                                                       \
 682         unsigned long flags;                                                   \
 683                                                                                \
 684         spin_lock_irqsave(&priv->reg_lock, flags);                             \
 685         isr = in_be32(priv->reg + TALITOS_ISR);                                \
 686         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);                          \
 687         /* Acknowledge interrupt */                                            \
 688         out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
 689         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);                          \
 690                                                                                \
 691         if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) {    \
 692                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 693                 talitos_error(dev, isr & ch_err_mask, isr_lo);                 \
 694         }                                                                      \
 695         else {                                                                 \
 696                 if (likely(isr & ch_done_mask)) {                              \
 697                         /* mask further done interrupts. */                    \
 698                         setbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
 699                         /* done_task will unmask done interrupts at exit */    \
 700                         tasklet_schedule(&priv->done_task[tlet]);              \
 701                 }                                                              \
 702                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 703         }                                                                      \
 704                                                                                \
 705         return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
 706                                                                 IRQ_NONE;      \
 707 }
 708
 709 DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
 710
 711 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)          \
 712 static irqreturn_t talitos2_interrupt_##name(int irq, void *data)              \
 713 {                                                                              \
 714         struct device *dev = data;                                             \
 715         struct talitos_private *priv = dev_get_drvdata(dev);                   \
 716         u32 isr, isr_lo;                                                       \
 717         unsigned long flags;                                                   \
 718                                                                                \
 719         spin_lock_irqsave(&priv->reg_lock, flags);                             \
 720         isr = in_be32(priv->reg + TALITOS_ISR);                                \
 721         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);                          \
 722         /* Acknowledge interrupt */                                            \
 723         out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
 724         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);                          \
 725                                                                                \
 726         if (unlikely(isr & ch_err_mask || isr_lo)) {                           \
 727                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 728                 talitos_error(dev, isr & ch_err_mask, isr_lo);                 \
 729         }                                                                      \
 730         else {                                                                 \
 731                 if (likely(isr & ch_done_mask)) {                              \
 732                         /* mask further done interrupts. */                    \
 733                         clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
 734                         /* done_task will unmask done interrupts at exit */    \
 735                         tasklet_schedule(&priv->done_task[tlet]);              \
 736                 }                                                              \
 737                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 738         }                                                                      \
 739                                                                                \
 740         return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
 741                                                                 IRQ_NONE;      \
 742 }
 743
 744 DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
 745 DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
 746                        0)
 747 DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
 748                        1)
 749
 750 /*
 751  * hwrng
 752  */
 753 static int talitos_rng_data_present(struct hwrng *rng, int wait)
 754 {
 755         struct device *dev = (struct device *)rng->priv;
 756         struct talitos_private *priv = dev_get_drvdata(dev);
 757         u32 ofl;
 758         int i;
 759
 760         for (i = 0; i < 20; i++) {
 761                 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
 762                       TALITOS_RNGUSR_LO_OFL;
 763                 if (ofl || !wait)
 764                         break;
 765                 udelay(10);
 766         }
 767
 768         return !!ofl;
 769 }
 770
 771 static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
 772 {
 773         struct device *dev = (struct device *)rng->priv;
 774         struct talitos_private *priv = dev_get_drvdata(dev);
 775
 776         /* rng fifo requires 64-bit accesses */
 777         *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
 778         *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
 779
 780         return sizeof(u32);
 781 }
 782
 783 static int talitos_rng_init(struct hwrng *rng)
 784 {
 785         struct device *dev = (struct device *)rng->priv;
 786         struct talitos_private *priv = dev_get_drvdata(dev);
 787         unsigned int timeout = TALITOS_TIMEOUT;
 788
 789         setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
 790         while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
 791                  & TALITOS_RNGUSR_LO_RD)
 792                && --timeout)
 793                 cpu_relax();
 794         if (timeout == 0) {
 795                 dev_err(dev, "failed to reset rng hw\n");
 796                 return -ENODEV;
 797         }
 798
 799         /* start generating */
 800         setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
 801
 802         return 0;
 803 }
 804
 805 static int talitos_register_rng(struct device *dev)
 806 {
 807         struct talitos_private *priv = dev_get_drvdata(dev);
 808         int err;
 809
 810         priv->rng.name          = dev_driver_string(dev);
 811         priv->rng.init          = talitos_rng_init;
 812         priv->rng.data_present  = talitos_rng_data_present;
 813         priv->rng.data_read     = talitos_rng_data_read;
 814         priv->rng.priv          = (unsigned long)dev;
 815
 816         err = hwrng_register(&priv->rng);
 817         if (!err)
 818                 priv->rng_registered = true;
 819
 820         return err;
 821 }
 822
 823 static void talitos_unregister_rng(struct device *dev)
 824 {
 825         struct talitos_private *priv = dev_get_drvdata(dev);
 826
 827         if (!priv->rng_registered)
 828                 return;
 829
 830         hwrng_unregister(&priv->rng);
 831         priv->rng_registered = false;
 832 }
 833
 834 /*
 835  * crypto alg
 836  */
 837 #define TALITOS_CRA_PRIORITY            3000
 838 /*
 839  * Defines a priority for doing AEAD with descriptors type
 840  * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP
 841  */
 842 #define TALITOS_CRA_PRIORITY_AEAD_HSNA  (TALITOS_CRA_PRIORITY - 1)
 843 #ifdef CONFIG_CRYPTO_DEV_TALITOS2
 844 #define TALITOS_MAX_KEY_SIZE            (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE)
 845 #else
 846 #define TALITOS_MAX_KEY_SIZE            (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE)
 847 #endif
 848 #define TALITOS_MAX_IV_LENGTH           16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
 849
 850 struct talitos_ctx {
 851         struct device *dev;
 852         int ch;
 853         __be32 desc_hdr_template;
 854         u8 key[TALITOS_MAX_KEY_SIZE];
 855         u8 iv[TALITOS_MAX_IV_LENGTH];
 856         dma_addr_t dma_key;
 857         unsigned int keylen;
 858         unsigned int enckeylen;
 859         unsigned int authkeylen;
 860 };
 861
 862 #define HASH_MAX_BLOCK_SIZE             SHA512_BLOCK_SIZE
 863 #define TALITOS_MDEU_MAX_CONTEXT_SIZE   TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
 864
 865 struct talitos_ahash_req_ctx {
 866         u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
 867         unsigned int hw_context_size;
 868         u8 buf[2][HASH_MAX_BLOCK_SIZE];
 869         int buf_idx;
 870         unsigned int swinit;
 871         unsigned int first;
 872         unsigned int last;
 873         unsigned int to_hash_later;
 874         unsigned int nbuf;
 875         struct scatterlist bufsl[2];
 876         struct scatterlist *psrc;
 877 };
 878
 879 struct talitos_export_state {
 880         u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
 881         u8 buf[HASH_MAX_BLOCK_SIZE];
 882         unsigned int swinit;
 883         unsigned int first;
 884         unsigned int last;
 885         unsigned int to_hash_later;
 886         unsigned int nbuf;
 887 };
 888
 889 static int aead_setkey(struct crypto_aead *authenc,
 890                        const u8 *key, unsigned int keylen)
 891 {
 892         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 893         struct device *dev = ctx->dev;
 894         struct crypto_authenc_keys keys;
 895
 896         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 897                 goto badkey;
 898
 899         if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
 900                 goto badkey;
 901
 902         if (ctx->keylen)
 903                 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
 904
 905         memcpy(ctx->key, keys.authkey, keys.authkeylen);
 906         memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
 907
 908         ctx->keylen = keys.authkeylen + keys.enckeylen;
 909         ctx->enckeylen = keys.enckeylen;
 910         ctx->authkeylen = keys.authkeylen;
 911         ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen,
 912                                       DMA_TO_DEVICE);
 913
 914         memzero_explicit(&keys, sizeof(keys));
 915         return 0;
 916
 917 badkey:
 918         memzero_explicit(&keys, sizeof(keys));
 919         return -EINVAL;
 920 }
 921
 922 static int aead_des3_setkey(struct crypto_aead *authenc,
 923                             const u8 *key, unsigned int keylen)
 924 {
 925         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 926         struct device *dev = ctx->dev;
 927         struct crypto_authenc_keys keys;
 928         int err;
 929
 930         err = crypto_authenc_extractkeys(&keys, key, keylen);
 931         if (unlikely(err))
 932                 goto out;
 933
 934         err = -EINVAL;
 935         if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
 936                 goto out;
 937
 938         err = verify_aead_des3_key(authenc, keys.enckey, keys.enckeylen);
 939         if (err)
 940                 goto out;
 941
 942         if (ctx->keylen)
 943                 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
 944
 945         memcpy(ctx->key, keys.authkey, keys.authkeylen);
 946         memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
 947
 948         ctx->keylen = keys.authkeylen + keys.enckeylen;
 949         ctx->enckeylen = keys.enckeylen;
 950         ctx->authkeylen = keys.authkeylen;
 951         ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen,
 952                                       DMA_TO_DEVICE);
 953
 954 out:
 955         memzero_explicit(&keys, sizeof(keys));
 956         return err;
 957 }
 958
 959 static void talitos_sg_unmap(struct device *dev,
 960                              struct talitos_edesc *edesc,
 961                              struct scatterlist *src,
 962                              struct scatterlist *dst,
 963                              unsigned int len, unsigned int offset)
 964 {
 965         struct talitos_private *priv = dev_get_drvdata(dev);
 966         bool is_sec1 = has_ftr_sec1(priv);
 967         unsigned int src_nents = edesc->src_nents ? : 1;
 968         unsigned int dst_nents = edesc->dst_nents ? : 1;
 969
 970         if (is_sec1 && dst && dst_nents > 1) {
 971                 dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset,
 972                                            len, DMA_FROM_DEVICE);
 973                 sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len,
 974                                      offset);
 975         }
 976         if (src != dst) {
 977                 if (src_nents == 1 || !is_sec1)
 978                         dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 979
 980                 if (dst && (dst_nents == 1 || !is_sec1))
 981                         dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
 982         } else if (src_nents == 1 || !is_sec1) {
 983                 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 984         }
 985 }
 986
 987 static void ipsec_esp_unmap(struct device *dev,
 988                             struct talitos_edesc *edesc,
 989                             struct aead_request *areq, bool encrypt)
 990 {
 991         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
 992         struct talitos_ctx *ctx = crypto_aead_ctx(aead);
 993         unsigned int ivsize = crypto_aead_ivsize(aead);
 994         unsigned int authsize = crypto_aead_authsize(aead);
 995         unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize);
 996         bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP;
 997         struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3];
 998
 999         if (is_ipsec_esp)
1000                 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6],
1001                                          DMA_FROM_DEVICE);
1002         unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE);
1003
1004         talitos_sg_unmap(dev, edesc, areq->src, areq->dst,
1005                          cryptlen + authsize, areq->assoclen);
1006
1007         if (edesc->dma_len)
1008                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1009                                  DMA_BIDIRECTIONAL);
1010
1011         if (!is_ipsec_esp) {
1012                 unsigned int dst_nents = edesc->dst_nents ? : 1;
1013
1014                 sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize,
1015                                    areq->assoclen + cryptlen - ivsize);
1016         }
1017 }
1018
1019 /*
1020  * ipsec_esp descriptor callbacks
1021  */
1022 static void ipsec_esp_encrypt_done(struct device *dev,
1023                                    struct talitos_desc *desc, void *context,
1024                                    int err)
1025 {
1026         struct aead_request *areq = context;
1027         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1028         unsigned int ivsize = crypto_aead_ivsize(authenc);
1029         struct talitos_edesc *edesc;
1030
1031         edesc = container_of(desc, struct talitos_edesc, desc);
1032
1033         ipsec_esp_unmap(dev, edesc, areq, true);
1034
1035         dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE);
1036
1037         kfree(edesc);
1038
1039         aead_request_complete(areq, err);
1040 }
1041
1042 static void ipsec_esp_decrypt_swauth_done(struct device *dev,
1043                                           struct talitos_desc *desc,
1044                                           void *context, int err)
1045 {
1046         struct aead_request *req = context;
1047         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1048         unsigned int authsize = crypto_aead_authsize(authenc);
1049         struct talitos_edesc *edesc;
1050         char *oicv, *icv;
1051
1052         edesc = container_of(desc, struct talitos_edesc, desc);
1053
1054         ipsec_esp_unmap(dev, edesc, req, false);
1055
1056         if (!err) {
1057                 /* auth check */
1058                 oicv = edesc->buf + edesc->dma_len;
1059                 icv = oicv - authsize;
1060
1061                 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
1062         }
1063
1064         kfree(edesc);
1065
1066         aead_request_complete(req, err);
1067 }
1068
1069 static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
1070                                           struct talitos_desc *desc,
1071                                           void *context, int err)
1072 {
1073         struct aead_request *req = context;
1074         struct talitos_edesc *edesc;
1075
1076         edesc = container_of(desc, struct talitos_edesc, desc);
1077
1078         ipsec_esp_unmap(dev, edesc, req, false);
1079
1080         /* check ICV auth status */
1081         if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1082                      DESC_HDR_LO_ICCR1_PASS))
1083                 err = -EBADMSG;
1084
1085         kfree(edesc);
1086
1087         aead_request_complete(req, err);
1088 }
1089
1090 /*
1091  * convert scatterlist to SEC h/w link table format
1092  * stop at cryptlen bytes
1093  */
1094 static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1095                                  unsigned int offset, int datalen, int elen,
1096                                  struct talitos_ptr *link_tbl_ptr)
1097 {
1098         int n_sg = elen ? sg_count + 1 : sg_count;
1099         int count = 0;
1100         int cryptlen = datalen + elen;
1101
1102         while (cryptlen && sg && n_sg--) {
1103                 unsigned int len = sg_dma_len(sg);
1104
1105                 if (offset >= len) {
1106                         offset -= len;
1107                         goto next;
1108                 }
1109
1110                 len -= offset;
1111
1112                 if (len > cryptlen)
1113                         len = cryptlen;
1114
1115                 if (datalen > 0 && len > datalen) {
1116                         to_talitos_ptr(link_tbl_ptr + count,
1117                                        sg_dma_address(sg) + offset, datalen, 0);
1118                         to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
1119                         count++;
1120                         len -= datalen;
1121                         offset += datalen;
1122                 }
1123                 to_talitos_ptr(link_tbl_ptr + count,
1124                                sg_dma_address(sg) + offset, len, 0);
1125                 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
1126                 count++;
1127                 cryptlen -= len;
1128                 datalen -= len;
1129                 offset = 0;
1130
1131 next:
1132                 sg = sg_next(sg);
1133         }
1134
1135         /* tag end of link table */
1136         if (count > 0)
1137                 to_talitos_ptr_ext_set(link_tbl_ptr + count - 1,
1138                                        DESC_PTR_LNKTBL_RET, 0);
1139
1140         return count;
1141 }
1142
1143 static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src,
1144                               unsigned int len, struct talitos_edesc *edesc,
1145                               struct talitos_ptr *ptr, int sg_count,
1146                               unsigned int offset, int tbl_off, int elen,
1147                               bool force)
1148 {
1149         struct talitos_private *priv = dev_get_drvdata(dev);
1150         bool is_sec1 = has_ftr_sec1(priv);
1151
1152         if (!src) {
1153                 to_talitos_ptr(ptr, 0, 0, is_sec1);
1154                 return 1;
1155         }
1156         to_talitos_ptr_ext_set(ptr, elen, is_sec1);
1157         if (sg_count == 1 && !force) {
1158                 to_talitos_ptr(ptr, sg_dma_address(src) + offset, len, is_sec1);
1159                 return sg_count;
1160         }
1161         if (is_sec1) {
1162                 to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1);
1163                 return sg_count;
1164         }
1165         sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, elen,
1166                                          &edesc->link_tbl[tbl_off]);
1167         if (sg_count == 1 && !force) {
1168                 /* Only one segment now, so no link tbl needed*/
1169                 copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1);
1170                 return sg_count;
1171         }
1172         to_talitos_ptr(ptr, edesc->dma_link_tbl +
1173                             tbl_off * sizeof(struct talitos_ptr), len, is_sec1);
1174         to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1);
1175
1176         return sg_count;
1177 }
1178
1179 static int talitos_sg_map(struct device *dev, struct scatterlist *src,
1180                           unsigned int len, struct talitos_edesc *edesc,
1181                           struct talitos_ptr *ptr, int sg_count,
1182                           unsigned int offset, int tbl_off)
1183 {
1184         return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset,
1185                                   tbl_off, 0, false);
1186 }
1187
1188 /*
1189  * fill in and submit ipsec_esp descriptor
1190  */
1191 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
1192                      bool encrypt,
1193                      void (*callback)(struct device *dev,
1194                                       struct talitos_desc *desc,
1195                                       void *context, int error))
1196 {
1197         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1198         unsigned int authsize = crypto_aead_authsize(aead);
1199         struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1200         struct device *dev = ctx->dev;
1201         struct talitos_desc *desc = &edesc->desc;
1202         unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize);
1203         unsigned int ivsize = crypto_aead_ivsize(aead);
1204         int tbl_off = 0;
1205         int sg_count, ret;
1206         int elen = 0;
1207         bool sync_needed = false;
1208         struct talitos_private *priv = dev_get_drvdata(dev);
1209         bool is_sec1 = has_ftr_sec1(priv);
1210         bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP;
1211         struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3];
1212         struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2];
1213         dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize;
1214
1215         /* hmac key */
1216         to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1);
1217
1218         sg_count = edesc->src_nents ?: 1;
1219         if (is_sec1 && sg_count > 1)
1220                 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1221                                   areq->assoclen + cryptlen);
1222         else
1223                 sg_count = dma_map_sg(dev, areq->src, sg_count,
1224                                       (areq->src == areq->dst) ?
1225                                       DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1226
1227         /* hmac data */
1228         ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc,
1229                              &desc->ptr[1], sg_count, 0, tbl_off);
1230
1231         if (ret > 1) {
1232                 tbl_off += ret;
1233                 sync_needed = true;
1234         }
1235
1236         /* cipher iv */
1237         to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1);
1238
1239         /* cipher key */
1240         to_talitos_ptr(ckey_ptr, ctx->dma_key  + ctx->authkeylen,
1241                        ctx->enckeylen, is_sec1);
1242
1243         /*
1244          * cipher in
1245          * map and adjust cipher len to aead request cryptlen.
1246          * extent is bytes of HMAC postpended to ciphertext,
1247          * typically 12 for ipsec
1248          */
1249         if (is_ipsec_esp && (desc->hdr & DESC_HDR_MODE1_MDEU_CICV))
1250                 elen = authsize;
1251
1252         ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4],
1253                                  sg_count, areq->assoclen, tbl_off, elen,
1254                                  false);
1255
1256         if (ret > 1) {
1257                 tbl_off += ret;
1258                 sync_needed = true;
1259         }
1260
1261         /* cipher out */
1262         if (areq->src != areq->dst) {
1263                 sg_count = edesc->dst_nents ? : 1;
1264                 if (!is_sec1 || sg_count == 1)
1265                         dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1266         }
1267
1268         if (is_ipsec_esp && encrypt)
1269                 elen = authsize;
1270         else
1271                 elen = 0;
1272         ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5],
1273                                  sg_count, areq->assoclen, tbl_off, elen,
1274                                  is_ipsec_esp && !encrypt);
1275         tbl_off += ret;
1276
1277         if (!encrypt && is_ipsec_esp) {
1278                 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1279
1280                 /* Add an entry to the link table for ICV data */
1281                 to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1);
1282                 to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RET, is_sec1);
1283
1284                 /* icv data follows link tables */
1285                 to_talitos_ptr(tbl_ptr, dma_icv, authsize, is_sec1);
1286                 to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1);
1287                 sync_needed = true;
1288         } else if (!encrypt) {
1289                 to_talitos_ptr(&desc->ptr[6], dma_icv, authsize, is_sec1);
1290                 sync_needed = true;
1291         } else if (!is_ipsec_esp) {
1292                 talitos_sg_map(dev, areq->dst, authsize, edesc, &desc->ptr[6],
1293                                sg_count, areq->assoclen + cryptlen, tbl_off);
1294         }
1295
1296         /* iv out */
1297         if (is_ipsec_esp)
1298                 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1299                                        DMA_FROM_DEVICE);
1300
1301         if (sync_needed)
1302                 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1303                                            edesc->dma_len,
1304                                            DMA_BIDIRECTIONAL);
1305
1306         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1307         if (ret != -EINPROGRESS) {
1308                 ipsec_esp_unmap(dev, edesc, areq, encrypt);
1309                 kfree(edesc);
1310         }
1311         return ret;
1312 }
1313
1314 /*
1315  * allocate and map the extended descriptor
1316  */
1317 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1318                                                  struct scatterlist *src,
1319                                                  struct scatterlist *dst,
1320                                                  u8 *iv,
1321                                                  unsigned int assoclen,
1322                                                  unsigned int cryptlen,
1323                                                  unsigned int authsize,
1324                                                  unsigned int ivsize,
1325                                                  int icv_stashing,
1326                                                  u32 cryptoflags,
1327                                                  bool encrypt)
1328 {
1329         struct talitos_edesc *edesc;
1330         int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len;
1331         dma_addr_t iv_dma = 0;
1332         gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1333                       GFP_ATOMIC;
1334         struct talitos_private *priv = dev_get_drvdata(dev);
1335         bool is_sec1 = has_ftr_sec1(priv);
1336         int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
1337
1338         if (cryptlen + authsize > max_len) {
1339                 dev_err(dev, "length exceeds h/w max limit\n");
1340                 return ERR_PTR(-EINVAL);
1341         }
1342
1343         if (!dst || dst == src) {
1344                 src_len = assoclen + cryptlen + authsize;
1345                 src_nents = sg_nents_for_len(src, src_len);
1346                 if (src_nents < 0) {
1347                         dev_err(dev, "Invalid number of src SG.\n");
1348                         return ERR_PTR(-EINVAL);
1349                 }
1350                 src_nents = (src_nents == 1) ? 0 : src_nents;
1351                 dst_nents = dst ? src_nents : 0;
1352                 dst_len = 0;
1353         } else { /* dst && dst != src*/
1354                 src_len = assoclen + cryptlen + (encrypt ? 0 : authsize);
1355                 src_nents = sg_nents_for_len(src, src_len);
1356                 if (src_nents < 0) {
1357                         dev_err(dev, "Invalid number of src SG.\n");
1358                         return ERR_PTR(-EINVAL);
1359                 }
1360                 src_nents = (src_nents == 1) ? 0 : src_nents;
1361                 dst_len = assoclen + cryptlen + (encrypt ? authsize : 0);
1362                 dst_nents = sg_nents_for_len(dst, dst_len);
1363                 if (dst_nents < 0) {
1364                         dev_err(dev, "Invalid number of dst SG.\n");
1365                         return ERR_PTR(-EINVAL);
1366                 }
1367                 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1368         }
1369
1370         /*
1371          * allocate space for base edesc plus the link tables,
1372          * allowing for two separate entries for AD and generated ICV (+ 2),
1373          * and space for two sets of ICVs (stashed and generated)
1374          */
1375         alloc_len = sizeof(struct talitos_edesc);
1376         if (src_nents || dst_nents || !encrypt) {
1377                 if (is_sec1)
1378                         dma_len = (src_nents ? src_len : 0) +
1379                                   (dst_nents ? dst_len : 0) + authsize;
1380                 else
1381                         dma_len = (src_nents + dst_nents + 2) *
1382                                   sizeof(struct talitos_ptr) + authsize;
1383                 alloc_len += dma_len;
1384         } else {
1385                 dma_len = 0;
1386         }
1387         alloc_len += icv_stashing ? authsize : 0;
1388
1389         /* if its a ahash, add space for a second desc next to the first one */
1390         if (is_sec1 && !dst)
1391                 alloc_len += sizeof(struct talitos_desc);
1392         alloc_len += ivsize;
1393
1394         edesc = kmalloc(alloc_len, GFP_DMA | flags);
1395         if (!edesc)
1396                 return ERR_PTR(-ENOMEM);
1397         if (ivsize) {
1398                 iv = memcpy(((u8 *)edesc) + alloc_len - ivsize, iv, ivsize);
1399                 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1400         }
1401         memset(&edesc->desc, 0, sizeof(edesc->desc));
1402
1403         edesc->src_nents = src_nents;
1404         edesc->dst_nents = dst_nents;
1405         edesc->iv_dma = iv_dma;
1406         edesc->dma_len = dma_len;
1407         if (dma_len)
1408                 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1409                                                      edesc->dma_len,
1410                                                      DMA_BIDIRECTIONAL);
1411
1412         return edesc;
1413 }
1414
1415 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
1416                                               int icv_stashing, bool encrypt)
1417 {
1418         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1419         unsigned int authsize = crypto_aead_authsize(authenc);
1420         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1421         unsigned int ivsize = crypto_aead_ivsize(authenc);
1422         unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize);
1423
1424         return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1425                                    iv, areq->assoclen, cryptlen,
1426                                    authsize, ivsize, icv_stashing,
1427                                    areq->base.flags, encrypt);
1428 }
1429
1430 static int aead_encrypt(struct aead_request *req)
1431 {
1432         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1433         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1434         struct talitos_edesc *edesc;
1435
1436         /* allocate extended descriptor */
1437         edesc = aead_edesc_alloc(req, req->iv, 0, true);
1438         if (IS_ERR(edesc))
1439                 return PTR_ERR(edesc);
1440
1441         /* set encrypt */
1442         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1443
1444         return ipsec_esp(edesc, req, true, ipsec_esp_encrypt_done);
1445 }
1446
1447 static int aead_decrypt(struct aead_request *req)
1448 {
1449         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1450         unsigned int authsize = crypto_aead_authsize(authenc);
1451         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1452         struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1453         struct talitos_edesc *edesc;
1454         void *icvdata;
1455
1456         /* allocate extended descriptor */
1457         edesc = aead_edesc_alloc(req, req->iv, 1, false);
1458         if (IS_ERR(edesc))
1459                 return PTR_ERR(edesc);
1460
1461         if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) &&
1462             (priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
1463             ((!edesc->src_nents && !edesc->dst_nents) ||
1464              priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
1465
1466                 /* decrypt and check the ICV */
1467                 edesc->desc.hdr = ctx->desc_hdr_template |
1468                                   DESC_HDR_DIR_INBOUND |
1469                                   DESC_HDR_MODE1_MDEU_CICV;
1470
1471                 /* reset integrity check result bits */
1472
1473                 return ipsec_esp(edesc, req, false,
1474                                  ipsec_esp_decrypt_hwauth_done);
1475         }
1476
1477         /* Have to check the ICV with software */
1478         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1479
1480         /* stash incoming ICV for later cmp with ICV generated by the h/w */
1481         icvdata = edesc->buf + edesc->dma_len;
1482
1483         sg_pcopy_to_buffer(req->src, edesc->src_nents ? : 1, icvdata, authsize,
1484                            req->assoclen + req->cryptlen - authsize);
1485
1486         return ipsec_esp(edesc, req, false, ipsec_esp_decrypt_swauth_done);
1487 }
1488
1489 static int skcipher_setkey(struct crypto_skcipher *cipher,
1490                              const u8 *key, unsigned int keylen)
1491 {
1492         struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher);
1493         struct device *dev = ctx->dev;
1494
1495         if (ctx->keylen)
1496                 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
1497
1498         memcpy(&ctx->key, key, keylen);
1499         ctx->keylen = keylen;
1500
1501         ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE);
1502
1503         return 0;
1504 }
1505
1506 static int skcipher_des_setkey(struct crypto_skcipher *cipher,
1507                                  const u8 *key, unsigned int keylen)
1508 {
1509         return verify_skcipher_des_key(cipher, key) ?:
1510                skcipher_setkey(cipher, key, keylen);
1511 }
1512
1513 static int skcipher_des3_setkey(struct crypto_skcipher *cipher,
1514                                   const u8 *key, unsigned int keylen)
1515 {
1516         return verify_skcipher_des3_key(cipher, key) ?:
1517                skcipher_setkey(cipher, key, keylen);
1518 }
1519
1520 static int skcipher_aes_setkey(struct crypto_skcipher *cipher,
1521                                   const u8 *key, unsigned int keylen)
1522 {
1523         if (keylen == AES_KEYSIZE_128 || keylen == AES_KEYSIZE_192 ||
1524             keylen == AES_KEYSIZE_256)
1525                 return skcipher_setkey(cipher, key, keylen);
1526
1527         return -EINVAL;
1528 }
1529
1530 static void common_nonsnoop_unmap(struct device *dev,
1531                                   struct talitos_edesc *edesc,
1532                                   struct skcipher_request *areq)
1533 {
1534         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1535
1536         talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, 0);
1537         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1538
1539         if (edesc->dma_len)
1540                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1541                                  DMA_BIDIRECTIONAL);
1542 }
1543
1544 static void skcipher_done(struct device *dev,
1545                             struct talitos_desc *desc, void *context,
1546                             int err)
1547 {
1548         struct skcipher_request *areq = context;
1549         struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1550         struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher);
1551         unsigned int ivsize = crypto_skcipher_ivsize(cipher);
1552         struct talitos_edesc *edesc;
1553
1554         edesc = container_of(desc, struct talitos_edesc, desc);
1555
1556         common_nonsnoop_unmap(dev, edesc, areq);
1557         memcpy(areq->iv, ctx->iv, ivsize);
1558
1559         kfree(edesc);
1560
1561         areq->base.complete(&areq->base, err);
1562 }
1563
1564 static int common_nonsnoop(struct talitos_edesc *edesc,
1565                            struct skcipher_request *areq,
1566                            void (*callback) (struct device *dev,
1567                                              struct talitos_desc *desc,
1568                                              void *context, int error))
1569 {
1570         struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1571         struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher);
1572         struct device *dev = ctx->dev;
1573         struct talitos_desc *desc = &edesc->desc;
1574         unsigned int cryptlen = areq->cryptlen;
1575         unsigned int ivsize = crypto_skcipher_ivsize(cipher);
1576         int sg_count, ret;
1577         bool sync_needed = false;
1578         struct talitos_private *priv = dev_get_drvdata(dev);
1579         bool is_sec1 = has_ftr_sec1(priv);
1580
1581         /* first DWORD empty */
1582
1583         /* cipher iv */
1584         to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1);
1585
1586         /* cipher key */
1587         to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, is_sec1);
1588
1589         sg_count = edesc->src_nents ?: 1;
1590         if (is_sec1 && sg_count > 1)
1591                 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1592                                   cryptlen);
1593         else
1594                 sg_count = dma_map_sg(dev, areq->src, sg_count,
1595                                       (areq->src == areq->dst) ?
1596                                       DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1597         /*
1598          * cipher in
1599          */
1600         sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc,
1601                                   &desc->ptr[3], sg_count, 0, 0);
1602         if (sg_count > 1)
1603                 sync_needed = true;
1604
1605         /* cipher out */
1606         if (areq->src != areq->dst) {
1607                 sg_count = edesc->dst_nents ? : 1;
1608                 if (!is_sec1 || sg_count == 1)
1609                         dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1610         }
1611
1612         ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4],
1613                              sg_count, 0, (edesc->src_nents + 1));
1614         if (ret > 1)
1615                 sync_needed = true;
1616
1617         /* iv out */
1618         map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
1619                                DMA_FROM_DEVICE);
1620
1621         /* last DWORD empty */
1622
1623         if (sync_needed)
1624                 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1625                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1626
1627         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1628         if (ret != -EINPROGRESS) {
1629                 common_nonsnoop_unmap(dev, edesc, areq);
1630                 kfree(edesc);
1631         }
1632         return ret;
1633 }
1634
1635 static struct talitos_edesc *skcipher_edesc_alloc(struct skcipher_request *
1636                                                     areq, bool encrypt)
1637 {
1638         struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1639         struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher);
1640         unsigned int ivsize = crypto_skcipher_ivsize(cipher);
1641
1642         return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1643                                    areq->iv, 0, areq->cryptlen, 0, ivsize, 0,
1644                                    areq->base.flags, encrypt);
1645 }
1646
1647 static int skcipher_encrypt(struct skcipher_request *areq)
1648 {
1649         struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1650         struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher);
1651         struct talitos_edesc *edesc;
1652         unsigned int blocksize =
1653                         crypto_tfm_alg_blocksize(crypto_skcipher_tfm(cipher));
1654
1655         if (!areq->cryptlen)
1656                 return 0;
1657
1658         if (areq->cryptlen % blocksize)
1659                 return -EINVAL;
1660
1661         /* allocate extended descriptor */
1662         edesc = skcipher_edesc_alloc(areq, true);
1663         if (IS_ERR(edesc))
1664                 return PTR_ERR(edesc);
1665
1666         /* set encrypt */
1667         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1668
1669         return common_nonsnoop(edesc, areq, skcipher_done);
1670 }
1671
1672 static int skcipher_decrypt(struct skcipher_request *areq)
1673 {
1674         struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1675         struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher);
1676         struct talitos_edesc *edesc;
1677         unsigned int blocksize =
1678                         crypto_tfm_alg_blocksize(crypto_skcipher_tfm(cipher));
1679
1680         if (!areq->cryptlen)
1681                 return 0;
1682
1683         if (areq->cryptlen % blocksize)
1684                 return -EINVAL;
1685
1686         /* allocate extended descriptor */
1687         edesc = skcipher_edesc_alloc(areq, false);
1688         if (IS_ERR(edesc))
1689                 return PTR_ERR(edesc);
1690
1691         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1692
1693         return common_nonsnoop(edesc, areq, skcipher_done);
1694 }
1695
1696 static void common_nonsnoop_hash_unmap(struct device *dev,
1697                                        struct talitos_edesc *edesc,
1698                                        struct ahash_request *areq)
1699 {
1700         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1701         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1702         struct talitos_private *priv = dev_get_drvdata(dev);
1703         bool is_sec1 = has_ftr_sec1(priv);
1704         struct talitos_desc *desc = &edesc->desc;
1705         struct talitos_desc *desc2 = (struct talitos_desc *)
1706                                      (edesc->buf + edesc->dma_len);
1707
1708         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1709         if (desc->next_desc &&
1710             desc->ptr[5].ptr != desc2->ptr[5].ptr)
1711                 unmap_single_talitos_ptr(dev, &desc2->ptr[5], DMA_FROM_DEVICE);
1712         if (req_ctx->last)
1713                 memcpy(areq->result, req_ctx->hw_context,
1714                        crypto_ahash_digestsize(tfm));
1715
1716         if (req_ctx->psrc)
1717                 talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);
1718
1719         /* When using hashctx-in, must unmap it. */
1720         if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))
1721                 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1722                                          DMA_TO_DEVICE);
1723         else if (desc->next_desc)
1724                 unmap_single_talitos_ptr(dev, &desc2->ptr[1],
1725                                          DMA_TO_DEVICE);
1726
1727         if (is_sec1 && req_ctx->nbuf)
1728                 unmap_single_talitos_ptr(dev, &desc->ptr[3],
1729                                          DMA_TO_DEVICE);
1730
1731         if (edesc->dma_len)
1732                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1733                                  DMA_BIDIRECTIONAL);
1734
1735         if (edesc->desc.next_desc)
1736                 dma_unmap_single(dev, be32_to_cpu(edesc->desc.next_desc),
1737                                  TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL);
1738 }
1739
1740 static void ahash_done(struct device *dev,
1741                        struct talitos_desc *desc, void *context,
1742                        int err)
1743 {
1744         struct ahash_request *areq = context;
1745         struct talitos_edesc *edesc =
1746                  container_of(desc, struct talitos_edesc, desc);
1747         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1748
1749         if (!req_ctx->last && req_ctx->to_hash_later) {
1750                 /* Position any partial block for next update/final/finup */
1751                 req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1;
1752                 req_ctx->nbuf = req_ctx->to_hash_later;
1753         }
1754         common_nonsnoop_hash_unmap(dev, edesc, areq);
1755
1756         kfree(edesc);
1757
1758         areq->base.complete(&areq->base, err);
1759 }
1760
1761 /*
1762  * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1763  * ourself and submit a padded block
1764  */
1765 static void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
1766                                struct talitos_edesc *edesc,
1767                                struct talitos_ptr *ptr)
1768 {
1769         static u8 padded_hash[64] = {
1770                 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1771                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1772                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1773                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1774         };
1775
1776         pr_err_once("Bug in SEC1, padding ourself\n");
1777         edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1778         map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1779                                (char *)padded_hash, DMA_TO_DEVICE);
1780 }
1781
1782 static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1783                                 struct ahash_request *areq, unsigned int length,
1784                                 void (*callback) (struct device *dev,
1785                                                   struct talitos_desc *desc,
1786                                                   void *context, int error))
1787 {
1788         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1789         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1790         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1791         struct device *dev = ctx->dev;
1792         struct talitos_desc *desc = &edesc->desc;
1793         int ret;
1794         bool sync_needed = false;
1795         struct talitos_private *priv = dev_get_drvdata(dev);
1796         bool is_sec1 = has_ftr_sec1(priv);
1797         int sg_count;
1798
1799         /* first DWORD empty */
1800
1801         /* hash context in */
1802         if (!req_ctx->first || req_ctx->swinit) {
1803                 map_single_talitos_ptr_nosync(dev, &desc->ptr[1],
1804                                               req_ctx->hw_context_size,
1805                                               req_ctx->hw_context,
1806                                               DMA_TO_DEVICE);
1807                 req_ctx->swinit = 0;
1808         }
1809         /* Indicate next op is not the first. */
1810         req_ctx->first = 0;
1811
1812         /* HMAC key */
1813         if (ctx->keylen)
1814                 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen,
1815                                is_sec1);
1816
1817         if (is_sec1 && req_ctx->nbuf)
1818                 length -= req_ctx->nbuf;
1819
1820         sg_count = edesc->src_nents ?: 1;
1821         if (is_sec1 && sg_count > 1)
1822                 sg_copy_to_buffer(req_ctx->psrc, sg_count, edesc->buf, length);
1823         else if (length)
1824                 sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count,
1825                                       DMA_TO_DEVICE);
1826         /*
1827          * data in
1828          */
1829         if (is_sec1 && req_ctx->nbuf) {
1830                 map_single_talitos_ptr(dev, &desc->ptr[3], req_ctx->nbuf,
1831                                        req_ctx->buf[req_ctx->buf_idx],
1832                                        DMA_TO_DEVICE);
1833         } else {
1834                 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
1835                                           &desc->ptr[3], sg_count, 0, 0);
1836                 if (sg_count > 1)
1837                         sync_needed = true;
1838         }
1839
1840         /* fifth DWORD empty */
1841
1842         /* hash/HMAC out -or- hash context out */
1843         if (req_ctx->last)
1844                 map_single_talitos_ptr(dev, &desc->ptr[5],
1845                                        crypto_ahash_digestsize(tfm),
1846                                        req_ctx->hw_context, DMA_FROM_DEVICE);
1847         else
1848                 map_single_talitos_ptr_nosync(dev, &desc->ptr[5],
1849                                               req_ctx->hw_context_size,
1850                                               req_ctx->hw_context,
1851                                               DMA_FROM_DEVICE);
1852
1853         /* last DWORD empty */
1854
1855         if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1856                 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1857
1858         if (is_sec1 && req_ctx->nbuf && length) {
1859                 struct talitos_desc *desc2 = (struct talitos_desc *)
1860                                              (edesc->buf + edesc->dma_len);
1861                 dma_addr_t next_desc;
1862
1863                 memset(desc2, 0, sizeof(*desc2));
1864                 desc2->hdr = desc->hdr;
1865                 desc2->hdr &= ~DESC_HDR_MODE0_MDEU_INIT;
1866                 desc2->hdr1 = desc2->hdr;
1867                 desc->hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1868                 desc->hdr |= DESC_HDR_MODE0_MDEU_CONT;
1869                 desc->hdr &= ~DESC_HDR_DONE_NOTIFY;
1870
1871                 if (desc->ptr[1].ptr)
1872                         copy_talitos_ptr(&desc2->ptr[1], &desc->ptr[1],
1873                                          is_sec1);
1874                 else
1875                         map_single_talitos_ptr_nosync(dev, &desc2->ptr[1],
1876                                                       req_ctx->hw_context_size,
1877                                                       req_ctx->hw_context,
1878                                                       DMA_TO_DEVICE);
1879                 copy_talitos_ptr(&desc2->ptr[2], &desc->ptr[2], is_sec1);
1880                 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
1881                                           &desc2->ptr[3], sg_count, 0, 0);
1882                 if (sg_count > 1)
1883                         sync_needed = true;
1884                 copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1);
1885                 if (req_ctx->last)
1886                         map_single_talitos_ptr_nosync(dev, &desc->ptr[5],
1887                                                       req_ctx->hw_context_size,
1888                                                       req_ctx->hw_context,
1889                                                       DMA_FROM_DEVICE);
1890
1891                 next_desc = dma_map_single(dev, &desc2->hdr1, TALITOS_DESC_SIZE,
1892                                            DMA_BIDIRECTIONAL);
1893                 desc->next_desc = cpu_to_be32(next_desc);
1894         }
1895
1896         if (sync_needed)
1897                 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1898                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1899
1900         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1901         if (ret != -EINPROGRESS) {
1902                 common_nonsnoop_hash_unmap(dev, edesc, areq);
1903                 kfree(edesc);
1904         }
1905         return ret;
1906 }
1907
1908 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1909                                                unsigned int nbytes)
1910 {
1911         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1912         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1913         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1914         struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1915         bool is_sec1 = has_ftr_sec1(priv);
1916
1917         if (is_sec1)
1918                 nbytes -= req_ctx->nbuf;
1919
1920         return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0,
1921                                    nbytes, 0, 0, 0, areq->base.flags, false);
1922 }
1923
1924 static int ahash_init(struct ahash_request *areq)
1925 {
1926         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1927         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1928         struct device *dev = ctx->dev;
1929         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1930         unsigned int size;
1931         dma_addr_t dma;
1932
1933         /* Initialize the context */
1934         req_ctx->buf_idx = 0;
1935         req_ctx->nbuf = 0;
1936         req_ctx->first = 1; /* first indicates h/w must init its context */
1937         req_ctx->swinit = 0; /* assume h/w init of context */
1938         size =  (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1939                         ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1940                         : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1941         req_ctx->hw_context_size = size;
1942
1943         dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
1944                              DMA_TO_DEVICE);
1945         dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE);
1946
1947         return 0;
1948 }
1949
1950 /*
1951  * on h/w without explicit sha224 support, we initialize h/w context
1952  * manually with sha224 constants, and tell it to run sha256.
1953  */
1954 static int ahash_init_sha224_swinit(struct ahash_request *areq)
1955 {
1956         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1957
1958         req_ctx->hw_context[0] = SHA224_H0;
1959         req_ctx->hw_context[1] = SHA224_H1;
1960         req_ctx->hw_context[2] = SHA224_H2;
1961         req_ctx->hw_context[3] = SHA224_H3;
1962         req_ctx->hw_context[4] = SHA224_H4;
1963         req_ctx->hw_context[5] = SHA224_H5;
1964         req_ctx->hw_context[6] = SHA224_H6;
1965         req_ctx->hw_context[7] = SHA224_H7;
1966
1967         /* init 64-bit count */
1968         req_ctx->hw_context[8] = 0;
1969         req_ctx->hw_context[9] = 0;
1970
1971         ahash_init(areq);
1972         req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1973
1974         return 0;
1975 }
1976
1977 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1978 {
1979         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1980         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1981         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1982         struct talitos_edesc *edesc;
1983         unsigned int blocksize =
1984                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1985         unsigned int nbytes_to_hash;
1986         unsigned int to_hash_later;
1987         unsigned int nsg;
1988         int nents;
1989         struct device *dev = ctx->dev;
1990         struct talitos_private *priv = dev_get_drvdata(dev);
1991         bool is_sec1 = has_ftr_sec1(priv);
1992         u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx];
1993
1994         if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1995                 /* Buffer up to one whole block */
1996                 nents = sg_nents_for_len(areq->src, nbytes);
1997                 if (nents < 0) {
1998                         dev_err(ctx->dev, "Invalid number of src SG.\n");
1999                         return nents;
2000                 }
2001                 sg_copy_to_buffer(areq->src, nents,
2002                                   ctx_buf + req_ctx->nbuf, nbytes);
2003                 req_ctx->nbuf += nbytes;
2004                 return 0;
2005         }
2006
2007         /* At least (blocksize + 1) bytes are available to hash */
2008         nbytes_to_hash = nbytes + req_ctx->nbuf;
2009         to_hash_later = nbytes_to_hash & (blocksize - 1);
2010
2011         if (req_ctx->last)
2012                 to_hash_later = 0;
2013         else if (to_hash_later)
2014                 /* There is a partial block. Hash the full block(s) now */
2015                 nbytes_to_hash -= to_hash_later;
2016         else {
2017                 /* Keep one block buffered */
2018                 nbytes_to_hash -= blocksize;
2019                 to_hash_later = blocksize;
2020         }
2021
2022         /* Chain in any previously buffered data */
2023         if (!is_sec1 && req_ctx->nbuf) {
2024                 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
2025                 sg_init_table(req_ctx->bufsl, nsg);
2026                 sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf);
2027                 if (nsg > 1)
2028                         sg_chain(req_ctx->bufsl, 2, areq->src);
2029                 req_ctx->psrc = req_ctx->bufsl;
2030         } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) {
2031                 int offset;
2032
2033                 if (nbytes_to_hash > blocksize)
2034                         offset = blocksize - req_ctx->nbuf;
2035                 else
2036                         offset = nbytes_to_hash - req_ctx->nbuf;
2037                 nents = sg_nents_for_len(areq->src, offset);
2038                 if (nents < 0) {
2039                         dev_err(ctx->dev, "Invalid number of src SG.\n");
2040                         return nents;
2041                 }
2042                 sg_copy_to_buffer(areq->src, nents,
2043                                   ctx_buf + req_ctx->nbuf, offset);
2044                 req_ctx->nbuf += offset;
2045                 req_ctx->psrc = scatterwalk_ffwd(req_ctx->bufsl, areq->src,
2046                                                  offset);
2047         } else
2048                 req_ctx->psrc = areq->src;
2049
2050         if (to_hash_later) {
2051                 nents = sg_nents_for_len(areq->src, nbytes);
2052                 if (nents < 0) {
2053                         dev_err(ctx->dev, "Invalid number of src SG.\n");
2054                         return nents;
2055                 }
2056                 sg_pcopy_to_buffer(areq->src, nents,
2057                                    req_ctx->buf[(req_ctx->buf_idx + 1) & 1],
2058                                       to_hash_later,
2059                                       nbytes - to_hash_later);
2060         }
2061         req_ctx->to_hash_later = to_hash_later;
2062
2063         /* Allocate extended descriptor */
2064         edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
2065         if (IS_ERR(edesc))
2066                 return PTR_ERR(edesc);
2067
2068         edesc->desc.hdr = ctx->desc_hdr_template;
2069
2070         /* On last one, request SEC to pad; otherwise continue */
2071         if (req_ctx->last)
2072                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
2073         else
2074                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
2075
2076         /* request SEC to INIT hash. */
2077         if (req_ctx->first && !req_ctx->swinit)
2078                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
2079
2080         /* When the tfm context has a keylen, it's an HMAC.
2081          * A first or last (ie. not middle) descriptor must request HMAC.
2082          */
2083         if (ctx->keylen && (req_ctx->first || req_ctx->last))
2084                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
2085
2086         return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, ahash_done);
2087 }
2088
2089 static int ahash_update(struct ahash_request *areq)
2090 {
2091         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2092
2093         req_ctx->last = 0;
2094
2095         return ahash_process_req(areq, areq->nbytes);
2096 }
2097
2098 static int ahash_final(struct ahash_request *areq)
2099 {
2100         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2101
2102         req_ctx->last = 1;
2103
2104         return ahash_process_req(areq, 0);
2105 }
2106
2107 static int ahash_finup(struct ahash_request *areq)
2108 {
2109         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2110
2111         req_ctx->last = 1;
2112
2113         return ahash_process_req(areq, areq->nbytes);
2114 }
2115
2116 static int ahash_digest(struct ahash_request *areq)
2117 {
2118         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2119         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
2120
2121         ahash->init(areq);
2122         req_ctx->last = 1;
2123
2124         return ahash_process_req(areq, areq->nbytes);
2125 }
2126
2127 static int ahash_export(struct ahash_request *areq, void *out)
2128 {
2129         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2130         struct talitos_export_state *export = out;
2131         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2132         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2133         struct device *dev = ctx->dev;
2134         dma_addr_t dma;
2135
2136         dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
2137                              DMA_FROM_DEVICE);
2138         dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_FROM_DEVICE);
2139
2140         memcpy(export->hw_context, req_ctx->hw_context,
2141                req_ctx->hw_context_size);
2142         memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf);
2143         export->swinit = req_ctx->swinit;
2144         export->first = req_ctx->first;
2145         export->last = req_ctx->last;
2146         export->to_hash_later = req_ctx->to_hash_later;
2147         export->nbuf = req_ctx->nbuf;
2148
2149         return 0;
2150 }
2151
2152 static int ahash_import(struct ahash_request *areq, const void *in)
2153 {
2154         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2155         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2156         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2157         struct device *dev = ctx->dev;
2158         const struct talitos_export_state *export = in;
2159         unsigned int size;
2160         dma_addr_t dma;
2161
2162         memset(req_ctx, 0, sizeof(*req_ctx));
2163         size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
2164                         ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
2165                         : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
2166         req_ctx->hw_context_size = size;
2167         memcpy(req_ctx->hw_context, export->hw_context, size);
2168         memcpy(req_ctx->buf[0], export->buf, export->nbuf);
2169         req_ctx->swinit = export->swinit;
2170         req_ctx->first = export->first;
2171         req_ctx->last = export->last;
2172         req_ctx->to_hash_later = export->to_hash_later;
2173         req_ctx->nbuf = export->nbuf;
2174
2175         dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
2176                              DMA_TO_DEVICE);
2177         dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE);
2178
2179         return 0;
2180 }
2181
2182 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2183                    u8 *hash)
2184 {
2185         struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2186
2187         struct scatterlist sg[1];
2188         struct ahash_request *req;
2189         struct crypto_wait wait;
2190         int ret;
2191
2192         crypto_init_wait(&wait);
2193
2194         req = ahash_request_alloc(tfm, GFP_KERNEL);
2195         if (!req)
2196                 return -ENOMEM;
2197
2198         /* Keep tfm keylen == 0 during hash of the long key */
2199         ctx->keylen = 0;
2200         ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2201                                    crypto_req_done, &wait);
2202
2203         sg_init_one(&sg[0], key, keylen);
2204
2205         ahash_request_set_crypt(req, sg, hash, keylen);
2206         ret = crypto_wait_req(crypto_ahash_digest(req), &wait);
2207
2208         ahash_request_free(req);
2209
2210         return ret;
2211 }
2212
2213 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2214                         unsigned int keylen)
2215 {
2216         struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2217         struct device *dev = ctx->dev;
2218         unsigned int blocksize =
2219                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2220         unsigned int digestsize = crypto_ahash_digestsize(tfm);
2221         unsigned int keysize = keylen;
2222         u8 hash[SHA512_DIGEST_SIZE];
2223         int ret;
2224
2225         if (keylen <= blocksize)
2226                 memcpy(ctx->key, key, keysize);
2227         else {
2228                 /* Must get the hash of the long key */
2229                 ret = keyhash(tfm, key, keylen, hash);
2230
2231                 if (ret)
2232                         return -EINVAL;
2233
2234                 keysize = digestsize;
2235                 memcpy(ctx->key, hash, digestsize);
2236         }
2237
2238         if (ctx->keylen)
2239                 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
2240
2241         ctx->keylen = keysize;
2242         ctx->dma_key = dma_map_single(dev, ctx->key, keysize, DMA_TO_DEVICE);
2243
2244         return 0;
2245 }
2246
2247
2248 struct talitos_alg_template {
2249         u32 type;
2250         u32 priority;
2251         union {
2252                 struct skcipher_alg skcipher;
2253                 struct ahash_alg hash;
2254                 struct aead_alg aead;
2255         } alg;
2256         __be32 desc_hdr_template;
2257 };
2258
2259 static struct talitos_alg_template driver_algs[] = {
2260         /* AEAD algorithms.  These use a single-pass ipsec_esp descriptor */
2261         {       .type = CRYPTO_ALG_TYPE_AEAD,
2262                 .alg.aead = {
2263                         .base = {
2264                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2265                                 .cra_driver_name = "authenc-hmac-sha1-"
2266                                                    "cbc-aes-talitos",
2267                                 .cra_blocksize = AES_BLOCK_SIZE,
2268                                 .cra_flags = CRYPTO_ALG_ASYNC |
2269                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2270                         },
2271                         .ivsize = AES_BLOCK_SIZE,
2272                         .maxauthsize = SHA1_DIGEST_SIZE,
2273                 },
2274                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2275                                      DESC_HDR_SEL0_AESU |
2276                                      DESC_HDR_MODE0_AESU_CBC |
2277                                      DESC_HDR_SEL1_MDEUA |
2278                                      DESC_HDR_MODE1_MDEU_INIT |
2279                                      DESC_HDR_MODE1_MDEU_PAD |
2280                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2281         },
2282         {       .type = CRYPTO_ALG_TYPE_AEAD,
2283                 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2284                 .alg.aead = {
2285                         .base = {
2286                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2287                                 .cra_driver_name = "authenc-hmac-sha1-"
2288                                                    "cbc-aes-talitos-hsna",
2289                                 .cra_blocksize = AES_BLOCK_SIZE,
2290                                 .cra_flags = CRYPTO_ALG_ASYNC |
2291                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2292                         },
2293                         .ivsize = AES_BLOCK_SIZE,
2294                         .maxauthsize = SHA1_DIGEST_SIZE,
2295                 },
2296                 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2297                                      DESC_HDR_SEL0_AESU |
2298                                      DESC_HDR_MODE0_AESU_CBC |
2299                                      DESC_HDR_SEL1_MDEUA |
2300                                      DESC_HDR_MODE1_MDEU_INIT |
2301                                      DESC_HDR_MODE1_MDEU_PAD |
2302                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2303         },
2304         {       .type = CRYPTO_ALG_TYPE_AEAD,
2305                 .alg.aead = {
2306                         .base = {
2307                                 .cra_name = "authenc(hmac(sha1),"
2308                                             "cbc(des3_ede))",
2309                                 .cra_driver_name = "authenc-hmac-sha1-"
2310                                                    "cbc-3des-talitos",
2311                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2312                                 .cra_flags = CRYPTO_ALG_ASYNC |
2313                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2314                         },
2315                         .ivsize = DES3_EDE_BLOCK_SIZE,
2316                         .maxauthsize = SHA1_DIGEST_SIZE,
2317                         .setkey = aead_des3_setkey,
2318                 },
2319                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2320                                      DESC_HDR_SEL0_DEU |
2321                                      DESC_HDR_MODE0_DEU_CBC |
2322                                      DESC_HDR_MODE0_DEU_3DES |
2323                                      DESC_HDR_SEL1_MDEUA |
2324                                      DESC_HDR_MODE1_MDEU_INIT |
2325                                      DESC_HDR_MODE1_MDEU_PAD |
2326                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2327         },
2328         {       .type = CRYPTO_ALG_TYPE_AEAD,
2329                 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2330                 .alg.aead = {
2331                         .base = {
2332                                 .cra_name = "authenc(hmac(sha1),"
2333                                             "cbc(des3_ede))",
2334                                 .cra_driver_name = "authenc-hmac-sha1-"
2335                                                    "cbc-3des-talitos-hsna",
2336                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2337                                 .cra_flags = CRYPTO_ALG_ASYNC |
2338                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2339                         },
2340                         .ivsize = DES3_EDE_BLOCK_SIZE,
2341                         .maxauthsize = SHA1_DIGEST_SIZE,
2342                         .setkey = aead_des3_setkey,
2343                 },
2344                 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2345                                      DESC_HDR_SEL0_DEU |
2346                                      DESC_HDR_MODE0_DEU_CBC |
2347                                      DESC_HDR_MODE0_DEU_3DES |
2348                                      DESC_HDR_SEL1_MDEUA |
2349                                      DESC_HDR_MODE1_MDEU_INIT |
2350                                      DESC_HDR_MODE1_MDEU_PAD |
2351                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2352         },
2353         {       .type = CRYPTO_ALG_TYPE_AEAD,
2354                 .alg.aead = {
2355                         .base = {
2356                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2357                                 .cra_driver_name = "authenc-hmac-sha224-"
2358                                                    "cbc-aes-talitos",
2359                                 .cra_blocksize = AES_BLOCK_SIZE,
2360                                 .cra_flags = CRYPTO_ALG_ASYNC |
2361                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2362                         },
2363                         .ivsize = AES_BLOCK_SIZE,
2364                         .maxauthsize = SHA224_DIGEST_SIZE,
2365                 },
2366                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2367                                      DESC_HDR_SEL0_AESU |
2368                                      DESC_HDR_MODE0_AESU_CBC |
2369                                      DESC_HDR_SEL1_MDEUA |
2370                                      DESC_HDR_MODE1_MDEU_INIT |
2371                                      DESC_HDR_MODE1_MDEU_PAD |
2372                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2373         },
2374         {       .type = CRYPTO_ALG_TYPE_AEAD,
2375                 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2376                 .alg.aead = {
2377                         .base = {
2378                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2379                                 .cra_driver_name = "authenc-hmac-sha224-"
2380                                                    "cbc-aes-talitos-hsna",
2381                                 .cra_blocksize = AES_BLOCK_SIZE,
2382                                 .cra_flags = CRYPTO_ALG_ASYNC |
2383                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2384                         },
2385                         .ivsize = AES_BLOCK_SIZE,
2386                         .maxauthsize = SHA224_DIGEST_SIZE,
2387                 },
2388                 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2389                                      DESC_HDR_SEL0_AESU |
2390                                      DESC_HDR_MODE0_AESU_CBC |
2391                                      DESC_HDR_SEL1_MDEUA |
2392                                      DESC_HDR_MODE1_MDEU_INIT |
2393                                      DESC_HDR_MODE1_MDEU_PAD |
2394                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2395         },
2396         {       .type = CRYPTO_ALG_TYPE_AEAD,
2397                 .alg.aead = {
2398                         .base = {
2399                                 .cra_name = "authenc(hmac(sha224),"
2400                                             "cbc(des3_ede))",
2401                                 .cra_driver_name = "authenc-hmac-sha224-"
2402                                                    "cbc-3des-talitos",
2403                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2404                                 .cra_flags = CRYPTO_ALG_ASYNC |
2405                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2406                         },
2407                         .ivsize = DES3_EDE_BLOCK_SIZE,
2408                         .maxauthsize = SHA224_DIGEST_SIZE,
2409                         .setkey = aead_des3_setkey,
2410                 },
2411                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2412                                      DESC_HDR_SEL0_DEU |
2413                                      DESC_HDR_MODE0_DEU_CBC |
2414                                      DESC_HDR_MODE0_DEU_3DES |
2415                                      DESC_HDR_SEL1_MDEUA |
2416                                      DESC_HDR_MODE1_MDEU_INIT |
2417                                      DESC_HDR_MODE1_MDEU_PAD |
2418                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2419         },
2420         {       .type = CRYPTO_ALG_TYPE_AEAD,
2421                 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2422                 .alg.aead = {
2423                         .base = {
2424                                 .cra_name = "authenc(hmac(sha224),"
2425                                             "cbc(des3_ede))",
2426                                 .cra_driver_name = "authenc-hmac-sha224-"
2427                                                    "cbc-3des-talitos-hsna",
2428                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2429                                 .cra_flags = CRYPTO_ALG_ASYNC |
2430                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2431                         },
2432                         .ivsize = DES3_EDE_BLOCK_SIZE,
2433                         .maxauthsize = SHA224_DIGEST_SIZE,
2434                         .setkey = aead_des3_setkey,
2435                 },
2436                 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2437                                      DESC_HDR_SEL0_DEU |
2438                                      DESC_HDR_MODE0_DEU_CBC |
2439                                      DESC_HDR_MODE0_DEU_3DES |
2440                                      DESC_HDR_SEL1_MDEUA |
2441                                      DESC_HDR_MODE1_MDEU_INIT |
2442                                      DESC_HDR_MODE1_MDEU_PAD |
2443                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2444         },
2445         {       .type = CRYPTO_ALG_TYPE_AEAD,
2446                 .alg.aead = {
2447                         .base = {
2448                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2449                                 .cra_driver_name = "authenc-hmac-sha256-"
2450                                                    "cbc-aes-talitos",
2451                                 .cra_blocksize = AES_BLOCK_SIZE,
2452                                 .cra_flags = CRYPTO_ALG_ASYNC |
2453                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2454                         },
2455                         .ivsize = AES_BLOCK_SIZE,
2456                         .maxauthsize = SHA256_DIGEST_SIZE,
2457                 },
2458                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2459                                      DESC_HDR_SEL0_AESU |
2460                                      DESC_HDR_MODE0_AESU_CBC |
2461                                      DESC_HDR_SEL1_MDEUA |
2462                                      DESC_HDR_MODE1_MDEU_INIT |
2463                                      DESC_HDR_MODE1_MDEU_PAD |
2464                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2465         },
2466         {       .type = CRYPTO_ALG_TYPE_AEAD,
2467                 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2468                 .alg.aead = {
2469                         .base = {
2470                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2471                                 .cra_driver_name = "authenc-hmac-sha256-"
2472                                                    "cbc-aes-talitos-hsna",
2473                                 .cra_blocksize = AES_BLOCK_SIZE,
2474                                 .cra_flags = CRYPTO_ALG_ASYNC |
2475                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2476                         },
2477                         .ivsize = AES_BLOCK_SIZE,
2478                         .maxauthsize = SHA256_DIGEST_SIZE,
2479                 },
2480                 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2481                                      DESC_HDR_SEL0_AESU |
2482                                      DESC_HDR_MODE0_AESU_CBC |
2483                                      DESC_HDR_SEL1_MDEUA |
2484                                      DESC_HDR_MODE1_MDEU_INIT |
2485                                      DESC_HDR_MODE1_MDEU_PAD |
2486                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2487         },
2488         {       .type = CRYPTO_ALG_TYPE_AEAD,
2489                 .alg.aead = {
2490                         .base = {
2491                                 .cra_name = "authenc(hmac(sha256),"
2492                                             "cbc(des3_ede))",
2493                                 .cra_driver_name = "authenc-hmac-sha256-"
2494                                                    "cbc-3des-talitos",
2495                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2496                                 .cra_flags = CRYPTO_ALG_ASYNC |
2497                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2498                         },
2499                         .ivsize = DES3_EDE_BLOCK_SIZE,
2500                         .maxauthsize = SHA256_DIGEST_SIZE,
2501                         .setkey = aead_des3_setkey,
2502                 },
2503                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2504                                      DESC_HDR_SEL0_DEU |
2505                                      DESC_HDR_MODE0_DEU_CBC |
2506                                      DESC_HDR_MODE0_DEU_3DES |
2507                                      DESC_HDR_SEL1_MDEUA |
2508                                      DESC_HDR_MODE1_MDEU_INIT |
2509                                      DESC_HDR_MODE1_MDEU_PAD |
2510                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2511         },
2512         {       .type = CRYPTO_ALG_TYPE_AEAD,
2513                 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2514                 .alg.aead = {
2515                         .base = {
2516                                 .cra_name = "authenc(hmac(sha256),"
2517                                             "cbc(des3_ede))",
2518                                 .cra_driver_name = "authenc-hmac-sha256-"
2519                                                    "cbc-3des-talitos-hsna",
2520                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2521                                 .cra_flags = CRYPTO_ALG_ASYNC |
2522                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2523                         },
2524                         .ivsize = DES3_EDE_BLOCK_SIZE,
2525                         .maxauthsize = SHA256_DIGEST_SIZE,
2526                         .setkey = aead_des3_setkey,
2527                 },
2528                 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2529                                      DESC_HDR_SEL0_DEU |
2530                                      DESC_HDR_MODE0_DEU_CBC |
2531                                      DESC_HDR_MODE0_DEU_3DES |
2532                                      DESC_HDR_SEL1_MDEUA |
2533                                      DESC_HDR_MODE1_MDEU_INIT |
2534                                      DESC_HDR_MODE1_MDEU_PAD |
2535                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2536         },
2537         {       .type = CRYPTO_ALG_TYPE_AEAD,
2538                 .alg.aead = {
2539                         .base = {
2540                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2541                                 .cra_driver_name = "authenc-hmac-sha384-"
2542                                                    "cbc-aes-talitos",
2543                                 .cra_blocksize = AES_BLOCK_SIZE,
2544                                 .cra_flags = CRYPTO_ALG_ASYNC |
2545                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2546                         },
2547                         .ivsize = AES_BLOCK_SIZE,
2548                         .maxauthsize = SHA384_DIGEST_SIZE,
2549                 },
2550                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2551                                      DESC_HDR_SEL0_AESU |
2552                                      DESC_HDR_MODE0_AESU_CBC |
2553                                      DESC_HDR_SEL1_MDEUB |
2554                                      DESC_HDR_MODE1_MDEU_INIT |
2555                                      DESC_HDR_MODE1_MDEU_PAD |
2556                                      DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2557         },
2558         {       .type = CRYPTO_ALG_TYPE_AEAD,
2559                 .alg.aead = {
2560                         .base = {
2561                                 .cra_name = "authenc(hmac(sha384),"
2562                                             "cbc(des3_ede))",
2563                                 .cra_driver_name = "authenc-hmac-sha384-"
2564                                                    "cbc-3des-talitos",
2565                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2566                                 .cra_flags = CRYPTO_ALG_ASYNC |
2567                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2568                         },
2569                         .ivsize = DES3_EDE_BLOCK_SIZE,
2570                         .maxauthsize = SHA384_DIGEST_SIZE,
2571                         .setkey = aead_des3_setkey,
2572                 },
2573                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2574                                      DESC_HDR_SEL0_DEU |
2575                                      DESC_HDR_MODE0_DEU_CBC |
2576                                      DESC_HDR_MODE0_DEU_3DES |
2577                                      DESC_HDR_SEL1_MDEUB |
2578                                      DESC_HDR_MODE1_MDEU_INIT |
2579                                      DESC_HDR_MODE1_MDEU_PAD |
2580                                      DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2581         },
2582         {       .type = CRYPTO_ALG_TYPE_AEAD,
2583                 .alg.aead = {
2584                         .base = {
2585                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2586                                 .cra_driver_name = "authenc-hmac-sha512-"
2587                                                    "cbc-aes-talitos",
2588                                 .cra_blocksize = AES_BLOCK_SIZE,
2589                                 .cra_flags = CRYPTO_ALG_ASYNC |
2590                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2591                         },
2592                         .ivsize = AES_BLOCK_SIZE,
2593                         .maxauthsize = SHA512_DIGEST_SIZE,
2594                 },
2595                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2596                                      DESC_HDR_SEL0_AESU |
2597                                      DESC_HDR_MODE0_AESU_CBC |
2598                                      DESC_HDR_SEL1_MDEUB |
2599                                      DESC_HDR_MODE1_MDEU_INIT |
2600                                      DESC_HDR_MODE1_MDEU_PAD |
2601                                      DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2602         },
2603         {       .type = CRYPTO_ALG_TYPE_AEAD,
2604                 .alg.aead = {
2605                         .base = {
2606                                 .cra_name = "authenc(hmac(sha512),"
2607                                             "cbc(des3_ede))",
2608                                 .cra_driver_name = "authenc-hmac-sha512-"
2609                                                    "cbc-3des-talitos",
2610                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2611                                 .cra_flags = CRYPTO_ALG_ASYNC |
2612                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2613                         },
2614                         .ivsize = DES3_EDE_BLOCK_SIZE,
2615                         .maxauthsize = SHA512_DIGEST_SIZE,
2616                         .setkey = aead_des3_setkey,
2617                 },
2618                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2619                                      DESC_HDR_SEL0_DEU |
2620                                      DESC_HDR_MODE0_DEU_CBC |
2621                                      DESC_HDR_MODE0_DEU_3DES |
2622                                      DESC_HDR_SEL1_MDEUB |
2623                                      DESC_HDR_MODE1_MDEU_INIT |
2624                                      DESC_HDR_MODE1_MDEU_PAD |
2625                                      DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2626         },
2627         {       .type = CRYPTO_ALG_TYPE_AEAD,
2628                 .alg.aead = {
2629                         .base = {
2630                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
2631                                 .cra_driver_name = "authenc-hmac-md5-"
2632                                                    "cbc-aes-talitos",
2633                                 .cra_blocksize = AES_BLOCK_SIZE,
2634                                 .cra_flags = CRYPTO_ALG_ASYNC |
2635                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2636                         },
2637                         .ivsize = AES_BLOCK_SIZE,
2638                         .maxauthsize = MD5_DIGEST_SIZE,
2639                 },
2640                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2641                                      DESC_HDR_SEL0_AESU |
2642                                      DESC_HDR_MODE0_AESU_CBC |
2643                                      DESC_HDR_SEL1_MDEUA |
2644                                      DESC_HDR_MODE1_MDEU_INIT |
2645                                      DESC_HDR_MODE1_MDEU_PAD |
2646                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2647         },
2648         {       .type = CRYPTO_ALG_TYPE_AEAD,
2649                 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2650                 .alg.aead = {
2651                         .base = {
2652                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
2653                                 .cra_driver_name = "authenc-hmac-md5-"
2654                                                    "cbc-aes-talitos-hsna",
2655                                 .cra_blocksize = AES_BLOCK_SIZE,
2656                                 .cra_flags = CRYPTO_ALG_ASYNC |
2657                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2658                         },
2659                         .ivsize = AES_BLOCK_SIZE,
2660                         .maxauthsize = MD5_DIGEST_SIZE,
2661                 },
2662                 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2663                                      DESC_HDR_SEL0_AESU |
2664                                      DESC_HDR_MODE0_AESU_CBC |
2665                                      DESC_HDR_SEL1_MDEUA |
2666                                      DESC_HDR_MODE1_MDEU_INIT |
2667                                      DESC_HDR_MODE1_MDEU_PAD |
2668                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2669         },
2670         {       .type = CRYPTO_ALG_TYPE_AEAD,
2671                 .alg.aead = {
2672                         .base = {
2673                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2674                                 .cra_driver_name = "authenc-hmac-md5-"
2675                                                    "cbc-3des-talitos",
2676                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2677                                 .cra_flags = CRYPTO_ALG_ASYNC |
2678                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2679                         },
2680                         .ivsize = DES3_EDE_BLOCK_SIZE,
2681                         .maxauthsize = MD5_DIGEST_SIZE,
2682                         .setkey = aead_des3_setkey,
2683                 },
2684                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2685                                      DESC_HDR_SEL0_DEU |
2686                                      DESC_HDR_MODE0_DEU_CBC |
2687                                      DESC_HDR_MODE0_DEU_3DES |
2688                                      DESC_HDR_SEL1_MDEUA |
2689                                      DESC_HDR_MODE1_MDEU_INIT |
2690                                      DESC_HDR_MODE1_MDEU_PAD |
2691                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2692         },
2693         {       .type = CRYPTO_ALG_TYPE_AEAD,
2694                 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2695                 .alg.aead = {
2696                         .base = {
2697                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2698                                 .cra_driver_name = "authenc-hmac-md5-"
2699                                                    "cbc-3des-talitos-hsna",
2700                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2701                                 .cra_flags = CRYPTO_ALG_ASYNC |
2702                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2703                         },
2704                         .ivsize = DES3_EDE_BLOCK_SIZE,
2705                         .maxauthsize = MD5_DIGEST_SIZE,
2706                         .setkey = aead_des3_setkey,
2707                 },
2708                 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2709                                      DESC_HDR_SEL0_DEU |
2710                                      DESC_HDR_MODE0_DEU_CBC |
2711                                      DESC_HDR_MODE0_DEU_3DES |
2712                                      DESC_HDR_SEL1_MDEUA |
2713                                      DESC_HDR_MODE1_MDEU_INIT |
2714                                      DESC_HDR_MODE1_MDEU_PAD |
2715                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2716         },
2717         /* SKCIPHER algorithms. */
2718         {       .type = CRYPTO_ALG_TYPE_SKCIPHER,
2719                 .alg.skcipher = {
2720                         .base.cra_name = "ecb(aes)",
2721                         .base.cra_driver_name = "ecb-aes-talitos",
2722                         .base.cra_blocksize = AES_BLOCK_SIZE,
2723                         .base.cra_flags = CRYPTO_ALG_ASYNC |
2724                                           CRYPTO_ALG_ALLOCATES_MEMORY,
2725                         .min_keysize = AES_MIN_KEY_SIZE,
2726                         .max_keysize = AES_MAX_KEY_SIZE,
2727                         .setkey = skcipher_aes_setkey,
2728                 },
2729                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2730                                      DESC_HDR_SEL0_AESU,
2731         },
2732         {       .type = CRYPTO_ALG_TYPE_SKCIPHER,
2733                 .alg.skcipher = {
2734                         .base.cra_name = "cbc(aes)",
2735                         .base.cra_driver_name = "cbc-aes-talitos",
2736                         .base.cra_blocksize = AES_BLOCK_SIZE,
2737                         .base.cra_flags = CRYPTO_ALG_ASYNC |
2738                                           CRYPTO_ALG_ALLOCATES_MEMORY,
2739                         .min_keysize = AES_MIN_KEY_SIZE,
2740                         .max_keysize = AES_MAX_KEY_SIZE,
2741                         .ivsize = AES_BLOCK_SIZE,
2742                         .setkey = skcipher_aes_setkey,
2743                 },
2744                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2745                                      DESC_HDR_SEL0_AESU |
2746                                      DESC_HDR_MODE0_AESU_CBC,
2747         },
2748         {       .type = CRYPTO_ALG_TYPE_SKCIPHER,
2749                 .alg.skcipher = {
2750                         .base.cra_name = "ctr(aes)",
2751                         .base.cra_driver_name = "ctr-aes-talitos",
2752                         .base.cra_blocksize = 1,
2753                         .base.cra_flags = CRYPTO_ALG_ASYNC |
2754                                           CRYPTO_ALG_ALLOCATES_MEMORY,
2755                         .min_keysize = AES_MIN_KEY_SIZE,
2756                         .max_keysize = AES_MAX_KEY_SIZE,
2757                         .ivsize = AES_BLOCK_SIZE,
2758                         .setkey = skcipher_aes_setkey,
2759                 },
2760                 .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP |
2761                                      DESC_HDR_SEL0_AESU |
2762                                      DESC_HDR_MODE0_AESU_CTR,
2763         },
2764         {       .type = CRYPTO_ALG_TYPE_SKCIPHER,
2765                 .alg.skcipher = {
2766                         .base.cra_name = "ecb(des)",
2767                         .base.cra_driver_name = "ecb-des-talitos",
2768                         .base.cra_blocksize = DES_BLOCK_SIZE,
2769                         .base.cra_flags = CRYPTO_ALG_ASYNC |
2770                                           CRYPTO_ALG_ALLOCATES_MEMORY,
2771                         .min_keysize = DES_KEY_SIZE,
2772                         .max_keysize = DES_KEY_SIZE,
2773                         .setkey = skcipher_des_setkey,
2774                 },
2775                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2776                                      DESC_HDR_SEL0_DEU,
2777         },
2778         {       .type = CRYPTO_ALG_TYPE_SKCIPHER,
2779                 .alg.skcipher = {
2780                         .base.cra_name = "cbc(des)",
2781                         .base.cra_driver_name = "cbc-des-talitos",
2782                         .base.cra_blocksize = DES_BLOCK_SIZE,
2783                         .base.cra_flags = CRYPTO_ALG_ASYNC |
2784                                           CRYPTO_ALG_ALLOCATES_MEMORY,
2785                         .min_keysize = DES_KEY_SIZE,
2786                         .max_keysize = DES_KEY_SIZE,
2787                         .ivsize = DES_BLOCK_SIZE,
2788                         .setkey = skcipher_des_setkey,
2789                 },
2790                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2791                                      DESC_HDR_SEL0_DEU |
2792                                      DESC_HDR_MODE0_DEU_CBC,
2793         },
2794         {       .type = CRYPTO_ALG_TYPE_SKCIPHER,
2795                 .alg.skcipher = {
2796                         .base.cra_name = "ecb(des3_ede)",
2797                         .base.cra_driver_name = "ecb-3des-talitos",
2798                         .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2799                         .base.cra_flags = CRYPTO_ALG_ASYNC |
2800                                           CRYPTO_ALG_ALLOCATES_MEMORY,
2801                         .min_keysize = DES3_EDE_KEY_SIZE,
2802                         .max_keysize = DES3_EDE_KEY_SIZE,
2803                         .setkey = skcipher_des3_setkey,
2804                 },
2805                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2806                                      DESC_HDR_SEL0_DEU |
2807                                      DESC_HDR_MODE0_DEU_3DES,
2808         },
2809         {       .type = CRYPTO_ALG_TYPE_SKCIPHER,
2810                 .alg.skcipher = {
2811                         .base.cra_name = "cbc(des3_ede)",
2812                         .base.cra_driver_name = "cbc-3des-talitos",
2813                         .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2814                         .base.cra_flags = CRYPTO_ALG_ASYNC |
2815                                           CRYPTO_ALG_ALLOCATES_MEMORY,
2816                         .min_keysize = DES3_EDE_KEY_SIZE,
2817                         .max_keysize = DES3_EDE_KEY_SIZE,
2818                         .ivsize = DES3_EDE_BLOCK_SIZE,
2819                         .setkey = skcipher_des3_setkey,
2820                 },
2821                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2822                                      DESC_HDR_SEL0_DEU |
2823                                      DESC_HDR_MODE0_DEU_CBC |
2824                                      DESC_HDR_MODE0_DEU_3DES,
2825         },
2826         /* AHASH algorithms. */
2827         {       .type = CRYPTO_ALG_TYPE_AHASH,
2828                 .alg.hash = {
2829                         .halg.digestsize = MD5_DIGEST_SIZE,
2830                         .halg.statesize = sizeof(struct talitos_export_state),
2831                         .halg.base = {
2832                                 .cra_name = "md5",
2833                                 .cra_driver_name = "md5-talitos",
2834                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2835                                 .cra_flags = CRYPTO_ALG_ASYNC |
2836                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2837                         }
2838                 },
2839                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2840                                      DESC_HDR_SEL0_MDEUA |
2841                                      DESC_HDR_MODE0_MDEU_MD5,
2842         },
2843         {       .type = CRYPTO_ALG_TYPE_AHASH,
2844                 .alg.hash = {
2845                         .halg.digestsize = SHA1_DIGEST_SIZE,
2846                         .halg.statesize = sizeof(struct talitos_export_state),
2847                         .halg.base = {
2848                                 .cra_name = "sha1",
2849                                 .cra_driver_name = "sha1-talitos",
2850                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2851                                 .cra_flags = CRYPTO_ALG_ASYNC |
2852                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2853                         }
2854                 },
2855                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2856                                      DESC_HDR_SEL0_MDEUA |
2857                                      DESC_HDR_MODE0_MDEU_SHA1,
2858         },
2859         {       .type = CRYPTO_ALG_TYPE_AHASH,
2860                 .alg.hash = {
2861                         .halg.digestsize = SHA224_DIGEST_SIZE,
2862                         .halg.statesize = sizeof(struct talitos_export_state),
2863                         .halg.base = {
2864                                 .cra_name = "sha224",
2865                                 .cra_driver_name = "sha224-talitos",
2866                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2867                                 .cra_flags = CRYPTO_ALG_ASYNC |
2868                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2869                         }
2870                 },
2871                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2872                                      DESC_HDR_SEL0_MDEUA |
2873                                      DESC_HDR_MODE0_MDEU_SHA224,
2874         },
2875         {       .type = CRYPTO_ALG_TYPE_AHASH,
2876                 .alg.hash = {
2877                         .halg.digestsize = SHA256_DIGEST_SIZE,
2878                         .halg.statesize = sizeof(struct talitos_export_state),
2879                         .halg.base = {
2880                                 .cra_name = "sha256",
2881                                 .cra_driver_name = "sha256-talitos",
2882                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2883                                 .cra_flags = CRYPTO_ALG_ASYNC |
2884                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2885                         }
2886                 },
2887                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2888                                      DESC_HDR_SEL0_MDEUA |
2889                                      DESC_HDR_MODE0_MDEU_SHA256,
2890         },
2891         {       .type = CRYPTO_ALG_TYPE_AHASH,
2892                 .alg.hash = {
2893                         .halg.digestsize = SHA384_DIGEST_SIZE,
2894                         .halg.statesize = sizeof(struct talitos_export_state),
2895                         .halg.base = {
2896                                 .cra_name = "sha384",
2897                                 .cra_driver_name = "sha384-talitos",
2898                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2899                                 .cra_flags = CRYPTO_ALG_ASYNC |
2900                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2901                         }
2902                 },
2903                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2904                                      DESC_HDR_SEL0_MDEUB |
2905                                      DESC_HDR_MODE0_MDEUB_SHA384,
2906         },
2907         {       .type = CRYPTO_ALG_TYPE_AHASH,
2908                 .alg.hash = {
2909                         .halg.digestsize = SHA512_DIGEST_SIZE,
2910                         .halg.statesize = sizeof(struct talitos_export_state),
2911                         .halg.base = {
2912                                 .cra_name = "sha512",
2913                                 .cra_driver_name = "sha512-talitos",
2914                                 .cra_blocksize = SHA512_BLOCK_SIZE,
2915                                 .cra_flags = CRYPTO_ALG_ASYNC |
2916                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2917                         }
2918                 },
2919                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2920                                      DESC_HDR_SEL0_MDEUB |
2921                                      DESC_HDR_MODE0_MDEUB_SHA512,
2922         },
2923         {       .type = CRYPTO_ALG_TYPE_AHASH,
2924                 .alg.hash = {
2925                         .halg.digestsize = MD5_DIGEST_SIZE,
2926                         .halg.statesize = sizeof(struct talitos_export_state),
2927                         .halg.base = {
2928                                 .cra_name = "hmac(md5)",
2929                                 .cra_driver_name = "hmac-md5-talitos",
2930                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2931                                 .cra_flags = CRYPTO_ALG_ASYNC |
2932                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2933                         }
2934                 },
2935                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2936                                      DESC_HDR_SEL0_MDEUA |
2937                                      DESC_HDR_MODE0_MDEU_MD5,
2938         },
2939         {       .type = CRYPTO_ALG_TYPE_AHASH,
2940                 .alg.hash = {
2941                         .halg.digestsize = SHA1_DIGEST_SIZE,
2942                         .halg.statesize = sizeof(struct talitos_export_state),
2943                         .halg.base = {
2944                                 .cra_name = "hmac(sha1)",
2945                                 .cra_driver_name = "hmac-sha1-talitos",
2946                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2947                                 .cra_flags = CRYPTO_ALG_ASYNC |
2948                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2949                         }
2950                 },
2951                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2952                                      DESC_HDR_SEL0_MDEUA |
2953                                      DESC_HDR_MODE0_MDEU_SHA1,
2954         },
2955         {       .type = CRYPTO_ALG_TYPE_AHASH,
2956                 .alg.hash = {
2957                         .halg.digestsize = SHA224_DIGEST_SIZE,
2958                         .halg.statesize = sizeof(struct talitos_export_state),
2959                         .halg.base = {
2960                                 .cra_name = "hmac(sha224)",
2961                                 .cra_driver_name = "hmac-sha224-talitos",
2962                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2963                                 .cra_flags = CRYPTO_ALG_ASYNC |
2964                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2965                         }
2966                 },
2967                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2968                                      DESC_HDR_SEL0_MDEUA |
2969                                      DESC_HDR_MODE0_MDEU_SHA224,
2970         },
2971         {       .type = CRYPTO_ALG_TYPE_AHASH,
2972                 .alg.hash = {
2973                         .halg.digestsize = SHA256_DIGEST_SIZE,
2974                         .halg.statesize = sizeof(struct talitos_export_state),
2975                         .halg.base = {
2976                                 .cra_name = "hmac(sha256)",
2977                                 .cra_driver_name = "hmac-sha256-talitos",
2978                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2979                                 .cra_flags = CRYPTO_ALG_ASYNC |
2980                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2981                         }
2982                 },
2983                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2984                                      DESC_HDR_SEL0_MDEUA |
2985                                      DESC_HDR_MODE0_MDEU_SHA256,
2986         },
2987         {       .type = CRYPTO_ALG_TYPE_AHASH,
2988                 .alg.hash = {
2989                         .halg.digestsize = SHA384_DIGEST_SIZE,
2990                         .halg.statesize = sizeof(struct talitos_export_state),
2991                         .halg.base = {
2992                                 .cra_name = "hmac(sha384)",
2993                                 .cra_driver_name = "hmac-sha384-talitos",
2994                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2995                                 .cra_flags = CRYPTO_ALG_ASYNC |
2996                                              CRYPTO_ALG_ALLOCATES_MEMORY,
2997                         }
2998                 },
2999                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3000                                      DESC_HDR_SEL0_MDEUB |
3001                                      DESC_HDR_MODE0_MDEUB_SHA384,
3002         },
3003         {       .type = CRYPTO_ALG_TYPE_AHASH,
3004                 .alg.hash = {
3005                         .halg.digestsize = SHA512_DIGEST_SIZE,
3006                         .halg.statesize = sizeof(struct talitos_export_state),
3007                         .halg.base = {
3008                                 .cra_name = "hmac(sha512)",
3009                                 .cra_driver_name = "hmac-sha512-talitos",
3010                                 .cra_blocksize = SHA512_BLOCK_SIZE,
3011                                 .cra_flags = CRYPTO_ALG_ASYNC |
3012                                              CRYPTO_ALG_ALLOCATES_MEMORY,
3013                         }
3014                 },
3015                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3016                                      DESC_HDR_SEL0_MDEUB |
3017                                      DESC_HDR_MODE0_MDEUB_SHA512,
3018         }
3019 };
3020
3021 struct talitos_crypto_alg {
3022         struct list_head entry;
3023         struct device *dev;
3024         struct talitos_alg_template algt;
3025 };
3026
3027 static int talitos_init_common(struct talitos_ctx *ctx,
3028                                struct talitos_crypto_alg *talitos_alg)
3029 {
3030         struct talitos_private *priv;
3031
3032         /* update context with ptr to dev */
3033         ctx->dev = talitos_alg->dev;
3034
3035         /* assign SEC channel to tfm in round-robin fashion */
3036         priv = dev_get_drvdata(ctx->dev);
3037         ctx->ch = atomic_inc_return(&priv->last_chan) &
3038                   (priv->num_channels - 1);
3039
3040         /* copy descriptor header template value */
3041         ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
3042
3043         /* select done notification */
3044         ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
3045
3046         return 0;
3047 }
3048
3049 static int talitos_cra_init_aead(struct crypto_aead *tfm)
3050 {
3051         struct aead_alg *alg = crypto_aead_alg(tfm);
3052         struct talitos_crypto_alg *talitos_alg;
3053         struct talitos_ctx *ctx = crypto_aead_ctx(tfm);
3054
3055         talitos_alg = container_of(alg, struct talitos_crypto_alg,
3056                                    algt.alg.aead);
3057
3058         return talitos_init_common(ctx, talitos_alg);
3059 }
3060
3061 static int talitos_cra_init_skcipher(struct crypto_skcipher *tfm)
3062 {
3063         struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
3064         struct talitos_crypto_alg *talitos_alg;
3065         struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm);
3066
3067         talitos_alg = container_of(alg, struct talitos_crypto_alg,
3068                                    algt.alg.skcipher);
3069
3070         return talitos_init_common(ctx, talitos_alg);
3071 }
3072
3073 static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
3074 {
3075         struct crypto_alg *alg = tfm->__crt_alg;
3076         struct talitos_crypto_alg *talitos_alg;
3077         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3078
3079         talitos_alg = container_of(__crypto_ahash_alg(alg),
3080                                    struct talitos_crypto_alg,
3081                                    algt.alg.hash);
3082
3083         ctx->keylen = 0;
3084         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
3085                                  sizeof(struct talitos_ahash_req_ctx));
3086
3087         return talitos_init_common(ctx, talitos_alg);
3088 }
3089
3090 static void talitos_cra_exit(struct crypto_tfm *tfm)
3091 {
3092         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3093         struct device *dev = ctx->dev;
3094
3095         if (ctx->keylen)
3096                 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
3097 }
3098
3099 /*
3100  * given the alg's descriptor header template, determine whether descriptor
3101  * type and primary/secondary execution units required match the hw
3102  * capabilities description provided in the device tree node.
3103  */
3104 static int hw_supports(struct device *dev, __be32 desc_hdr_template)
3105 {
3106         struct talitos_private *priv = dev_get_drvdata(dev);
3107         int ret;
3108
3109         ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
3110               (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
3111
3112         if (SECONDARY_EU(desc_hdr_template))
3113                 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
3114                               & priv->exec_units);
3115
3116         return ret;
3117 }
3118
3119 static int talitos_remove(struct platform_device *ofdev)
3120 {
3121         struct device *dev = &ofdev->dev;
3122         struct talitos_private *priv = dev_get_drvdata(dev);
3123         struct talitos_crypto_alg *t_alg, *n;
3124         int i;
3125
3126         list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
3127                 switch (t_alg->algt.type) {
3128                 case CRYPTO_ALG_TYPE_SKCIPHER:
3129                         crypto_unregister_skcipher(&t_alg->algt.alg.skcipher);
3130                         break;
3131                 case CRYPTO_ALG_TYPE_AEAD:
3132                         crypto_unregister_aead(&t_alg->algt.alg.aead);
3133                         break;
3134                 case CRYPTO_ALG_TYPE_AHASH:
3135                         crypto_unregister_ahash(&t_alg->algt.alg.hash);
3136                         break;
3137                 }
3138                 list_del(&t_alg->entry);
3139         }
3140
3141         if (hw_supports(dev, DESC_HDR_SEL0_RNG))
3142                 talitos_unregister_rng(dev);
3143
3144         for (i = 0; i < 2; i++)
3145                 if (priv->irq[i]) {
3146                         free_irq(priv->irq[i], dev);
3147                         irq_dispose_mapping(priv->irq[i]);
3148                 }
3149
3150         tasklet_kill(&priv->done_task[0]);
3151         if (priv->irq[1])
3152                 tasklet_kill(&priv->done_task[1]);
3153
3154         return 0;
3155 }
3156
3157 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
3158                                                     struct talitos_alg_template
3159                                                            *template)
3160 {
3161         struct talitos_private *priv = dev_get_drvdata(dev);
3162         struct talitos_crypto_alg *t_alg;
3163         struct crypto_alg *alg;
3164
3165         t_alg = devm_kzalloc(dev, sizeof(struct talitos_crypto_alg),
3166                              GFP_KERNEL);
3167         if (!t_alg)
3168                 return ERR_PTR(-ENOMEM);
3169
3170         t_alg->algt = *template;
3171
3172         switch (t_alg->algt.type) {
3173         case CRYPTO_ALG_TYPE_SKCIPHER:
3174                 alg = &t_alg->algt.alg.skcipher.base;
3175                 alg->cra_exit = talitos_cra_exit;
3176                 t_alg->algt.alg.skcipher.init = talitos_cra_init_skcipher;
3177                 t_alg->algt.alg.skcipher.setkey =
3178                         t_alg->algt.alg.skcipher.setkey ?: skcipher_setkey;
3179                 t_alg->algt.alg.skcipher.encrypt = skcipher_encrypt;
3180                 t_alg->algt.alg.skcipher.decrypt = skcipher_decrypt;
3181                 break;
3182         case CRYPTO_ALG_TYPE_AEAD:
3183                 alg = &t_alg->algt.alg.aead.base;
3184                 alg->cra_exit = talitos_cra_exit;
3185                 t_alg->algt.alg.aead.init = talitos_cra_init_aead;
3186                 t_alg->algt.alg.aead.setkey = t_alg->algt.alg.aead.setkey ?:
3187                                               aead_setkey;
3188                 t_alg->algt.alg.aead.encrypt = aead_encrypt;
3189                 t_alg->algt.alg.aead.decrypt = aead_decrypt;
3190                 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3191                     !strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) {
3192                         devm_kfree(dev, t_alg);
3193                         return ERR_PTR(-ENOTSUPP);
3194                 }
3195                 break;
3196         case CRYPTO_ALG_TYPE_AHASH:
3197                 alg = &t_alg->algt.alg.hash.halg.base;
3198                 alg->cra_init = talitos_cra_init_ahash;
3199                 alg->cra_exit = talitos_cra_exit;
3200                 t_alg->algt.alg.hash.init = ahash_init;
3201                 t_alg->algt.alg.hash.update = ahash_update;
3202                 t_alg->algt.alg.hash.final = ahash_final;
3203                 t_alg->algt.alg.hash.finup = ahash_finup;
3204                 t_alg->algt.alg.hash.digest = ahash_digest;
3205                 if (!strncmp(alg->cra_name, "hmac", 4))
3206                         t_alg->algt.alg.hash.setkey = ahash_setkey;
3207                 t_alg->algt.alg.hash.import = ahash_import;
3208                 t_alg->algt.alg.hash.export = ahash_export;
3209
3210                 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
3211                     !strncmp(alg->cra_name, "hmac", 4)) {
3212                         devm_kfree(dev, t_alg);
3213                         return ERR_PTR(-ENOTSUPP);
3214                 }
3215                 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3216                     (!strcmp(alg->cra_name, "sha224") ||
3217                      !strcmp(alg->cra_name, "hmac(sha224)"))) {
3218                         t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
3219                         t_alg->algt.desc_hdr_template =
3220                                         DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3221                                         DESC_HDR_SEL0_MDEUA |
3222                                         DESC_HDR_MODE0_MDEU_SHA256;
3223                 }
3224                 break;
3225         default:
3226                 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
3227                 devm_kfree(dev, t_alg);
3228                 return ERR_PTR(-EINVAL);
3229         }
3230
3231         alg->cra_module = THIS_MODULE;
3232         if (t_alg->algt.priority)
3233                 alg->cra_priority = t_alg->algt.priority;
3234         else
3235                 alg->cra_priority = TALITOS_CRA_PRIORITY;
3236         if (has_ftr_sec1(priv))
3237                 alg->cra_alignmask = 3;
3238         else
3239                 alg->cra_alignmask = 0;
3240         alg->cra_ctxsize = sizeof(struct talitos_ctx);
3241         alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
3242
3243         t_alg->dev = dev;
3244
3245         return t_alg;
3246 }
3247
3248 static int talitos_probe_irq(struct platform_device *ofdev)
3249 {
3250         struct device *dev = &ofdev->dev;
3251         struct device_node *np = ofdev->dev.of_node;
3252         struct talitos_private *priv = dev_get_drvdata(dev);
3253         int err;
3254         bool is_sec1 = has_ftr_sec1(priv);
3255
3256         priv->irq[0] = irq_of_parse_and_map(np, 0);
3257         if (!priv->irq[0]) {
3258                 dev_err(dev, "failed to map irq\n");
3259                 return -EINVAL;
3260         }
3261         if (is_sec1) {
3262                 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
3263                                   dev_driver_string(dev), dev);
3264                 goto primary_out;
3265         }
3266
3267         priv->irq[1] = irq_of_parse_and_map(np, 1);
3268
3269         /* get the primary irq line */
3270         if (!priv->irq[1]) {
3271                 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
3272                                   dev_driver_string(dev), dev);
3273                 goto primary_out;
3274         }
3275
3276         err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
3277                           dev_driver_string(dev), dev);
3278         if (err)
3279                 goto primary_out;
3280
3281         /* get the secondary irq line */
3282         err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
3283                           dev_driver_string(dev), dev);
3284         if (err) {
3285                 dev_err(dev, "failed to request secondary irq\n");
3286                 irq_dispose_mapping(priv->irq[1]);
3287                 priv->irq[1] = 0;
3288         }
3289
3290         return err;
3291
3292 primary_out:
3293         if (err) {
3294                 dev_err(dev, "failed to request primary irq\n");
3295                 irq_dispose_mapping(priv->irq[0]);
3296                 priv->irq[0] = 0;
3297         }
3298
3299         return err;
3300 }
3301
3302 static int talitos_probe(struct platform_device *ofdev)
3303 {
3304         struct device *dev = &ofdev->dev;
3305         struct device_node *np = ofdev->dev.of_node;
3306         struct talitos_private *priv;
3307         int i, err;
3308         int stride;
3309         struct resource *res;
3310
3311         priv = devm_kzalloc(dev, sizeof(struct talitos_private), GFP_KERNEL);
3312         if (!priv)
3313                 return -ENOMEM;
3314
3315         INIT_LIST_HEAD(&priv->alg_list);
3316
3317         dev_set_drvdata(dev, priv);
3318
3319         priv->ofdev = ofdev;
3320
3321         spin_lock_init(&priv->reg_lock);
3322
3323         res = platform_get_resource(ofdev, IORESOURCE_MEM, 0);
3324         if (!res)
3325                 return -ENXIO;
3326         priv->reg = devm_ioremap(dev, res->start, resource_size(res));
3327         if (!priv->reg) {
3328                 dev_err(dev, "failed to of_iomap\n");
3329                 err = -ENOMEM;
3330                 goto err_out;
3331         }
3332
3333         /* get SEC version capabilities from device tree */
3334         of_property_read_u32(np, "fsl,num-channels", &priv->num_channels);
3335         of_property_read_u32(np, "fsl,channel-fifo-len", &priv->chfifo_len);
3336         of_property_read_u32(np, "fsl,exec-units-mask", &priv->exec_units);
3337         of_property_read_u32(np, "fsl,descriptor-types-mask",
3338                              &priv->desc_types);
3339
3340         if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
3341             !priv->exec_units || !priv->desc_types) {
3342                 dev_err(dev, "invalid property data in device tree node\n");
3343                 err = -EINVAL;
3344                 goto err_out;
3345         }
3346
3347         if (of_device_is_compatible(np, "fsl,sec3.0"))
3348                 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
3349
3350         if (of_device_is_compatible(np, "fsl,sec2.1"))
3351                 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
3352                                   TALITOS_FTR_SHA224_HWINIT |
3353                                   TALITOS_FTR_HMAC_OK;
3354
3355         if (of_device_is_compatible(np, "fsl,sec1.0"))
3356                 priv->features |= TALITOS_FTR_SEC1;
3357
3358         if (of_device_is_compatible(np, "fsl,sec1.2")) {
3359                 priv->reg_deu = priv->reg + TALITOS12_DEU;
3360                 priv->reg_aesu = priv->reg + TALITOS12_AESU;
3361                 priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
3362                 stride = TALITOS1_CH_STRIDE;
3363         } else if (of_device_is_compatible(np, "fsl,sec1.0")) {
3364                 priv->reg_deu = priv->reg + TALITOS10_DEU;
3365                 priv->reg_aesu = priv->reg + TALITOS10_AESU;
3366                 priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
3367                 priv->reg_afeu = priv->reg + TALITOS10_AFEU;
3368                 priv->reg_rngu = priv->reg + TALITOS10_RNGU;
3369                 priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
3370                 stride = TALITOS1_CH_STRIDE;
3371         } else {
3372                 priv->reg_deu = priv->reg + TALITOS2_DEU;
3373                 priv->reg_aesu = priv->reg + TALITOS2_AESU;
3374                 priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
3375                 priv->reg_afeu = priv->reg + TALITOS2_AFEU;
3376                 priv->reg_rngu = priv->reg + TALITOS2_RNGU;
3377                 priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
3378                 priv->reg_keu = priv->reg + TALITOS2_KEU;
3379                 priv->reg_crcu = priv->reg + TALITOS2_CRCU;
3380                 stride = TALITOS2_CH_STRIDE;
3381         }
3382
3383         err = talitos_probe_irq(ofdev);
3384         if (err)
3385                 goto err_out;
3386
3387         if (has_ftr_sec1(priv)) {
3388                 if (priv->num_channels == 1)
3389                         tasklet_init(&priv->done_task[0], talitos1_done_ch0,
3390                                      (unsigned long)dev);
3391                 else
3392                         tasklet_init(&priv->done_task[0], talitos1_done_4ch,
3393                                      (unsigned long)dev);
3394         } else {
3395                 if (priv->irq[1]) {
3396                         tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
3397                                      (unsigned long)dev);
3398                         tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
3399                                      (unsigned long)dev);
3400                 } else if (priv->num_channels == 1) {
3401                         tasklet_init(&priv->done_task[0], talitos2_done_ch0,
3402                                      (unsigned long)dev);
3403                 } else {
3404                         tasklet_init(&priv->done_task[0], talitos2_done_4ch,
3405                                      (unsigned long)dev);
3406                 }
3407         }
3408
3409         priv->chan = devm_kcalloc(dev,
3410                                   priv->num_channels,
3411                                   sizeof(struct talitos_channel),
3412                                   GFP_KERNEL);
3413         if (!priv->chan) {
3414                 dev_err(dev, "failed to allocate channel management space\n");
3415                 err = -ENOMEM;
3416                 goto err_out;
3417         }
3418
3419         priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
3420
3421         for (i = 0; i < priv->num_channels; i++) {
3422                 priv->chan[i].reg = priv->reg + stride * (i + 1);
3423                 if (!priv->irq[1] || !(i & 1))
3424                         priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
3425
3426                 spin_lock_init(&priv->chan[i].head_lock);
3427                 spin_lock_init(&priv->chan[i].tail_lock);
3428
3429                 priv->chan[i].fifo = devm_kcalloc(dev,
3430                                                 priv->fifo_len,
3431                                                 sizeof(struct talitos_request),
3432                                                 GFP_KERNEL);
3433                 if (!priv->chan[i].fifo) {
3434                         dev_err(dev, "failed to allocate request fifo %d\n", i);
3435                         err = -ENOMEM;
3436                         goto err_out;
3437                 }
3438
3439                 atomic_set(&priv->chan[i].submit_count,
3440                            -(priv->chfifo_len - 1));
3441         }
3442
3443         dma_set_mask(dev, DMA_BIT_MASK(36));
3444
3445         /* reset and initialize the h/w */
3446         err = init_device(dev);
3447         if (err) {
3448                 dev_err(dev, "failed to initialize device\n");
3449                 goto err_out;
3450         }
3451
3452         /* register the RNG, if available */
3453         if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3454                 err = talitos_register_rng(dev);
3455                 if (err) {
3456                         dev_err(dev, "failed to register hwrng: %d\n", err);
3457                         goto err_out;
3458                 } else
3459                         dev_info(dev, "hwrng\n");
3460         }
3461
3462         /* register crypto algorithms the device supports */
3463         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3464                 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
3465                         struct talitos_crypto_alg *t_alg;
3466                         struct crypto_alg *alg = NULL;
3467
3468                         t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
3469                         if (IS_ERR(t_alg)) {
3470                                 err = PTR_ERR(t_alg);
3471                                 if (err == -ENOTSUPP)
3472                                         continue;
3473                                 goto err_out;
3474                         }
3475
3476                         switch (t_alg->algt.type) {
3477                         case CRYPTO_ALG_TYPE_SKCIPHER:
3478                                 err = crypto_register_skcipher(
3479                                                 &t_alg->algt.alg.skcipher);
3480                                 alg = &t_alg->algt.alg.skcipher.base;
3481                                 break;
3482
3483                         case CRYPTO_ALG_TYPE_AEAD:
3484                                 err = crypto_register_aead(
3485                                         &t_alg->algt.alg.aead);
3486                                 alg = &t_alg->algt.alg.aead.base;
3487                                 break;
3488
3489                         case CRYPTO_ALG_TYPE_AHASH:
3490                                 err = crypto_register_ahash(
3491                                                 &t_alg->algt.alg.hash);
3492                                 alg = &t_alg->algt.alg.hash.halg.base;
3493                                 break;
3494                         }
3495                         if (err) {
3496                                 dev_err(dev, "%s alg registration failed\n",
3497                                         alg->cra_driver_name);
3498                                 devm_kfree(dev, t_alg);
3499                         } else
3500                                 list_add_tail(&t_alg->entry, &priv->alg_list);
3501                 }
3502         }
3503         if (!list_empty(&priv->alg_list))
3504                 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3505                          (char *)of_get_property(np, "compatible", NULL));
3506
3507         return 0;
3508
3509 err_out:
3510         talitos_remove(ofdev);
3511
3512         return err;
3513 }
3514
3515 static const struct of_device_id talitos_match[] = {
3516 #ifdef CONFIG_CRYPTO_DEV_TALITOS1
3517         {
3518                 .compatible = "fsl,sec1.0",
3519         },
3520 #endif
3521 #ifdef CONFIG_CRYPTO_DEV_TALITOS2
3522         {
3523                 .compatible = "fsl,sec2.0",
3524         },
3525 #endif
3526         {},
3527 };
3528 MODULE_DEVICE_TABLE(of, talitos_match);
3529
3530 static struct platform_driver talitos_driver = {
3531         .driver = {
3532                 .name = "talitos",
3533                 .of_match_table = talitos_match,
3534         },
3535         .probe = talitos_probe,
3536         .remove = talitos_remove,
3537 };
3538
3539 module_platform_driver(talitos_driver);
3540
3541 MODULE_LICENSE("GPL");
3542 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3543 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");