Merge branch 'dmi-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/jdelvar...
[linux-2.6-microblaze.git] / drivers / crypto / caam / jr.c
1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * CAAM/SEC 4.x transport/backend driver
4  * JobR backend functionality
5  *
6  * Copyright 2008-2012 Freescale Semiconductor, Inc.
7  * Copyright 2019 NXP
8  */
9
10 #include <linux/of_irq.h>
11 #include <linux/of_address.h>
12
13 #include "compat.h"
14 #include "ctrl.h"
15 #include "regs.h"
16 #include "jr.h"
17 #include "desc.h"
18 #include "intern.h"
19
20 struct jr_driver_data {
21         /* List of Physical JobR's with the Driver */
22         struct list_head        jr_list;
23         spinlock_t              jr_alloc_lock;  /* jr_list lock */
24 } ____cacheline_aligned;
25
26 static struct jr_driver_data driver_data;
27 static DEFINE_MUTEX(algs_lock);
28 static unsigned int active_devs;
29
30 static void register_algs(struct caam_drv_private_jr *jrpriv,
31                           struct device *dev)
32 {
33         mutex_lock(&algs_lock);
34
35         if (++active_devs != 1)
36                 goto algs_unlock;
37
38         caam_algapi_init(dev);
39         caam_algapi_hash_init(dev);
40         caam_pkc_init(dev);
41         jrpriv->hwrng = !caam_rng_init(dev);
42         caam_qi_algapi_init(dev);
43
44 algs_unlock:
45         mutex_unlock(&algs_lock);
46 }
47
48 static void unregister_algs(void)
49 {
50         mutex_lock(&algs_lock);
51
52         if (--active_devs != 0)
53                 goto algs_unlock;
54
55         caam_qi_algapi_exit();
56
57         caam_pkc_exit();
58         caam_algapi_hash_exit();
59         caam_algapi_exit();
60
61 algs_unlock:
62         mutex_unlock(&algs_lock);
63 }
64
65 static void caam_jr_crypto_engine_exit(void *data)
66 {
67         struct device *jrdev = data;
68         struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev);
69
70         /* Free the resources of crypto-engine */
71         crypto_engine_exit(jrpriv->engine);
72 }
73
74 static int caam_reset_hw_jr(struct device *dev)
75 {
76         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
77         unsigned int timeout = 100000;
78
79         /*
80          * mask interrupts since we are going to poll
81          * for reset completion status
82          */
83         clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JRCFG_IMSK);
84
85         /* initiate flush (required prior to reset) */
86         wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
87         while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) ==
88                 JRINT_ERR_HALT_INPROGRESS) && --timeout)
89                 cpu_relax();
90
91         if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) !=
92             JRINT_ERR_HALT_COMPLETE || timeout == 0) {
93                 dev_err(dev, "failed to flush job ring %d\n", jrp->ridx);
94                 return -EIO;
95         }
96
97         /* initiate reset */
98         timeout = 100000;
99         wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
100         while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout)
101                 cpu_relax();
102
103         if (timeout == 0) {
104                 dev_err(dev, "failed to reset job ring %d\n", jrp->ridx);
105                 return -EIO;
106         }
107
108         /* unmask interrupts */
109         clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0);
110
111         return 0;
112 }
113
114 /*
115  * Shutdown JobR independent of platform property code
116  */
117 static int caam_jr_shutdown(struct device *dev)
118 {
119         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
120         int ret;
121
122         ret = caam_reset_hw_jr(dev);
123
124         tasklet_kill(&jrp->irqtask);
125
126         return ret;
127 }
128
129 static int caam_jr_remove(struct platform_device *pdev)
130 {
131         int ret;
132         struct device *jrdev;
133         struct caam_drv_private_jr *jrpriv;
134
135         jrdev = &pdev->dev;
136         jrpriv = dev_get_drvdata(jrdev);
137
138         if (jrpriv->hwrng)
139                 caam_rng_exit(jrdev->parent);
140
141         /*
142          * Return EBUSY if job ring already allocated.
143          */
144         if (atomic_read(&jrpriv->tfm_count)) {
145                 dev_err(jrdev, "Device is busy\n");
146                 return -EBUSY;
147         }
148
149         /* Unregister JR-based RNG & crypto algorithms */
150         unregister_algs();
151
152         /* Remove the node from Physical JobR list maintained by driver */
153         spin_lock(&driver_data.jr_alloc_lock);
154         list_del(&jrpriv->list_node);
155         spin_unlock(&driver_data.jr_alloc_lock);
156
157         /* Release ring */
158         ret = caam_jr_shutdown(jrdev);
159         if (ret)
160                 dev_err(jrdev, "Failed to shut down job ring\n");
161
162         return ret;
163 }
164
165 /* Main per-ring interrupt handler */
166 static irqreturn_t caam_jr_interrupt(int irq, void *st_dev)
167 {
168         struct device *dev = st_dev;
169         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
170         u32 irqstate;
171
172         /*
173          * Check the output ring for ready responses, kick
174          * tasklet if jobs done.
175          */
176         irqstate = rd_reg32(&jrp->rregs->jrintstatus);
177         if (!irqstate)
178                 return IRQ_NONE;
179
180         /*
181          * If JobR error, we got more development work to do
182          * Flag a bug now, but we really need to shut down and
183          * restart the queue (and fix code).
184          */
185         if (irqstate & JRINT_JR_ERROR) {
186                 dev_err(dev, "job ring error: irqstate: %08x\n", irqstate);
187                 BUG();
188         }
189
190         /* mask valid interrupts */
191         clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JRCFG_IMSK);
192
193         /* Have valid interrupt at this point, just ACK and trigger */
194         wr_reg32(&jrp->rregs->jrintstatus, irqstate);
195
196         preempt_disable();
197         tasklet_schedule(&jrp->irqtask);
198         preempt_enable();
199
200         return IRQ_HANDLED;
201 }
202
203 /* Deferred service handler, run as interrupt-fired tasklet */
204 static void caam_jr_dequeue(unsigned long devarg)
205 {
206         int hw_idx, sw_idx, i, head, tail;
207         struct device *dev = (struct device *)devarg;
208         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
209         void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg);
210         u32 *userdesc, userstatus;
211         void *userarg;
212         u32 outring_used = 0;
213
214         while (outring_used ||
215                (outring_used = rd_reg32(&jrp->rregs->outring_used))) {
216
217                 head = READ_ONCE(jrp->head);
218
219                 sw_idx = tail = jrp->tail;
220                 hw_idx = jrp->out_ring_read_index;
221
222                 for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) {
223                         sw_idx = (tail + i) & (JOBR_DEPTH - 1);
224
225                         if (jr_outentry_desc(jrp->outring, hw_idx) ==
226                             caam_dma_to_cpu(jrp->entinfo[sw_idx].desc_addr_dma))
227                                 break; /* found */
228                 }
229                 /* we should never fail to find a matching descriptor */
230                 BUG_ON(CIRC_CNT(head, tail + i, JOBR_DEPTH) <= 0);
231
232                 /* Unmap just-run descriptor so we can post-process */
233                 dma_unmap_single(dev,
234                                  caam_dma_to_cpu(jr_outentry_desc(jrp->outring,
235                                                                   hw_idx)),
236                                  jrp->entinfo[sw_idx].desc_size,
237                                  DMA_TO_DEVICE);
238
239                 /* mark completed, avoid matching on a recycled desc addr */
240                 jrp->entinfo[sw_idx].desc_addr_dma = 0;
241
242                 /* Stash callback params */
243                 usercall = jrp->entinfo[sw_idx].callbk;
244                 userarg = jrp->entinfo[sw_idx].cbkarg;
245                 userdesc = jrp->entinfo[sw_idx].desc_addr_virt;
246                 userstatus = caam32_to_cpu(jr_outentry_jrstatus(jrp->outring,
247                                                                 hw_idx));
248
249                 /*
250                  * Make sure all information from the job has been obtained
251                  * before telling CAAM that the job has been removed from the
252                  * output ring.
253                  */
254                 mb();
255
256                 /* set done */
257                 wr_reg32(&jrp->rregs->outring_rmvd, 1);
258
259                 jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) &
260                                            (JOBR_DEPTH - 1);
261
262                 /*
263                  * if this job completed out-of-order, do not increment
264                  * the tail.  Otherwise, increment tail by 1 plus the
265                  * number of subsequent jobs already completed out-of-order
266                  */
267                 if (sw_idx == tail) {
268                         do {
269                                 tail = (tail + 1) & (JOBR_DEPTH - 1);
270                         } while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 &&
271                                  jrp->entinfo[tail].desc_addr_dma == 0);
272
273                         jrp->tail = tail;
274                 }
275
276                 /* Finally, execute user's callback */
277                 usercall(dev, userdesc, userstatus, userarg);
278                 outring_used--;
279         }
280
281         /* reenable / unmask IRQs */
282         clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0);
283 }
284
285 /**
286  * caam_jr_alloc() - Alloc a job ring for someone to use as needed.
287  *
288  * returns :  pointer to the newly allocated physical
289  *            JobR dev can be written to if successful.
290  **/
291 struct device *caam_jr_alloc(void)
292 {
293         struct caam_drv_private_jr *jrpriv, *min_jrpriv = NULL;
294         struct device *dev = ERR_PTR(-ENODEV);
295         int min_tfm_cnt = INT_MAX;
296         int tfm_cnt;
297
298         spin_lock(&driver_data.jr_alloc_lock);
299
300         if (list_empty(&driver_data.jr_list)) {
301                 spin_unlock(&driver_data.jr_alloc_lock);
302                 return ERR_PTR(-ENODEV);
303         }
304
305         list_for_each_entry(jrpriv, &driver_data.jr_list, list_node) {
306                 tfm_cnt = atomic_read(&jrpriv->tfm_count);
307                 if (tfm_cnt < min_tfm_cnt) {
308                         min_tfm_cnt = tfm_cnt;
309                         min_jrpriv = jrpriv;
310                 }
311                 if (!min_tfm_cnt)
312                         break;
313         }
314
315         if (min_jrpriv) {
316                 atomic_inc(&min_jrpriv->tfm_count);
317                 dev = min_jrpriv->dev;
318         }
319         spin_unlock(&driver_data.jr_alloc_lock);
320
321         return dev;
322 }
323 EXPORT_SYMBOL(caam_jr_alloc);
324
325 /**
326  * caam_jr_free() - Free the Job Ring
327  * @rdev:      points to the dev that identifies the Job ring to
328  *             be released.
329  **/
330 void caam_jr_free(struct device *rdev)
331 {
332         struct caam_drv_private_jr *jrpriv = dev_get_drvdata(rdev);
333
334         atomic_dec(&jrpriv->tfm_count);
335 }
336 EXPORT_SYMBOL(caam_jr_free);
337
338 /**
339  * caam_jr_enqueue() - Enqueue a job descriptor head. Returns -EINPROGRESS
340  * if OK, -ENOSPC if the queue is full, -EIO if it cannot map the caller's
341  * descriptor.
342  * @dev:  struct device of the job ring to be used
343  * @desc: points to a job descriptor that execute our request. All
344  *        descriptors (and all referenced data) must be in a DMAable
345  *        region, and all data references must be physical addresses
346  *        accessible to CAAM (i.e. within a PAMU window granted
347  *        to it).
348  * @cbk:  pointer to a callback function to be invoked upon completion
349  *        of this request. This has the form:
350  *        callback(struct device *dev, u32 *desc, u32 stat, void *arg)
351  *        where:
352  *        dev:     contains the job ring device that processed this
353  *                 response.
354  *        desc:    descriptor that initiated the request, same as
355  *                 "desc" being argued to caam_jr_enqueue().
356  *        status:  untranslated status received from CAAM. See the
357  *                 reference manual for a detailed description of
358  *                 error meaning, or see the JRSTA definitions in the
359  *                 register header file
360  *        areq:    optional pointer to an argument passed with the
361  *                 original request
362  * @areq: optional pointer to a user argument for use at callback
363  *        time.
364  **/
365 int caam_jr_enqueue(struct device *dev, u32 *desc,
366                     void (*cbk)(struct device *dev, u32 *desc,
367                                 u32 status, void *areq),
368                     void *areq)
369 {
370         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
371         struct caam_jrentry_info *head_entry;
372         int head, tail, desc_size;
373         dma_addr_t desc_dma;
374
375         desc_size = (caam32_to_cpu(*desc) & HDR_JD_LENGTH_MASK) * sizeof(u32);
376         desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE);
377         if (dma_mapping_error(dev, desc_dma)) {
378                 dev_err(dev, "caam_jr_enqueue(): can't map jobdesc\n");
379                 return -EIO;
380         }
381
382         spin_lock_bh(&jrp->inplock);
383
384         head = jrp->head;
385         tail = READ_ONCE(jrp->tail);
386
387         if (!jrp->inpring_avail ||
388             CIRC_SPACE(head, tail, JOBR_DEPTH) <= 0) {
389                 spin_unlock_bh(&jrp->inplock);
390                 dma_unmap_single(dev, desc_dma, desc_size, DMA_TO_DEVICE);
391                 return -ENOSPC;
392         }
393
394         head_entry = &jrp->entinfo[head];
395         head_entry->desc_addr_virt = desc;
396         head_entry->desc_size = desc_size;
397         head_entry->callbk = (void *)cbk;
398         head_entry->cbkarg = areq;
399         head_entry->desc_addr_dma = desc_dma;
400
401         jr_inpentry_set(jrp->inpring, head, cpu_to_caam_dma(desc_dma));
402
403         /*
404          * Guarantee that the descriptor's DMA address has been written to
405          * the next slot in the ring before the write index is updated, since
406          * other cores may update this index independently.
407          */
408         smp_wmb();
409
410         jrp->head = (head + 1) & (JOBR_DEPTH - 1);
411
412         /*
413          * Ensure that all job information has been written before
414          * notifying CAAM that a new job was added to the input ring
415          * using a memory barrier. The wr_reg32() uses api iowrite32()
416          * to do the register write. iowrite32() issues a memory barrier
417          * before the write operation.
418          */
419
420         wr_reg32(&jrp->rregs->inpring_jobadd, 1);
421
422         jrp->inpring_avail--;
423         if (!jrp->inpring_avail)
424                 jrp->inpring_avail = rd_reg32(&jrp->rregs->inpring_avail);
425
426         spin_unlock_bh(&jrp->inplock);
427
428         return -EINPROGRESS;
429 }
430 EXPORT_SYMBOL(caam_jr_enqueue);
431
432 /*
433  * Init JobR independent of platform property detection
434  */
435 static int caam_jr_init(struct device *dev)
436 {
437         struct caam_drv_private_jr *jrp;
438         dma_addr_t inpbusaddr, outbusaddr;
439         int i, error;
440
441         jrp = dev_get_drvdata(dev);
442
443         error = caam_reset_hw_jr(dev);
444         if (error)
445                 return error;
446
447         jrp->inpring = dmam_alloc_coherent(dev, SIZEOF_JR_INPENTRY *
448                                            JOBR_DEPTH, &inpbusaddr,
449                                            GFP_KERNEL);
450         if (!jrp->inpring)
451                 return -ENOMEM;
452
453         jrp->outring = dmam_alloc_coherent(dev, SIZEOF_JR_OUTENTRY *
454                                            JOBR_DEPTH, &outbusaddr,
455                                            GFP_KERNEL);
456         if (!jrp->outring)
457                 return -ENOMEM;
458
459         jrp->entinfo = devm_kcalloc(dev, JOBR_DEPTH, sizeof(*jrp->entinfo),
460                                     GFP_KERNEL);
461         if (!jrp->entinfo)
462                 return -ENOMEM;
463
464         for (i = 0; i < JOBR_DEPTH; i++)
465                 jrp->entinfo[i].desc_addr_dma = !0;
466
467         /* Setup rings */
468         jrp->out_ring_read_index = 0;
469         jrp->head = 0;
470         jrp->tail = 0;
471
472         wr_reg64(&jrp->rregs->inpring_base, inpbusaddr);
473         wr_reg64(&jrp->rregs->outring_base, outbusaddr);
474         wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH);
475         wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH);
476
477         jrp->inpring_avail = JOBR_DEPTH;
478
479         spin_lock_init(&jrp->inplock);
480
481         /* Select interrupt coalescing parameters */
482         clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JOBR_INTC |
483                       (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
484                       (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
485
486         tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
487
488         /* Connect job ring interrupt handler. */
489         error = devm_request_irq(dev, jrp->irq, caam_jr_interrupt, IRQF_SHARED,
490                                  dev_name(dev), dev);
491         if (error) {
492                 dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
493                         jrp->ridx, jrp->irq);
494                 tasklet_kill(&jrp->irqtask);
495         }
496
497         return error;
498 }
499
500 static void caam_jr_irq_dispose_mapping(void *data)
501 {
502         irq_dispose_mapping((unsigned long)data);
503 }
504
505 /*
506  * Probe routine for each detected JobR subsystem.
507  */
508 static int caam_jr_probe(struct platform_device *pdev)
509 {
510         struct device *jrdev;
511         struct device_node *nprop;
512         struct caam_job_ring __iomem *ctrl;
513         struct caam_drv_private_jr *jrpriv;
514         static int total_jobrs;
515         struct resource *r;
516         int error;
517
518         jrdev = &pdev->dev;
519         jrpriv = devm_kzalloc(jrdev, sizeof(*jrpriv), GFP_KERNEL);
520         if (!jrpriv)
521                 return -ENOMEM;
522
523         dev_set_drvdata(jrdev, jrpriv);
524
525         /* save ring identity relative to detection */
526         jrpriv->ridx = total_jobrs++;
527
528         nprop = pdev->dev.of_node;
529         /* Get configuration properties from device tree */
530         /* First, get register page */
531         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
532         if (!r) {
533                 dev_err(jrdev, "platform_get_resource() failed\n");
534                 return -ENOMEM;
535         }
536
537         ctrl = devm_ioremap(jrdev, r->start, resource_size(r));
538         if (!ctrl) {
539                 dev_err(jrdev, "devm_ioremap() failed\n");
540                 return -ENOMEM;
541         }
542
543         jrpriv->rregs = (struct caam_job_ring __iomem __force *)ctrl;
544
545         error = dma_set_mask_and_coherent(jrdev, caam_get_dma_mask(jrdev));
546         if (error) {
547                 dev_err(jrdev, "dma_set_mask_and_coherent failed (%d)\n",
548                         error);
549                 return error;
550         }
551
552         /* Initialize crypto engine */
553         jrpriv->engine = crypto_engine_alloc_init_and_set(jrdev, true, NULL,
554                                                           false,
555                                                           CRYPTO_ENGINE_MAX_QLEN);
556         if (!jrpriv->engine) {
557                 dev_err(jrdev, "Could not init crypto-engine\n");
558                 return -ENOMEM;
559         }
560
561         error = devm_add_action_or_reset(jrdev, caam_jr_crypto_engine_exit,
562                                          jrdev);
563         if (error)
564                 return error;
565
566         /* Start crypto engine */
567         error = crypto_engine_start(jrpriv->engine);
568         if (error) {
569                 dev_err(jrdev, "Could not start crypto-engine\n");
570                 return error;
571         }
572
573         /* Identify the interrupt */
574         jrpriv->irq = irq_of_parse_and_map(nprop, 0);
575         if (!jrpriv->irq) {
576                 dev_err(jrdev, "irq_of_parse_and_map failed\n");
577                 return -EINVAL;
578         }
579
580         error = devm_add_action_or_reset(jrdev, caam_jr_irq_dispose_mapping,
581                                          (void *)(unsigned long)jrpriv->irq);
582         if (error)
583                 return error;
584
585         /* Now do the platform independent part */
586         error = caam_jr_init(jrdev); /* now turn on hardware */
587         if (error)
588                 return error;
589
590         jrpriv->dev = jrdev;
591         spin_lock(&driver_data.jr_alloc_lock);
592         list_add_tail(&jrpriv->list_node, &driver_data.jr_list);
593         spin_unlock(&driver_data.jr_alloc_lock);
594
595         atomic_set(&jrpriv->tfm_count, 0);
596
597         register_algs(jrpriv, jrdev->parent);
598
599         return 0;
600 }
601
602 static const struct of_device_id caam_jr_match[] = {
603         {
604                 .compatible = "fsl,sec-v4.0-job-ring",
605         },
606         {
607                 .compatible = "fsl,sec4.0-job-ring",
608         },
609         {},
610 };
611 MODULE_DEVICE_TABLE(of, caam_jr_match);
612
613 static struct platform_driver caam_jr_driver = {
614         .driver = {
615                 .name = "caam_jr",
616                 .of_match_table = caam_jr_match,
617         },
618         .probe       = caam_jr_probe,
619         .remove      = caam_jr_remove,
620 };
621
622 static int __init jr_driver_init(void)
623 {
624         spin_lock_init(&driver_data.jr_alloc_lock);
625         INIT_LIST_HEAD(&driver_data.jr_list);
626         return platform_driver_register(&caam_jr_driver);
627 }
628
629 static void __exit jr_driver_exit(void)
630 {
631         platform_driver_unregister(&caam_jr_driver);
632 }
633
634 module_init(jr_driver_init);
635 module_exit(jr_driver_exit);
636
637 MODULE_LICENSE("GPL");
638 MODULE_DESCRIPTION("FSL CAAM JR request backend");
639 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");