Merge remote-tracking branch 'asoc/fix/intel' into asoc-linus
[linux-2.6-microblaze.git] / drivers / dma / omap-dma.c
1 /*
2  * OMAP DMAengine support
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  */
8 #include <linux/delay.h>
9 #include <linux/dmaengine.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/dmapool.h>
12 #include <linux/err.h>
13 #include <linux/init.h>
14 #include <linux/interrupt.h>
15 #include <linux/list.h>
16 #include <linux/module.h>
17 #include <linux/omap-dma.h>
18 #include <linux/platform_device.h>
19 #include <linux/slab.h>
20 #include <linux/spinlock.h>
21 #include <linux/of_dma.h>
22 #include <linux/of_device.h>
23
24 #include "virt-dma.h"
25
26 #define OMAP_SDMA_REQUESTS      127
27 #define OMAP_SDMA_CHANNELS      32
28
29 struct omap_dmadev {
30         struct dma_device ddev;
31         spinlock_t lock;
32         void __iomem *base;
33         const struct omap_dma_reg *reg_map;
34         struct omap_system_dma_plat_info *plat;
35         bool legacy;
36         bool ll123_supported;
37         struct dma_pool *desc_pool;
38         unsigned dma_requests;
39         spinlock_t irq_lock;
40         uint32_t irq_enable_mask;
41         struct omap_chan **lch_map;
42 };
43
44 struct omap_chan {
45         struct virt_dma_chan vc;
46         void __iomem *channel_base;
47         const struct omap_dma_reg *reg_map;
48         uint32_t ccr;
49
50         struct dma_slave_config cfg;
51         unsigned dma_sig;
52         bool cyclic;
53         bool paused;
54         bool running;
55
56         int dma_ch;
57         struct omap_desc *desc;
58         unsigned sgidx;
59 };
60
61 #define DESC_NXT_SV_REFRESH     (0x1 << 24)
62 #define DESC_NXT_SV_REUSE       (0x2 << 24)
63 #define DESC_NXT_DV_REFRESH     (0x1 << 26)
64 #define DESC_NXT_DV_REUSE       (0x2 << 26)
65 #define DESC_NTYPE_TYPE2        (0x2 << 29)
66
67 /* Type 2 descriptor with Source or Destination address update */
68 struct omap_type2_desc {
69         uint32_t next_desc;
70         uint32_t en;
71         uint32_t addr; /* src or dst */
72         uint16_t fn;
73         uint16_t cicr;
74         int16_t cdei;
75         int16_t csei;
76         int32_t cdfi;
77         int32_t csfi;
78 } __packed;
79
80 struct omap_sg {
81         dma_addr_t addr;
82         uint32_t en;            /* number of elements (24-bit) */
83         uint32_t fn;            /* number of frames (16-bit) */
84         int32_t fi;             /* for double indexing */
85         int16_t ei;             /* for double indexing */
86
87         /* Linked list */
88         struct omap_type2_desc *t2_desc;
89         dma_addr_t t2_desc_paddr;
90 };
91
92 struct omap_desc {
93         struct virt_dma_desc vd;
94         bool using_ll;
95         enum dma_transfer_direction dir;
96         dma_addr_t dev_addr;
97
98         int32_t fi;             /* for OMAP_DMA_SYNC_PACKET / double indexing */
99         int16_t ei;             /* for double indexing */
100         uint8_t es;             /* CSDP_DATA_TYPE_xxx */
101         uint32_t ccr;           /* CCR value */
102         uint16_t clnk_ctrl;     /* CLNK_CTRL value */
103         uint16_t cicr;          /* CICR value */
104         uint32_t csdp;          /* CSDP value */
105
106         unsigned sglen;
107         struct omap_sg sg[0];
108 };
109
110 enum {
111         CAPS_0_SUPPORT_LL123    = BIT(20),      /* Linked List type1/2/3 */
112         CAPS_0_SUPPORT_LL4      = BIT(21),      /* Linked List type4 */
113
114         CCR_FS                  = BIT(5),
115         CCR_READ_PRIORITY       = BIT(6),
116         CCR_ENABLE              = BIT(7),
117         CCR_AUTO_INIT           = BIT(8),       /* OMAP1 only */
118         CCR_REPEAT              = BIT(9),       /* OMAP1 only */
119         CCR_OMAP31_DISABLE      = BIT(10),      /* OMAP1 only */
120         CCR_SUSPEND_SENSITIVE   = BIT(8),       /* OMAP2+ only */
121         CCR_RD_ACTIVE           = BIT(9),       /* OMAP2+ only */
122         CCR_WR_ACTIVE           = BIT(10),      /* OMAP2+ only */
123         CCR_SRC_AMODE_CONSTANT  = 0 << 12,
124         CCR_SRC_AMODE_POSTINC   = 1 << 12,
125         CCR_SRC_AMODE_SGLIDX    = 2 << 12,
126         CCR_SRC_AMODE_DBLIDX    = 3 << 12,
127         CCR_DST_AMODE_CONSTANT  = 0 << 14,
128         CCR_DST_AMODE_POSTINC   = 1 << 14,
129         CCR_DST_AMODE_SGLIDX    = 2 << 14,
130         CCR_DST_AMODE_DBLIDX    = 3 << 14,
131         CCR_CONSTANT_FILL       = BIT(16),
132         CCR_TRANSPARENT_COPY    = BIT(17),
133         CCR_BS                  = BIT(18),
134         CCR_SUPERVISOR          = BIT(22),
135         CCR_PREFETCH            = BIT(23),
136         CCR_TRIGGER_SRC         = BIT(24),
137         CCR_BUFFERING_DISABLE   = BIT(25),
138         CCR_WRITE_PRIORITY      = BIT(26),
139         CCR_SYNC_ELEMENT        = 0,
140         CCR_SYNC_FRAME          = CCR_FS,
141         CCR_SYNC_BLOCK          = CCR_BS,
142         CCR_SYNC_PACKET         = CCR_BS | CCR_FS,
143
144         CSDP_DATA_TYPE_8        = 0,
145         CSDP_DATA_TYPE_16       = 1,
146         CSDP_DATA_TYPE_32       = 2,
147         CSDP_SRC_PORT_EMIFF     = 0 << 2, /* OMAP1 only */
148         CSDP_SRC_PORT_EMIFS     = 1 << 2, /* OMAP1 only */
149         CSDP_SRC_PORT_OCP_T1    = 2 << 2, /* OMAP1 only */
150         CSDP_SRC_PORT_TIPB      = 3 << 2, /* OMAP1 only */
151         CSDP_SRC_PORT_OCP_T2    = 4 << 2, /* OMAP1 only */
152         CSDP_SRC_PORT_MPUI      = 5 << 2, /* OMAP1 only */
153         CSDP_SRC_PACKED         = BIT(6),
154         CSDP_SRC_BURST_1        = 0 << 7,
155         CSDP_SRC_BURST_16       = 1 << 7,
156         CSDP_SRC_BURST_32       = 2 << 7,
157         CSDP_SRC_BURST_64       = 3 << 7,
158         CSDP_DST_PORT_EMIFF     = 0 << 9, /* OMAP1 only */
159         CSDP_DST_PORT_EMIFS     = 1 << 9, /* OMAP1 only */
160         CSDP_DST_PORT_OCP_T1    = 2 << 9, /* OMAP1 only */
161         CSDP_DST_PORT_TIPB      = 3 << 9, /* OMAP1 only */
162         CSDP_DST_PORT_OCP_T2    = 4 << 9, /* OMAP1 only */
163         CSDP_DST_PORT_MPUI      = 5 << 9, /* OMAP1 only */
164         CSDP_DST_PACKED         = BIT(13),
165         CSDP_DST_BURST_1        = 0 << 14,
166         CSDP_DST_BURST_16       = 1 << 14,
167         CSDP_DST_BURST_32       = 2 << 14,
168         CSDP_DST_BURST_64       = 3 << 14,
169
170         CICR_TOUT_IE            = BIT(0),       /* OMAP1 only */
171         CICR_DROP_IE            = BIT(1),
172         CICR_HALF_IE            = BIT(2),
173         CICR_FRAME_IE           = BIT(3),
174         CICR_LAST_IE            = BIT(4),
175         CICR_BLOCK_IE           = BIT(5),
176         CICR_PKT_IE             = BIT(7),       /* OMAP2+ only */
177         CICR_TRANS_ERR_IE       = BIT(8),       /* OMAP2+ only */
178         CICR_SUPERVISOR_ERR_IE  = BIT(10),      /* OMAP2+ only */
179         CICR_MISALIGNED_ERR_IE  = BIT(11),      /* OMAP2+ only */
180         CICR_DRAIN_IE           = BIT(12),      /* OMAP2+ only */
181         CICR_SUPER_BLOCK_IE     = BIT(14),      /* OMAP2+ only */
182
183         CLNK_CTRL_ENABLE_LNK    = BIT(15),
184
185         CDP_DST_VALID_INC       = 0 << 0,
186         CDP_DST_VALID_RELOAD    = 1 << 0,
187         CDP_DST_VALID_REUSE     = 2 << 0,
188         CDP_SRC_VALID_INC       = 0 << 2,
189         CDP_SRC_VALID_RELOAD    = 1 << 2,
190         CDP_SRC_VALID_REUSE     = 2 << 2,
191         CDP_NTYPE_TYPE1         = 1 << 4,
192         CDP_NTYPE_TYPE2         = 2 << 4,
193         CDP_NTYPE_TYPE3         = 3 << 4,
194         CDP_TMODE_NORMAL        = 0 << 8,
195         CDP_TMODE_LLIST         = 1 << 8,
196         CDP_FAST                = BIT(10),
197 };
198
199 static const unsigned es_bytes[] = {
200         [CSDP_DATA_TYPE_8] = 1,
201         [CSDP_DATA_TYPE_16] = 2,
202         [CSDP_DATA_TYPE_32] = 4,
203 };
204
205 static struct of_dma_filter_info omap_dma_info = {
206         .filter_fn = omap_dma_filter_fn,
207 };
208
209 static inline struct omap_dmadev *to_omap_dma_dev(struct dma_device *d)
210 {
211         return container_of(d, struct omap_dmadev, ddev);
212 }
213
214 static inline struct omap_chan *to_omap_dma_chan(struct dma_chan *c)
215 {
216         return container_of(c, struct omap_chan, vc.chan);
217 }
218
219 static inline struct omap_desc *to_omap_dma_desc(struct dma_async_tx_descriptor *t)
220 {
221         return container_of(t, struct omap_desc, vd.tx);
222 }
223
224 static void omap_dma_desc_free(struct virt_dma_desc *vd)
225 {
226         struct omap_desc *d = to_omap_dma_desc(&vd->tx);
227
228         if (d->using_ll) {
229                 struct omap_dmadev *od = to_omap_dma_dev(vd->tx.chan->device);
230                 int i;
231
232                 for (i = 0; i < d->sglen; i++) {
233                         if (d->sg[i].t2_desc)
234                                 dma_pool_free(od->desc_pool, d->sg[i].t2_desc,
235                                               d->sg[i].t2_desc_paddr);
236                 }
237         }
238
239         kfree(d);
240 }
241
242 static void omap_dma_fill_type2_desc(struct omap_desc *d, int idx,
243                                      enum dma_transfer_direction dir, bool last)
244 {
245         struct omap_sg *sg = &d->sg[idx];
246         struct omap_type2_desc *t2_desc = sg->t2_desc;
247
248         if (idx)
249                 d->sg[idx - 1].t2_desc->next_desc = sg->t2_desc_paddr;
250         if (last)
251                 t2_desc->next_desc = 0xfffffffc;
252
253         t2_desc->en = sg->en;
254         t2_desc->addr = sg->addr;
255         t2_desc->fn = sg->fn & 0xffff;
256         t2_desc->cicr = d->cicr;
257         if (!last)
258                 t2_desc->cicr &= ~CICR_BLOCK_IE;
259
260         switch (dir) {
261         case DMA_DEV_TO_MEM:
262                 t2_desc->cdei = sg->ei;
263                 t2_desc->csei = d->ei;
264                 t2_desc->cdfi = sg->fi;
265                 t2_desc->csfi = d->fi;
266
267                 t2_desc->en |= DESC_NXT_DV_REFRESH;
268                 t2_desc->en |= DESC_NXT_SV_REUSE;
269                 break;
270         case DMA_MEM_TO_DEV:
271                 t2_desc->cdei = d->ei;
272                 t2_desc->csei = sg->ei;
273                 t2_desc->cdfi = d->fi;
274                 t2_desc->csfi = sg->fi;
275
276                 t2_desc->en |= DESC_NXT_SV_REFRESH;
277                 t2_desc->en |= DESC_NXT_DV_REUSE;
278                 break;
279         default:
280                 return;
281         }
282
283         t2_desc->en |= DESC_NTYPE_TYPE2;
284 }
285
286 static void omap_dma_write(uint32_t val, unsigned type, void __iomem *addr)
287 {
288         switch (type) {
289         case OMAP_DMA_REG_16BIT:
290                 writew_relaxed(val, addr);
291                 break;
292         case OMAP_DMA_REG_2X16BIT:
293                 writew_relaxed(val, addr);
294                 writew_relaxed(val >> 16, addr + 2);
295                 break;
296         case OMAP_DMA_REG_32BIT:
297                 writel_relaxed(val, addr);
298                 break;
299         default:
300                 WARN_ON(1);
301         }
302 }
303
304 static unsigned omap_dma_read(unsigned type, void __iomem *addr)
305 {
306         unsigned val;
307
308         switch (type) {
309         case OMAP_DMA_REG_16BIT:
310                 val = readw_relaxed(addr);
311                 break;
312         case OMAP_DMA_REG_2X16BIT:
313                 val = readw_relaxed(addr);
314                 val |= readw_relaxed(addr + 2) << 16;
315                 break;
316         case OMAP_DMA_REG_32BIT:
317                 val = readl_relaxed(addr);
318                 break;
319         default:
320                 WARN_ON(1);
321                 val = 0;
322         }
323
324         return val;
325 }
326
327 static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val)
328 {
329         const struct omap_dma_reg *r = od->reg_map + reg;
330
331         WARN_ON(r->stride);
332
333         omap_dma_write(val, r->type, od->base + r->offset);
334 }
335
336 static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg)
337 {
338         const struct omap_dma_reg *r = od->reg_map + reg;
339
340         WARN_ON(r->stride);
341
342         return omap_dma_read(r->type, od->base + r->offset);
343 }
344
345 static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val)
346 {
347         const struct omap_dma_reg *r = c->reg_map + reg;
348
349         omap_dma_write(val, r->type, c->channel_base + r->offset);
350 }
351
352 static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg)
353 {
354         const struct omap_dma_reg *r = c->reg_map + reg;
355
356         return omap_dma_read(r->type, c->channel_base + r->offset);
357 }
358
359 static void omap_dma_clear_csr(struct omap_chan *c)
360 {
361         if (dma_omap1())
362                 omap_dma_chan_read(c, CSR);
363         else
364                 omap_dma_chan_write(c, CSR, ~0);
365 }
366
367 static unsigned omap_dma_get_csr(struct omap_chan *c)
368 {
369         unsigned val = omap_dma_chan_read(c, CSR);
370
371         if (!dma_omap1())
372                 omap_dma_chan_write(c, CSR, val);
373
374         return val;
375 }
376
377 static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c,
378         unsigned lch)
379 {
380         c->channel_base = od->base + od->plat->channel_stride * lch;
381
382         od->lch_map[lch] = c;
383 }
384
385 static void omap_dma_start(struct omap_chan *c, struct omap_desc *d)
386 {
387         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
388         uint16_t cicr = d->cicr;
389
390         if (__dma_omap15xx(od->plat->dma_attr))
391                 omap_dma_chan_write(c, CPC, 0);
392         else
393                 omap_dma_chan_write(c, CDAC, 0);
394
395         omap_dma_clear_csr(c);
396
397         if (d->using_ll) {
398                 uint32_t cdp = CDP_TMODE_LLIST | CDP_NTYPE_TYPE2 | CDP_FAST;
399
400                 if (d->dir == DMA_DEV_TO_MEM)
401                         cdp |= (CDP_DST_VALID_RELOAD | CDP_SRC_VALID_REUSE);
402                 else
403                         cdp |= (CDP_DST_VALID_REUSE | CDP_SRC_VALID_RELOAD);
404                 omap_dma_chan_write(c, CDP, cdp);
405
406                 omap_dma_chan_write(c, CNDP, d->sg[0].t2_desc_paddr);
407                 omap_dma_chan_write(c, CCDN, 0);
408                 omap_dma_chan_write(c, CCFN, 0xffff);
409                 omap_dma_chan_write(c, CCEN, 0xffffff);
410
411                 cicr &= ~CICR_BLOCK_IE;
412         } else if (od->ll123_supported) {
413                 omap_dma_chan_write(c, CDP, 0);
414         }
415
416         /* Enable interrupts */
417         omap_dma_chan_write(c, CICR, cicr);
418
419         /* Enable channel */
420         omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE);
421
422         c->running = true;
423 }
424
425 static void omap_dma_stop(struct omap_chan *c)
426 {
427         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
428         uint32_t val;
429
430         /* disable irq */
431         omap_dma_chan_write(c, CICR, 0);
432
433         omap_dma_clear_csr(c);
434
435         val = omap_dma_chan_read(c, CCR);
436         if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) {
437                 uint32_t sysconfig;
438                 unsigned i;
439
440                 sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG);
441                 val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK;
442                 val |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE);
443                 omap_dma_glbl_write(od, OCP_SYSCONFIG, val);
444
445                 val = omap_dma_chan_read(c, CCR);
446                 val &= ~CCR_ENABLE;
447                 omap_dma_chan_write(c, CCR, val);
448
449                 /* Wait for sDMA FIFO to drain */
450                 for (i = 0; ; i++) {
451                         val = omap_dma_chan_read(c, CCR);
452                         if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)))
453                                 break;
454
455                         if (i > 100)
456                                 break;
457
458                         udelay(5);
459                 }
460
461                 if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))
462                         dev_err(c->vc.chan.device->dev,
463                                 "DMA drain did not complete on lch %d\n",
464                                 c->dma_ch);
465
466                 omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig);
467         } else {
468                 val &= ~CCR_ENABLE;
469                 omap_dma_chan_write(c, CCR, val);
470         }
471
472         mb();
473
474         if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) {
475                 val = omap_dma_chan_read(c, CLNK_CTRL);
476
477                 if (dma_omap1())
478                         val |= 1 << 14; /* set the STOP_LNK bit */
479                 else
480                         val &= ~CLNK_CTRL_ENABLE_LNK;
481
482                 omap_dma_chan_write(c, CLNK_CTRL, val);
483         }
484
485         c->running = false;
486 }
487
488 static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d)
489 {
490         struct omap_sg *sg = d->sg + c->sgidx;
491         unsigned cxsa, cxei, cxfi;
492
493         if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
494                 cxsa = CDSA;
495                 cxei = CDEI;
496                 cxfi = CDFI;
497         } else {
498                 cxsa = CSSA;
499                 cxei = CSEI;
500                 cxfi = CSFI;
501         }
502
503         omap_dma_chan_write(c, cxsa, sg->addr);
504         omap_dma_chan_write(c, cxei, sg->ei);
505         omap_dma_chan_write(c, cxfi, sg->fi);
506         omap_dma_chan_write(c, CEN, sg->en);
507         omap_dma_chan_write(c, CFN, sg->fn);
508
509         omap_dma_start(c, d);
510         c->sgidx++;
511 }
512
513 static void omap_dma_start_desc(struct omap_chan *c)
514 {
515         struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
516         struct omap_desc *d;
517         unsigned cxsa, cxei, cxfi;
518
519         if (!vd) {
520                 c->desc = NULL;
521                 return;
522         }
523
524         list_del(&vd->node);
525
526         c->desc = d = to_omap_dma_desc(&vd->tx);
527         c->sgidx = 0;
528
529         /*
530          * This provides the necessary barrier to ensure data held in
531          * DMA coherent memory is visible to the DMA engine prior to
532          * the transfer starting.
533          */
534         mb();
535
536         omap_dma_chan_write(c, CCR, d->ccr);
537         if (dma_omap1())
538                 omap_dma_chan_write(c, CCR2, d->ccr >> 16);
539
540         if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
541                 cxsa = CSSA;
542                 cxei = CSEI;
543                 cxfi = CSFI;
544         } else {
545                 cxsa = CDSA;
546                 cxei = CDEI;
547                 cxfi = CDFI;
548         }
549
550         omap_dma_chan_write(c, cxsa, d->dev_addr);
551         omap_dma_chan_write(c, cxei, d->ei);
552         omap_dma_chan_write(c, cxfi, d->fi);
553         omap_dma_chan_write(c, CSDP, d->csdp);
554         omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl);
555
556         omap_dma_start_sg(c, d);
557 }
558
559 static void omap_dma_callback(int ch, u16 status, void *data)
560 {
561         struct omap_chan *c = data;
562         struct omap_desc *d;
563         unsigned long flags;
564
565         spin_lock_irqsave(&c->vc.lock, flags);
566         d = c->desc;
567         if (d) {
568                 if (c->cyclic) {
569                         vchan_cyclic_callback(&d->vd);
570                 } else if (d->using_ll || c->sgidx == d->sglen) {
571                         omap_dma_start_desc(c);
572                         vchan_cookie_complete(&d->vd);
573                 } else {
574                         omap_dma_start_sg(c, d);
575                 }
576         }
577         spin_unlock_irqrestore(&c->vc.lock, flags);
578 }
579
580 static irqreturn_t omap_dma_irq(int irq, void *devid)
581 {
582         struct omap_dmadev *od = devid;
583         unsigned status, channel;
584
585         spin_lock(&od->irq_lock);
586
587         status = omap_dma_glbl_read(od, IRQSTATUS_L1);
588         status &= od->irq_enable_mask;
589         if (status == 0) {
590                 spin_unlock(&od->irq_lock);
591                 return IRQ_NONE;
592         }
593
594         while ((channel = ffs(status)) != 0) {
595                 unsigned mask, csr;
596                 struct omap_chan *c;
597
598                 channel -= 1;
599                 mask = BIT(channel);
600                 status &= ~mask;
601
602                 c = od->lch_map[channel];
603                 if (c == NULL) {
604                         /* This should never happen */
605                         dev_err(od->ddev.dev, "invalid channel %u\n", channel);
606                         continue;
607                 }
608
609                 csr = omap_dma_get_csr(c);
610                 omap_dma_glbl_write(od, IRQSTATUS_L1, mask);
611
612                 omap_dma_callback(channel, csr, c);
613         }
614
615         spin_unlock(&od->irq_lock);
616
617         return IRQ_HANDLED;
618 }
619
620 static int omap_dma_alloc_chan_resources(struct dma_chan *chan)
621 {
622         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
623         struct omap_chan *c = to_omap_dma_chan(chan);
624         struct device *dev = od->ddev.dev;
625         int ret;
626
627         if (od->legacy) {
628                 ret = omap_request_dma(c->dma_sig, "DMA engine",
629                                        omap_dma_callback, c, &c->dma_ch);
630         } else {
631                 ret = omap_request_dma(c->dma_sig, "DMA engine", NULL, NULL,
632                                        &c->dma_ch);
633         }
634
635         dev_dbg(dev, "allocating channel %u for %u\n", c->dma_ch, c->dma_sig);
636
637         if (ret >= 0) {
638                 omap_dma_assign(od, c, c->dma_ch);
639
640                 if (!od->legacy) {
641                         unsigned val;
642
643                         spin_lock_irq(&od->irq_lock);
644                         val = BIT(c->dma_ch);
645                         omap_dma_glbl_write(od, IRQSTATUS_L1, val);
646                         od->irq_enable_mask |= val;
647                         omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
648
649                         val = omap_dma_glbl_read(od, IRQENABLE_L0);
650                         val &= ~BIT(c->dma_ch);
651                         omap_dma_glbl_write(od, IRQENABLE_L0, val);
652                         spin_unlock_irq(&od->irq_lock);
653                 }
654         }
655
656         if (dma_omap1()) {
657                 if (__dma_omap16xx(od->plat->dma_attr)) {
658                         c->ccr = CCR_OMAP31_DISABLE;
659                         /* Duplicate what plat-omap/dma.c does */
660                         c->ccr |= c->dma_ch + 1;
661                 } else {
662                         c->ccr = c->dma_sig & 0x1f;
663                 }
664         } else {
665                 c->ccr = c->dma_sig & 0x1f;
666                 c->ccr |= (c->dma_sig & ~0x1f) << 14;
667         }
668         if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING)
669                 c->ccr |= CCR_BUFFERING_DISABLE;
670
671         return ret;
672 }
673
674 static void omap_dma_free_chan_resources(struct dma_chan *chan)
675 {
676         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
677         struct omap_chan *c = to_omap_dma_chan(chan);
678
679         if (!od->legacy) {
680                 spin_lock_irq(&od->irq_lock);
681                 od->irq_enable_mask &= ~BIT(c->dma_ch);
682                 omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
683                 spin_unlock_irq(&od->irq_lock);
684         }
685
686         c->channel_base = NULL;
687         od->lch_map[c->dma_ch] = NULL;
688         vchan_free_chan_resources(&c->vc);
689         omap_free_dma(c->dma_ch);
690
691         dev_dbg(od->ddev.dev, "freeing channel %u used for %u\n", c->dma_ch,
692                 c->dma_sig);
693         c->dma_sig = 0;
694 }
695
696 static size_t omap_dma_sg_size(struct omap_sg *sg)
697 {
698         return sg->en * sg->fn;
699 }
700
701 static size_t omap_dma_desc_size(struct omap_desc *d)
702 {
703         unsigned i;
704         size_t size;
705
706         for (size = i = 0; i < d->sglen; i++)
707                 size += omap_dma_sg_size(&d->sg[i]);
708
709         return size * es_bytes[d->es];
710 }
711
712 static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr)
713 {
714         unsigned i;
715         size_t size, es_size = es_bytes[d->es];
716
717         for (size = i = 0; i < d->sglen; i++) {
718                 size_t this_size = omap_dma_sg_size(&d->sg[i]) * es_size;
719
720                 if (size)
721                         size += this_size;
722                 else if (addr >= d->sg[i].addr &&
723                          addr < d->sg[i].addr + this_size)
724                         size += d->sg[i].addr + this_size - addr;
725         }
726         return size;
727 }
728
729 /*
730  * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
731  * read before the DMA controller finished disabling the channel.
732  */
733 static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg)
734 {
735         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
736         uint32_t val;
737
738         val = omap_dma_chan_read(c, reg);
739         if (val == 0 && od->plat->errata & DMA_ERRATA_3_3)
740                 val = omap_dma_chan_read(c, reg);
741
742         return val;
743 }
744
745 static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c)
746 {
747         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
748         dma_addr_t addr, cdac;
749
750         if (__dma_omap15xx(od->plat->dma_attr)) {
751                 addr = omap_dma_chan_read(c, CPC);
752         } else {
753                 addr = omap_dma_chan_read_3_3(c, CSAC);
754                 cdac = omap_dma_chan_read_3_3(c, CDAC);
755
756                 /*
757                  * CDAC == 0 indicates that the DMA transfer on the channel has
758                  * not been started (no data has been transferred so far).
759                  * Return the programmed source start address in this case.
760                  */
761                 if (cdac == 0)
762                         addr = omap_dma_chan_read(c, CSSA);
763         }
764
765         if (dma_omap1())
766                 addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000;
767
768         return addr;
769 }
770
771 static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c)
772 {
773         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
774         dma_addr_t addr;
775
776         if (__dma_omap15xx(od->plat->dma_attr)) {
777                 addr = omap_dma_chan_read(c, CPC);
778         } else {
779                 addr = omap_dma_chan_read_3_3(c, CDAC);
780
781                 /*
782                  * CDAC == 0 indicates that the DMA transfer on the channel
783                  * has not been started (no data has been transferred so
784                  * far).  Return the programmed destination start address in
785                  * this case.
786                  */
787                 if (addr == 0)
788                         addr = omap_dma_chan_read(c, CDSA);
789         }
790
791         if (dma_omap1())
792                 addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000;
793
794         return addr;
795 }
796
797 static enum dma_status omap_dma_tx_status(struct dma_chan *chan,
798         dma_cookie_t cookie, struct dma_tx_state *txstate)
799 {
800         struct omap_chan *c = to_omap_dma_chan(chan);
801         struct virt_dma_desc *vd;
802         enum dma_status ret;
803         unsigned long flags;
804
805         ret = dma_cookie_status(chan, cookie, txstate);
806
807         if (!c->paused && c->running) {
808                 uint32_t ccr = omap_dma_chan_read(c, CCR);
809                 /*
810                  * The channel is no longer active, set the return value
811                  * accordingly
812                  */
813                 if (!(ccr & CCR_ENABLE))
814                         ret = DMA_COMPLETE;
815         }
816
817         if (ret == DMA_COMPLETE || !txstate)
818                 return ret;
819
820         spin_lock_irqsave(&c->vc.lock, flags);
821         vd = vchan_find_desc(&c->vc, cookie);
822         if (vd) {
823                 txstate->residue = omap_dma_desc_size(to_omap_dma_desc(&vd->tx));
824         } else if (c->desc && c->desc->vd.tx.cookie == cookie) {
825                 struct omap_desc *d = c->desc;
826                 dma_addr_t pos;
827
828                 if (d->dir == DMA_MEM_TO_DEV)
829                         pos = omap_dma_get_src_pos(c);
830                 else if (d->dir == DMA_DEV_TO_MEM  || d->dir == DMA_MEM_TO_MEM)
831                         pos = omap_dma_get_dst_pos(c);
832                 else
833                         pos = 0;
834
835                 txstate->residue = omap_dma_desc_size_pos(d, pos);
836         } else {
837                 txstate->residue = 0;
838         }
839         spin_unlock_irqrestore(&c->vc.lock, flags);
840
841         return ret;
842 }
843
844 static void omap_dma_issue_pending(struct dma_chan *chan)
845 {
846         struct omap_chan *c = to_omap_dma_chan(chan);
847         unsigned long flags;
848
849         spin_lock_irqsave(&c->vc.lock, flags);
850         if (vchan_issue_pending(&c->vc) && !c->desc)
851                 omap_dma_start_desc(c);
852         spin_unlock_irqrestore(&c->vc.lock, flags);
853 }
854
855 static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
856         struct dma_chan *chan, struct scatterlist *sgl, unsigned sglen,
857         enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
858 {
859         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
860         struct omap_chan *c = to_omap_dma_chan(chan);
861         enum dma_slave_buswidth dev_width;
862         struct scatterlist *sgent;
863         struct omap_desc *d;
864         dma_addr_t dev_addr;
865         unsigned i, es, en, frame_bytes;
866         bool ll_failed = false;
867         u32 burst;
868
869         if (dir == DMA_DEV_TO_MEM) {
870                 dev_addr = c->cfg.src_addr;
871                 dev_width = c->cfg.src_addr_width;
872                 burst = c->cfg.src_maxburst;
873         } else if (dir == DMA_MEM_TO_DEV) {
874                 dev_addr = c->cfg.dst_addr;
875                 dev_width = c->cfg.dst_addr_width;
876                 burst = c->cfg.dst_maxburst;
877         } else {
878                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
879                 return NULL;
880         }
881
882         /* Bus width translates to the element size (ES) */
883         switch (dev_width) {
884         case DMA_SLAVE_BUSWIDTH_1_BYTE:
885                 es = CSDP_DATA_TYPE_8;
886                 break;
887         case DMA_SLAVE_BUSWIDTH_2_BYTES:
888                 es = CSDP_DATA_TYPE_16;
889                 break;
890         case DMA_SLAVE_BUSWIDTH_4_BYTES:
891                 es = CSDP_DATA_TYPE_32;
892                 break;
893         default: /* not reached */
894                 return NULL;
895         }
896
897         /* Now allocate and setup the descriptor. */
898         d = kzalloc(sizeof(*d) + sglen * sizeof(d->sg[0]), GFP_ATOMIC);
899         if (!d)
900                 return NULL;
901
902         d->dir = dir;
903         d->dev_addr = dev_addr;
904         d->es = es;
905
906         d->ccr = c->ccr | CCR_SYNC_FRAME;
907         if (dir == DMA_DEV_TO_MEM) {
908                 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT;
909                 d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED;
910         } else {
911                 d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC;
912                 d->csdp = CSDP_SRC_BURST_64 | CSDP_SRC_PACKED;
913         }
914
915         d->cicr = CICR_DROP_IE | CICR_BLOCK_IE;
916         d->csdp |= es;
917
918         if (dma_omap1()) {
919                 d->cicr |= CICR_TOUT_IE;
920
921                 if (dir == DMA_DEV_TO_MEM)
922                         d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_TIPB;
923                 else
924                         d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF;
925         } else {
926                 if (dir == DMA_DEV_TO_MEM)
927                         d->ccr |= CCR_TRIGGER_SRC;
928
929                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
930         }
931         if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS)
932                 d->clnk_ctrl = c->dma_ch;
933
934         /*
935          * Build our scatterlist entries: each contains the address,
936          * the number of elements (EN) in each frame, and the number of
937          * frames (FN).  Number of bytes for this entry = ES * EN * FN.
938          *
939          * Burst size translates to number of elements with frame sync.
940          * Note: DMA engine defines burst to be the number of dev-width
941          * transfers.
942          */
943         en = burst;
944         frame_bytes = es_bytes[es] * en;
945
946         if (sglen >= 2)
947                 d->using_ll = od->ll123_supported;
948
949         for_each_sg(sgl, sgent, sglen, i) {
950                 struct omap_sg *osg = &d->sg[i];
951
952                 osg->addr = sg_dma_address(sgent);
953                 osg->en = en;
954                 osg->fn = sg_dma_len(sgent) / frame_bytes;
955
956                 if (d->using_ll) {
957                         osg->t2_desc = dma_pool_alloc(od->desc_pool, GFP_ATOMIC,
958                                                       &osg->t2_desc_paddr);
959                         if (!osg->t2_desc) {
960                                 dev_err(chan->device->dev,
961                                         "t2_desc[%d] allocation failed\n", i);
962                                 ll_failed = true;
963                                 d->using_ll = false;
964                                 continue;
965                         }
966
967                         omap_dma_fill_type2_desc(d, i, dir, (i == sglen - 1));
968                 }
969         }
970
971         d->sglen = sglen;
972
973         /* Release the dma_pool entries if one allocation failed */
974         if (ll_failed) {
975                 for (i = 0; i < d->sglen; i++) {
976                         struct omap_sg *osg = &d->sg[i];
977
978                         if (osg->t2_desc) {
979                                 dma_pool_free(od->desc_pool, osg->t2_desc,
980                                               osg->t2_desc_paddr);
981                                 osg->t2_desc = NULL;
982                         }
983                 }
984         }
985
986         return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
987 }
988
989 static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic(
990         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
991         size_t period_len, enum dma_transfer_direction dir, unsigned long flags)
992 {
993         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
994         struct omap_chan *c = to_omap_dma_chan(chan);
995         enum dma_slave_buswidth dev_width;
996         struct omap_desc *d;
997         dma_addr_t dev_addr;
998         unsigned es;
999         u32 burst;
1000
1001         if (dir == DMA_DEV_TO_MEM) {
1002                 dev_addr = c->cfg.src_addr;
1003                 dev_width = c->cfg.src_addr_width;
1004                 burst = c->cfg.src_maxburst;
1005         } else if (dir == DMA_MEM_TO_DEV) {
1006                 dev_addr = c->cfg.dst_addr;
1007                 dev_width = c->cfg.dst_addr_width;
1008                 burst = c->cfg.dst_maxburst;
1009         } else {
1010                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
1011                 return NULL;
1012         }
1013
1014         /* Bus width translates to the element size (ES) */
1015         switch (dev_width) {
1016         case DMA_SLAVE_BUSWIDTH_1_BYTE:
1017                 es = CSDP_DATA_TYPE_8;
1018                 break;
1019         case DMA_SLAVE_BUSWIDTH_2_BYTES:
1020                 es = CSDP_DATA_TYPE_16;
1021                 break;
1022         case DMA_SLAVE_BUSWIDTH_4_BYTES:
1023                 es = CSDP_DATA_TYPE_32;
1024                 break;
1025         default: /* not reached */
1026                 return NULL;
1027         }
1028
1029         /* Now allocate and setup the descriptor. */
1030         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1031         if (!d)
1032                 return NULL;
1033
1034         d->dir = dir;
1035         d->dev_addr = dev_addr;
1036         d->fi = burst;
1037         d->es = es;
1038         d->sg[0].addr = buf_addr;
1039         d->sg[0].en = period_len / es_bytes[es];
1040         d->sg[0].fn = buf_len / period_len;
1041         d->sglen = 1;
1042
1043         d->ccr = c->ccr;
1044         if (dir == DMA_DEV_TO_MEM)
1045                 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT;
1046         else
1047                 d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC;
1048
1049         d->cicr = CICR_DROP_IE;
1050         if (flags & DMA_PREP_INTERRUPT)
1051                 d->cicr |= CICR_FRAME_IE;
1052
1053         d->csdp = es;
1054
1055         if (dma_omap1()) {
1056                 d->cicr |= CICR_TOUT_IE;
1057
1058                 if (dir == DMA_DEV_TO_MEM)
1059                         d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_MPUI;
1060                 else
1061                         d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF;
1062         } else {
1063                 if (burst)
1064                         d->ccr |= CCR_SYNC_PACKET;
1065                 else
1066                         d->ccr |= CCR_SYNC_ELEMENT;
1067
1068                 if (dir == DMA_DEV_TO_MEM) {
1069                         d->ccr |= CCR_TRIGGER_SRC;
1070                         d->csdp |= CSDP_DST_PACKED;
1071                 } else {
1072                         d->csdp |= CSDP_SRC_PACKED;
1073                 }
1074
1075                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1076
1077                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1078         }
1079
1080         if (__dma_omap15xx(od->plat->dma_attr))
1081                 d->ccr |= CCR_AUTO_INIT | CCR_REPEAT;
1082         else
1083                 d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK;
1084
1085         c->cyclic = true;
1086
1087         return vchan_tx_prep(&c->vc, &d->vd, flags);
1088 }
1089
1090 static struct dma_async_tx_descriptor *omap_dma_prep_dma_memcpy(
1091         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1092         size_t len, unsigned long tx_flags)
1093 {
1094         struct omap_chan *c = to_omap_dma_chan(chan);
1095         struct omap_desc *d;
1096         uint8_t data_type;
1097
1098         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1099         if (!d)
1100                 return NULL;
1101
1102         data_type = __ffs((src | dest | len));
1103         if (data_type > CSDP_DATA_TYPE_32)
1104                 data_type = CSDP_DATA_TYPE_32;
1105
1106         d->dir = DMA_MEM_TO_MEM;
1107         d->dev_addr = src;
1108         d->fi = 0;
1109         d->es = data_type;
1110         d->sg[0].en = len / BIT(data_type);
1111         d->sg[0].fn = 1;
1112         d->sg[0].addr = dest;
1113         d->sglen = 1;
1114         d->ccr = c->ccr;
1115         d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC;
1116
1117         d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
1118
1119         d->csdp = data_type;
1120
1121         if (dma_omap1()) {
1122                 d->cicr |= CICR_TOUT_IE;
1123                 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1124         } else {
1125                 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1126                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1127                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1128         }
1129
1130         return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
1131 }
1132
1133 static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved(
1134         struct dma_chan *chan, struct dma_interleaved_template *xt,
1135         unsigned long flags)
1136 {
1137         struct omap_chan *c = to_omap_dma_chan(chan);
1138         struct omap_desc *d;
1139         struct omap_sg *sg;
1140         uint8_t data_type;
1141         size_t src_icg, dst_icg;
1142
1143         /* Slave mode is not supported */
1144         if (is_slave_direction(xt->dir))
1145                 return NULL;
1146
1147         if (xt->frame_size != 1 || xt->numf == 0)
1148                 return NULL;
1149
1150         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1151         if (!d)
1152                 return NULL;
1153
1154         data_type = __ffs((xt->src_start | xt->dst_start | xt->sgl[0].size));
1155         if (data_type > CSDP_DATA_TYPE_32)
1156                 data_type = CSDP_DATA_TYPE_32;
1157
1158         sg = &d->sg[0];
1159         d->dir = DMA_MEM_TO_MEM;
1160         d->dev_addr = xt->src_start;
1161         d->es = data_type;
1162         sg->en = xt->sgl[0].size / BIT(data_type);
1163         sg->fn = xt->numf;
1164         sg->addr = xt->dst_start;
1165         d->sglen = 1;
1166         d->ccr = c->ccr;
1167
1168         src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
1169         dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
1170         if (src_icg) {
1171                 d->ccr |= CCR_SRC_AMODE_DBLIDX;
1172                 d->ei = 1;
1173                 d->fi = src_icg;
1174         } else if (xt->src_inc) {
1175                 d->ccr |= CCR_SRC_AMODE_POSTINC;
1176                 d->fi = 0;
1177         } else {
1178                 dev_err(chan->device->dev,
1179                         "%s: SRC constant addressing is not supported\n",
1180                         __func__);
1181                 kfree(d);
1182                 return NULL;
1183         }
1184
1185         if (dst_icg) {
1186                 d->ccr |= CCR_DST_AMODE_DBLIDX;
1187                 sg->ei = 1;
1188                 sg->fi = dst_icg;
1189         } else if (xt->dst_inc) {
1190                 d->ccr |= CCR_DST_AMODE_POSTINC;
1191                 sg->fi = 0;
1192         } else {
1193                 dev_err(chan->device->dev,
1194                         "%s: DST constant addressing is not supported\n",
1195                         __func__);
1196                 kfree(d);
1197                 return NULL;
1198         }
1199
1200         d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
1201
1202         d->csdp = data_type;
1203
1204         if (dma_omap1()) {
1205                 d->cicr |= CICR_TOUT_IE;
1206                 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1207         } else {
1208                 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1209                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1210                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1211         }
1212
1213         return vchan_tx_prep(&c->vc, &d->vd, flags);
1214 }
1215
1216 static int omap_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg)
1217 {
1218         struct omap_chan *c = to_omap_dma_chan(chan);
1219
1220         if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
1221             cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
1222                 return -EINVAL;
1223
1224         memcpy(&c->cfg, cfg, sizeof(c->cfg));
1225
1226         return 0;
1227 }
1228
1229 static int omap_dma_terminate_all(struct dma_chan *chan)
1230 {
1231         struct omap_chan *c = to_omap_dma_chan(chan);
1232         unsigned long flags;
1233         LIST_HEAD(head);
1234
1235         spin_lock_irqsave(&c->vc.lock, flags);
1236
1237         /*
1238          * Stop DMA activity: we assume the callback will not be called
1239          * after omap_dma_stop() returns (even if it does, it will see
1240          * c->desc is NULL and exit.)
1241          */
1242         if (c->desc) {
1243                 omap_dma_desc_free(&c->desc->vd);
1244                 c->desc = NULL;
1245                 /* Avoid stopping the dma twice */
1246                 if (!c->paused)
1247                         omap_dma_stop(c);
1248         }
1249
1250         if (c->cyclic) {
1251                 c->cyclic = false;
1252                 c->paused = false;
1253         }
1254
1255         vchan_get_all_descriptors(&c->vc, &head);
1256         spin_unlock_irqrestore(&c->vc.lock, flags);
1257         vchan_dma_desc_free_list(&c->vc, &head);
1258
1259         return 0;
1260 }
1261
1262 static void omap_dma_synchronize(struct dma_chan *chan)
1263 {
1264         struct omap_chan *c = to_omap_dma_chan(chan);
1265
1266         vchan_synchronize(&c->vc);
1267 }
1268
1269 static int omap_dma_pause(struct dma_chan *chan)
1270 {
1271         struct omap_chan *c = to_omap_dma_chan(chan);
1272
1273         /* Pause/Resume only allowed with cyclic mode */
1274         if (!c->cyclic)
1275                 return -EINVAL;
1276
1277         if (!c->paused) {
1278                 omap_dma_stop(c);
1279                 c->paused = true;
1280         }
1281
1282         return 0;
1283 }
1284
1285 static int omap_dma_resume(struct dma_chan *chan)
1286 {
1287         struct omap_chan *c = to_omap_dma_chan(chan);
1288
1289         /* Pause/Resume only allowed with cyclic mode */
1290         if (!c->cyclic)
1291                 return -EINVAL;
1292
1293         if (c->paused) {
1294                 mb();
1295
1296                 /* Restore channel link register */
1297                 omap_dma_chan_write(c, CLNK_CTRL, c->desc->clnk_ctrl);
1298
1299                 omap_dma_start(c, c->desc);
1300                 c->paused = false;
1301         }
1302
1303         return 0;
1304 }
1305
1306 static int omap_dma_chan_init(struct omap_dmadev *od)
1307 {
1308         struct omap_chan *c;
1309
1310         c = kzalloc(sizeof(*c), GFP_KERNEL);
1311         if (!c)
1312                 return -ENOMEM;
1313
1314         c->reg_map = od->reg_map;
1315         c->vc.desc_free = omap_dma_desc_free;
1316         vchan_init(&c->vc, &od->ddev);
1317
1318         return 0;
1319 }
1320
1321 static void omap_dma_free(struct omap_dmadev *od)
1322 {
1323         while (!list_empty(&od->ddev.channels)) {
1324                 struct omap_chan *c = list_first_entry(&od->ddev.channels,
1325                         struct omap_chan, vc.chan.device_node);
1326
1327                 list_del(&c->vc.chan.device_node);
1328                 tasklet_kill(&c->vc.task);
1329                 kfree(c);
1330         }
1331 }
1332
1333 #define OMAP_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
1334                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
1335                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
1336
1337 static int omap_dma_probe(struct platform_device *pdev)
1338 {
1339         struct omap_dmadev *od;
1340         struct resource *res;
1341         int rc, i, irq;
1342
1343         od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
1344         if (!od)
1345                 return -ENOMEM;
1346
1347         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1348         od->base = devm_ioremap_resource(&pdev->dev, res);
1349         if (IS_ERR(od->base))
1350                 return PTR_ERR(od->base);
1351
1352         od->plat = omap_get_plat_info();
1353         if (!od->plat)
1354                 return -EPROBE_DEFER;
1355
1356         od->reg_map = od->plat->reg_map;
1357
1358         dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
1359         dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);
1360         dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);
1361         dma_cap_set(DMA_INTERLEAVE, od->ddev.cap_mask);
1362         od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources;
1363         od->ddev.device_free_chan_resources = omap_dma_free_chan_resources;
1364         od->ddev.device_tx_status = omap_dma_tx_status;
1365         od->ddev.device_issue_pending = omap_dma_issue_pending;
1366         od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg;
1367         od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic;
1368         od->ddev.device_prep_dma_memcpy = omap_dma_prep_dma_memcpy;
1369         od->ddev.device_prep_interleaved_dma = omap_dma_prep_dma_interleaved;
1370         od->ddev.device_config = omap_dma_slave_config;
1371         od->ddev.device_pause = omap_dma_pause;
1372         od->ddev.device_resume = omap_dma_resume;
1373         od->ddev.device_terminate_all = omap_dma_terminate_all;
1374         od->ddev.device_synchronize = omap_dma_synchronize;
1375         od->ddev.src_addr_widths = OMAP_DMA_BUSWIDTHS;
1376         od->ddev.dst_addr_widths = OMAP_DMA_BUSWIDTHS;
1377         od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1378         od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1379         od->ddev.dev = &pdev->dev;
1380         INIT_LIST_HEAD(&od->ddev.channels);
1381         spin_lock_init(&od->lock);
1382         spin_lock_init(&od->irq_lock);
1383
1384         if (!pdev->dev.of_node) {
1385                 od->dma_requests = od->plat->dma_attr->lch_count;
1386                 if (unlikely(!od->dma_requests))
1387                         od->dma_requests = OMAP_SDMA_REQUESTS;
1388         } else if (of_property_read_u32(pdev->dev.of_node, "dma-requests",
1389                                         &od->dma_requests)) {
1390                 dev_info(&pdev->dev,
1391                          "Missing dma-requests property, using %u.\n",
1392                          OMAP_SDMA_REQUESTS);
1393                 od->dma_requests = OMAP_SDMA_REQUESTS;
1394         }
1395
1396         od->lch_map = devm_kcalloc(&pdev->dev, od->dma_requests,
1397                                    sizeof(*od->lch_map), GFP_KERNEL);
1398         if (!od->lch_map)
1399                 return -ENOMEM;
1400
1401         for (i = 0; i < od->dma_requests; i++) {
1402                 rc = omap_dma_chan_init(od);
1403                 if (rc) {
1404                         omap_dma_free(od);
1405                         return rc;
1406                 }
1407         }
1408
1409         irq = platform_get_irq(pdev, 1);
1410         if (irq <= 0) {
1411                 dev_info(&pdev->dev, "failed to get L1 IRQ: %d\n", irq);
1412                 od->legacy = true;
1413         } else {
1414                 /* Disable all interrupts */
1415                 od->irq_enable_mask = 0;
1416                 omap_dma_glbl_write(od, IRQENABLE_L1, 0);
1417
1418                 rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq,
1419                                       IRQF_SHARED, "omap-dma-engine", od);
1420                 if (rc)
1421                         return rc;
1422         }
1423
1424         if (omap_dma_glbl_read(od, CAPS_0) & CAPS_0_SUPPORT_LL123)
1425                 od->ll123_supported = true;
1426
1427         od->ddev.filter.map = od->plat->slave_map;
1428         od->ddev.filter.mapcnt = od->plat->slavecnt;
1429         od->ddev.filter.fn = omap_dma_filter_fn;
1430
1431         if (od->ll123_supported) {
1432                 od->desc_pool = dma_pool_create(dev_name(&pdev->dev),
1433                                                 &pdev->dev,
1434                                                 sizeof(struct omap_type2_desc),
1435                                                 4, 0);
1436                 if (!od->desc_pool) {
1437                         dev_err(&pdev->dev,
1438                                 "unable to allocate descriptor pool\n");
1439                         od->ll123_supported = false;
1440                 }
1441         }
1442
1443         rc = dma_async_device_register(&od->ddev);
1444         if (rc) {
1445                 pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n",
1446                         rc);
1447                 omap_dma_free(od);
1448                 return rc;
1449         }
1450
1451         platform_set_drvdata(pdev, od);
1452
1453         if (pdev->dev.of_node) {
1454                 omap_dma_info.dma_cap = od->ddev.cap_mask;
1455
1456                 /* Device-tree DMA controller registration */
1457                 rc = of_dma_controller_register(pdev->dev.of_node,
1458                                 of_dma_simple_xlate, &omap_dma_info);
1459                 if (rc) {
1460                         pr_warn("OMAP-DMA: failed to register DMA controller\n");
1461                         dma_async_device_unregister(&od->ddev);
1462                         omap_dma_free(od);
1463                 }
1464         }
1465
1466         dev_info(&pdev->dev, "OMAP DMA engine driver%s\n",
1467                  od->ll123_supported ? " (LinkedList1/2/3 supported)" : "");
1468
1469         return rc;
1470 }
1471
1472 static int omap_dma_remove(struct platform_device *pdev)
1473 {
1474         struct omap_dmadev *od = platform_get_drvdata(pdev);
1475         int irq;
1476
1477         if (pdev->dev.of_node)
1478                 of_dma_controller_free(pdev->dev.of_node);
1479
1480         irq = platform_get_irq(pdev, 1);
1481         devm_free_irq(&pdev->dev, irq, od);
1482
1483         dma_async_device_unregister(&od->ddev);
1484
1485         if (!od->legacy) {
1486                 /* Disable all interrupts */
1487                 omap_dma_glbl_write(od, IRQENABLE_L0, 0);
1488         }
1489
1490         if (od->ll123_supported)
1491                 dma_pool_destroy(od->desc_pool);
1492
1493         omap_dma_free(od);
1494
1495         return 0;
1496 }
1497
1498 static const struct of_device_id omap_dma_match[] = {
1499         { .compatible = "ti,omap2420-sdma", },
1500         { .compatible = "ti,omap2430-sdma", },
1501         { .compatible = "ti,omap3430-sdma", },
1502         { .compatible = "ti,omap3630-sdma", },
1503         { .compatible = "ti,omap4430-sdma", },
1504         {},
1505 };
1506 MODULE_DEVICE_TABLE(of, omap_dma_match);
1507
1508 static struct platform_driver omap_dma_driver = {
1509         .probe  = omap_dma_probe,
1510         .remove = omap_dma_remove,
1511         .driver = {
1512                 .name = "omap-dma-engine",
1513                 .of_match_table = of_match_ptr(omap_dma_match),
1514         },
1515 };
1516
1517 bool omap_dma_filter_fn(struct dma_chan *chan, void *param)
1518 {
1519         if (chan->device->dev->driver == &omap_dma_driver.driver) {
1520                 struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1521                 struct omap_chan *c = to_omap_dma_chan(chan);
1522                 unsigned req = *(unsigned *)param;
1523
1524                 if (req <= od->dma_requests) {
1525                         c->dma_sig = req;
1526                         return true;
1527                 }
1528         }
1529         return false;
1530 }
1531 EXPORT_SYMBOL_GPL(omap_dma_filter_fn);
1532
1533 static int omap_dma_init(void)
1534 {
1535         return platform_driver_register(&omap_dma_driver);
1536 }
1537 subsys_initcall(omap_dma_init);
1538
1539 static void __exit omap_dma_exit(void)
1540 {
1541         platform_driver_unregister(&omap_dma_driver);
1542 }
1543 module_exit(omap_dma_exit);
1544
1545 MODULE_AUTHOR("Russell King");
1546 MODULE_LICENSE("GPL");