Merge tag 'modules-for-v4.16' of git://git.kernel.org/pub/scm/linux/kernel/git/jeyu...
[linux-2.6-microblaze.git] / drivers / dma / dma-jz4780.c
1 /*
2  * Ingenic JZ4780 DMA controller
3  *
4  * Copyright (c) 2015 Imagination Technologies
5  * Author: Alex Smith <alex@alex-smith.me.uk>
6  *
7  * This program is free software; you can redistribute it and/or modify it
8  * under the terms of the GNU General Public License as published by the
9  * Free Software Foundation;  either version 2 of the  License, or (at your
10  * option) any later version.
11  */
12
13 #include <linux/clk.h>
14 #include <linux/dmapool.h>
15 #include <linux/init.h>
16 #include <linux/interrupt.h>
17 #include <linux/module.h>
18 #include <linux/of.h>
19 #include <linux/of_dma.h>
20 #include <linux/platform_device.h>
21 #include <linux/slab.h>
22
23 #include "dmaengine.h"
24 #include "virt-dma.h"
25
26 #define JZ_DMA_NR_CHANNELS      32
27
28 /* Global registers. */
29 #define JZ_DMA_REG_DMAC         0x1000
30 #define JZ_DMA_REG_DIRQP        0x1004
31 #define JZ_DMA_REG_DDR          0x1008
32 #define JZ_DMA_REG_DDRS         0x100c
33 #define JZ_DMA_REG_DMACP        0x101c
34 #define JZ_DMA_REG_DSIRQP       0x1020
35 #define JZ_DMA_REG_DSIRQM       0x1024
36 #define JZ_DMA_REG_DCIRQP       0x1028
37 #define JZ_DMA_REG_DCIRQM       0x102c
38
39 /* Per-channel registers. */
40 #define JZ_DMA_REG_CHAN(n)      (n * 0x20)
41 #define JZ_DMA_REG_DSA(n)       (0x00 + JZ_DMA_REG_CHAN(n))
42 #define JZ_DMA_REG_DTA(n)       (0x04 + JZ_DMA_REG_CHAN(n))
43 #define JZ_DMA_REG_DTC(n)       (0x08 + JZ_DMA_REG_CHAN(n))
44 #define JZ_DMA_REG_DRT(n)       (0x0c + JZ_DMA_REG_CHAN(n))
45 #define JZ_DMA_REG_DCS(n)       (0x10 + JZ_DMA_REG_CHAN(n))
46 #define JZ_DMA_REG_DCM(n)       (0x14 + JZ_DMA_REG_CHAN(n))
47 #define JZ_DMA_REG_DDA(n)       (0x18 + JZ_DMA_REG_CHAN(n))
48 #define JZ_DMA_REG_DSD(n)       (0x1c + JZ_DMA_REG_CHAN(n))
49
50 #define JZ_DMA_DMAC_DMAE        BIT(0)
51 #define JZ_DMA_DMAC_AR          BIT(2)
52 #define JZ_DMA_DMAC_HLT         BIT(3)
53 #define JZ_DMA_DMAC_FMSC        BIT(31)
54
55 #define JZ_DMA_DRT_AUTO         0x8
56
57 #define JZ_DMA_DCS_CTE          BIT(0)
58 #define JZ_DMA_DCS_HLT          BIT(2)
59 #define JZ_DMA_DCS_TT           BIT(3)
60 #define JZ_DMA_DCS_AR           BIT(4)
61 #define JZ_DMA_DCS_DES8         BIT(30)
62
63 #define JZ_DMA_DCM_LINK         BIT(0)
64 #define JZ_DMA_DCM_TIE          BIT(1)
65 #define JZ_DMA_DCM_STDE         BIT(2)
66 #define JZ_DMA_DCM_TSZ_SHIFT    8
67 #define JZ_DMA_DCM_TSZ_MASK     (0x7 << JZ_DMA_DCM_TSZ_SHIFT)
68 #define JZ_DMA_DCM_DP_SHIFT     12
69 #define JZ_DMA_DCM_SP_SHIFT     14
70 #define JZ_DMA_DCM_DAI          BIT(22)
71 #define JZ_DMA_DCM_SAI          BIT(23)
72
73 #define JZ_DMA_SIZE_4_BYTE      0x0
74 #define JZ_DMA_SIZE_1_BYTE      0x1
75 #define JZ_DMA_SIZE_2_BYTE      0x2
76 #define JZ_DMA_SIZE_16_BYTE     0x3
77 #define JZ_DMA_SIZE_32_BYTE     0x4
78 #define JZ_DMA_SIZE_64_BYTE     0x5
79 #define JZ_DMA_SIZE_128_BYTE    0x6
80
81 #define JZ_DMA_WIDTH_32_BIT     0x0
82 #define JZ_DMA_WIDTH_8_BIT      0x1
83 #define JZ_DMA_WIDTH_16_BIT     0x2
84
85 #define JZ_DMA_BUSWIDTHS        (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE)  | \
86                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
87                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
88
89 /**
90  * struct jz4780_dma_hwdesc - descriptor structure read by the DMA controller.
91  * @dcm: value for the DCM (channel command) register
92  * @dsa: source address
93  * @dta: target address
94  * @dtc: transfer count (number of blocks of the transfer size specified in DCM
95  * to transfer) in the low 24 bits, offset of the next descriptor from the
96  * descriptor base address in the upper 8 bits.
97  * @sd: target/source stride difference (in stride transfer mode).
98  * @drt: request type
99  */
100 struct jz4780_dma_hwdesc {
101         uint32_t dcm;
102         uint32_t dsa;
103         uint32_t dta;
104         uint32_t dtc;
105         uint32_t sd;
106         uint32_t drt;
107         uint32_t reserved[2];
108 };
109
110 /* Size of allocations for hardware descriptor blocks. */
111 #define JZ_DMA_DESC_BLOCK_SIZE  PAGE_SIZE
112 #define JZ_DMA_MAX_DESC         \
113         (JZ_DMA_DESC_BLOCK_SIZE / sizeof(struct jz4780_dma_hwdesc))
114
115 struct jz4780_dma_desc {
116         struct virt_dma_desc vdesc;
117
118         struct jz4780_dma_hwdesc *desc;
119         dma_addr_t desc_phys;
120         unsigned int count;
121         enum dma_transaction_type type;
122         uint32_t status;
123 };
124
125 struct jz4780_dma_chan {
126         struct virt_dma_chan vchan;
127         unsigned int id;
128         struct dma_pool *desc_pool;
129
130         uint32_t transfer_type;
131         uint32_t transfer_shift;
132         struct dma_slave_config config;
133
134         struct jz4780_dma_desc *desc;
135         unsigned int curr_hwdesc;
136 };
137
138 struct jz4780_dma_dev {
139         struct dma_device dma_device;
140         void __iomem *base;
141         struct clk *clk;
142         unsigned int irq;
143
144         uint32_t chan_reserved;
145         struct jz4780_dma_chan chan[JZ_DMA_NR_CHANNELS];
146 };
147
148 struct jz4780_dma_filter_data {
149         struct device_node *of_node;
150         uint32_t transfer_type;
151         int channel;
152 };
153
154 static inline struct jz4780_dma_chan *to_jz4780_dma_chan(struct dma_chan *chan)
155 {
156         return container_of(chan, struct jz4780_dma_chan, vchan.chan);
157 }
158
159 static inline struct jz4780_dma_desc *to_jz4780_dma_desc(
160         struct virt_dma_desc *vdesc)
161 {
162         return container_of(vdesc, struct jz4780_dma_desc, vdesc);
163 }
164
165 static inline struct jz4780_dma_dev *jz4780_dma_chan_parent(
166         struct jz4780_dma_chan *jzchan)
167 {
168         return container_of(jzchan->vchan.chan.device, struct jz4780_dma_dev,
169                             dma_device);
170 }
171
172 static inline uint32_t jz4780_dma_readl(struct jz4780_dma_dev *jzdma,
173         unsigned int reg)
174 {
175         return readl(jzdma->base + reg);
176 }
177
178 static inline void jz4780_dma_writel(struct jz4780_dma_dev *jzdma,
179         unsigned int reg, uint32_t val)
180 {
181         writel(val, jzdma->base + reg);
182 }
183
184 static struct jz4780_dma_desc *jz4780_dma_desc_alloc(
185         struct jz4780_dma_chan *jzchan, unsigned int count,
186         enum dma_transaction_type type)
187 {
188         struct jz4780_dma_desc *desc;
189
190         if (count > JZ_DMA_MAX_DESC)
191                 return NULL;
192
193         desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
194         if (!desc)
195                 return NULL;
196
197         desc->desc = dma_pool_alloc(jzchan->desc_pool, GFP_NOWAIT,
198                                     &desc->desc_phys);
199         if (!desc->desc) {
200                 kfree(desc);
201                 return NULL;
202         }
203
204         desc->count = count;
205         desc->type = type;
206         return desc;
207 }
208
209 static void jz4780_dma_desc_free(struct virt_dma_desc *vdesc)
210 {
211         struct jz4780_dma_desc *desc = to_jz4780_dma_desc(vdesc);
212         struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(vdesc->tx.chan);
213
214         dma_pool_free(jzchan->desc_pool, desc->desc, desc->desc_phys);
215         kfree(desc);
216 }
217
218 static uint32_t jz4780_dma_transfer_size(unsigned long val, uint32_t *shift)
219 {
220         int ord = ffs(val) - 1;
221
222         /*
223          * 8 byte transfer sizes unsupported so fall back on 4. If it's larger
224          * than the maximum, just limit it. It is perfectly safe to fall back
225          * in this way since we won't exceed the maximum burst size supported
226          * by the device, the only effect is reduced efficiency. This is better
227          * than refusing to perform the request at all.
228          */
229         if (ord == 3)
230                 ord = 2;
231         else if (ord > 7)
232                 ord = 7;
233
234         *shift = ord;
235
236         switch (ord) {
237         case 0:
238                 return JZ_DMA_SIZE_1_BYTE;
239         case 1:
240                 return JZ_DMA_SIZE_2_BYTE;
241         case 2:
242                 return JZ_DMA_SIZE_4_BYTE;
243         case 4:
244                 return JZ_DMA_SIZE_16_BYTE;
245         case 5:
246                 return JZ_DMA_SIZE_32_BYTE;
247         case 6:
248                 return JZ_DMA_SIZE_64_BYTE;
249         default:
250                 return JZ_DMA_SIZE_128_BYTE;
251         }
252 }
253
254 static int jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan,
255         struct jz4780_dma_hwdesc *desc, dma_addr_t addr, size_t len,
256         enum dma_transfer_direction direction)
257 {
258         struct dma_slave_config *config = &jzchan->config;
259         uint32_t width, maxburst, tsz;
260
261         if (direction == DMA_MEM_TO_DEV) {
262                 desc->dcm = JZ_DMA_DCM_SAI;
263                 desc->dsa = addr;
264                 desc->dta = config->dst_addr;
265                 desc->drt = jzchan->transfer_type;
266
267                 width = config->dst_addr_width;
268                 maxburst = config->dst_maxburst;
269         } else {
270                 desc->dcm = JZ_DMA_DCM_DAI;
271                 desc->dsa = config->src_addr;
272                 desc->dta = addr;
273                 desc->drt = jzchan->transfer_type;
274
275                 width = config->src_addr_width;
276                 maxburst = config->src_maxburst;
277         }
278
279         /*
280          * This calculates the maximum transfer size that can be used with the
281          * given address, length, width and maximum burst size. The address
282          * must be aligned to the transfer size, the total length must be
283          * divisible by the transfer size, and we must not use more than the
284          * maximum burst specified by the user.
285          */
286         tsz = jz4780_dma_transfer_size(addr | len | (width * maxburst),
287                                        &jzchan->transfer_shift);
288
289         switch (width) {
290         case DMA_SLAVE_BUSWIDTH_1_BYTE:
291         case DMA_SLAVE_BUSWIDTH_2_BYTES:
292                 break;
293         case DMA_SLAVE_BUSWIDTH_4_BYTES:
294                 width = JZ_DMA_WIDTH_32_BIT;
295                 break;
296         default:
297                 return -EINVAL;
298         }
299
300         desc->dcm |= tsz << JZ_DMA_DCM_TSZ_SHIFT;
301         desc->dcm |= width << JZ_DMA_DCM_SP_SHIFT;
302         desc->dcm |= width << JZ_DMA_DCM_DP_SHIFT;
303
304         desc->dtc = len >> jzchan->transfer_shift;
305         return 0;
306 }
307
308 static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg(
309         struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len,
310         enum dma_transfer_direction direction, unsigned long flags,
311         void *context)
312 {
313         struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
314         struct jz4780_dma_desc *desc;
315         unsigned int i;
316         int err;
317
318         desc = jz4780_dma_desc_alloc(jzchan, sg_len, DMA_SLAVE);
319         if (!desc)
320                 return NULL;
321
322         for (i = 0; i < sg_len; i++) {
323                 err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i],
324                                               sg_dma_address(&sgl[i]),
325                                               sg_dma_len(&sgl[i]),
326                                               direction);
327                 if (err < 0) {
328                         jz4780_dma_desc_free(&jzchan->desc->vdesc);
329                         return NULL;
330                 }
331
332                 desc->desc[i].dcm |= JZ_DMA_DCM_TIE;
333
334                 if (i != (sg_len - 1)) {
335                         /* Automatically proceeed to the next descriptor. */
336                         desc->desc[i].dcm |= JZ_DMA_DCM_LINK;
337
338                         /*
339                          * The upper 8 bits of the DTC field in the descriptor
340                          * must be set to (offset from descriptor base of next
341                          * descriptor >> 4).
342                          */
343                         desc->desc[i].dtc |=
344                                 (((i + 1) * sizeof(*desc->desc)) >> 4) << 24;
345                 }
346         }
347
348         return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags);
349 }
350
351 static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_cyclic(
352         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
353         size_t period_len, enum dma_transfer_direction direction,
354         unsigned long flags)
355 {
356         struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
357         struct jz4780_dma_desc *desc;
358         unsigned int periods, i;
359         int err;
360
361         if (buf_len % period_len)
362                 return NULL;
363
364         periods = buf_len / period_len;
365
366         desc = jz4780_dma_desc_alloc(jzchan, periods, DMA_CYCLIC);
367         if (!desc)
368                 return NULL;
369
370         for (i = 0; i < periods; i++) {
371                 err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], buf_addr,
372                                               period_len, direction);
373                 if (err < 0) {
374                         jz4780_dma_desc_free(&jzchan->desc->vdesc);
375                         return NULL;
376                 }
377
378                 buf_addr += period_len;
379
380                 /*
381                  * Set the link bit to indicate that the controller should
382                  * automatically proceed to the next descriptor. In
383                  * jz4780_dma_begin(), this will be cleared if we need to issue
384                  * an interrupt after each period.
385                  */
386                 desc->desc[i].dcm |= JZ_DMA_DCM_TIE | JZ_DMA_DCM_LINK;
387
388                 /*
389                  * The upper 8 bits of the DTC field in the descriptor must be
390                  * set to (offset from descriptor base of next descriptor >> 4).
391                  * If this is the last descriptor, link it back to the first,
392                  * i.e. leave offset set to 0, otherwise point to the next one.
393                  */
394                 if (i != (periods - 1)) {
395                         desc->desc[i].dtc |=
396                                 (((i + 1) * sizeof(*desc->desc)) >> 4) << 24;
397                 }
398         }
399
400         return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags);
401 }
402
403 static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_memcpy(
404         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
405         size_t len, unsigned long flags)
406 {
407         struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
408         struct jz4780_dma_desc *desc;
409         uint32_t tsz;
410
411         desc = jz4780_dma_desc_alloc(jzchan, 1, DMA_MEMCPY);
412         if (!desc)
413                 return NULL;
414
415         tsz = jz4780_dma_transfer_size(dest | src | len,
416                                        &jzchan->transfer_shift);
417
418         desc->desc[0].dsa = src;
419         desc->desc[0].dta = dest;
420         desc->desc[0].drt = JZ_DMA_DRT_AUTO;
421         desc->desc[0].dcm = JZ_DMA_DCM_TIE | JZ_DMA_DCM_SAI | JZ_DMA_DCM_DAI |
422                             tsz << JZ_DMA_DCM_TSZ_SHIFT |
423                             JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_SP_SHIFT |
424                             JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_DP_SHIFT;
425         desc->desc[0].dtc = len >> jzchan->transfer_shift;
426
427         return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags);
428 }
429
430 static void jz4780_dma_begin(struct jz4780_dma_chan *jzchan)
431 {
432         struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan);
433         struct virt_dma_desc *vdesc;
434         unsigned int i;
435         dma_addr_t desc_phys;
436
437         if (!jzchan->desc) {
438                 vdesc = vchan_next_desc(&jzchan->vchan);
439                 if (!vdesc)
440                         return;
441
442                 list_del(&vdesc->node);
443
444                 jzchan->desc = to_jz4780_dma_desc(vdesc);
445                 jzchan->curr_hwdesc = 0;
446
447                 if (jzchan->desc->type == DMA_CYCLIC && vdesc->tx.callback) {
448                         /*
449                          * The DMA controller doesn't support triggering an
450                          * interrupt after processing each descriptor, only
451                          * after processing an entire terminated list of
452                          * descriptors. For a cyclic DMA setup the list of
453                          * descriptors is not terminated so we can never get an
454                          * interrupt.
455                          *
456                          * If the user requested a callback for a cyclic DMA
457                          * setup then we workaround this hardware limitation
458                          * here by degrading to a set of unlinked descriptors
459                          * which we will submit in sequence in response to the
460                          * completion of processing the previous descriptor.
461                          */
462                         for (i = 0; i < jzchan->desc->count; i++)
463                                 jzchan->desc->desc[i].dcm &= ~JZ_DMA_DCM_LINK;
464                 }
465         } else {
466                 /*
467                  * There is an existing transfer, therefore this must be one
468                  * for which we unlinked the descriptors above. Advance to the
469                  * next one in the list.
470                  */
471                 jzchan->curr_hwdesc =
472                         (jzchan->curr_hwdesc + 1) % jzchan->desc->count;
473         }
474
475         /* Use 8-word descriptors. */
476         jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id), JZ_DMA_DCS_DES8);
477
478         /* Write descriptor address and initiate descriptor fetch. */
479         desc_phys = jzchan->desc->desc_phys +
480                     (jzchan->curr_hwdesc * sizeof(*jzchan->desc->desc));
481         jz4780_dma_writel(jzdma, JZ_DMA_REG_DDA(jzchan->id), desc_phys);
482         jz4780_dma_writel(jzdma, JZ_DMA_REG_DDRS, BIT(jzchan->id));
483
484         /* Enable the channel. */
485         jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id),
486                           JZ_DMA_DCS_DES8 | JZ_DMA_DCS_CTE);
487 }
488
489 static void jz4780_dma_issue_pending(struct dma_chan *chan)
490 {
491         struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
492         unsigned long flags;
493
494         spin_lock_irqsave(&jzchan->vchan.lock, flags);
495
496         if (vchan_issue_pending(&jzchan->vchan) && !jzchan->desc)
497                 jz4780_dma_begin(jzchan);
498
499         spin_unlock_irqrestore(&jzchan->vchan.lock, flags);
500 }
501
502 static int jz4780_dma_terminate_all(struct dma_chan *chan)
503 {
504         struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
505         struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan);
506         unsigned long flags;
507         LIST_HEAD(head);
508
509         spin_lock_irqsave(&jzchan->vchan.lock, flags);
510
511         /* Clear the DMA status and stop the transfer. */
512         jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id), 0);
513         if (jzchan->desc) {
514                 vchan_terminate_vdesc(&jzchan->desc->vdesc);
515                 jzchan->desc = NULL;
516         }
517
518         vchan_get_all_descriptors(&jzchan->vchan, &head);
519
520         spin_unlock_irqrestore(&jzchan->vchan.lock, flags);
521
522         vchan_dma_desc_free_list(&jzchan->vchan, &head);
523         return 0;
524 }
525
526 static void jz4780_dma_synchronize(struct dma_chan *chan)
527 {
528         struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
529
530         vchan_synchronize(&jzchan->vchan);
531 }
532
533 static int jz4780_dma_config(struct dma_chan *chan,
534         struct dma_slave_config *config)
535 {
536         struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
537
538         if ((config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
539            || (config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES))
540                 return -EINVAL;
541
542         /* Copy the reset of the slave configuration, it is used later. */
543         memcpy(&jzchan->config, config, sizeof(jzchan->config));
544
545         return 0;
546 }
547
548 static size_t jz4780_dma_desc_residue(struct jz4780_dma_chan *jzchan,
549         struct jz4780_dma_desc *desc, unsigned int next_sg)
550 {
551         struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan);
552         unsigned int residue, count;
553         unsigned int i;
554
555         residue = 0;
556
557         for (i = next_sg; i < desc->count; i++)
558                 residue += desc->desc[i].dtc << jzchan->transfer_shift;
559
560         if (next_sg != 0) {
561                 count = jz4780_dma_readl(jzdma,
562                                          JZ_DMA_REG_DTC(jzchan->id));
563                 residue += count << jzchan->transfer_shift;
564         }
565
566         return residue;
567 }
568
569 static enum dma_status jz4780_dma_tx_status(struct dma_chan *chan,
570         dma_cookie_t cookie, struct dma_tx_state *txstate)
571 {
572         struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
573         struct virt_dma_desc *vdesc;
574         enum dma_status status;
575         unsigned long flags;
576
577         status = dma_cookie_status(chan, cookie, txstate);
578         if ((status == DMA_COMPLETE) || (txstate == NULL))
579                 return status;
580
581         spin_lock_irqsave(&jzchan->vchan.lock, flags);
582
583         vdesc = vchan_find_desc(&jzchan->vchan, cookie);
584         if (vdesc) {
585                 /* On the issued list, so hasn't been processed yet */
586                 txstate->residue = jz4780_dma_desc_residue(jzchan,
587                                         to_jz4780_dma_desc(vdesc), 0);
588         } else if (cookie == jzchan->desc->vdesc.tx.cookie) {
589                 txstate->residue = jz4780_dma_desc_residue(jzchan, jzchan->desc,
590                           (jzchan->curr_hwdesc + 1) % jzchan->desc->count);
591         } else
592                 txstate->residue = 0;
593
594         if (vdesc && jzchan->desc && vdesc == &jzchan->desc->vdesc
595             && jzchan->desc->status & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT))
596                 status = DMA_ERROR;
597
598         spin_unlock_irqrestore(&jzchan->vchan.lock, flags);
599         return status;
600 }
601
602 static void jz4780_dma_chan_irq(struct jz4780_dma_dev *jzdma,
603         struct jz4780_dma_chan *jzchan)
604 {
605         uint32_t dcs;
606
607         spin_lock(&jzchan->vchan.lock);
608
609         dcs = jz4780_dma_readl(jzdma, JZ_DMA_REG_DCS(jzchan->id));
610         jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id), 0);
611
612         if (dcs & JZ_DMA_DCS_AR) {
613                 dev_warn(&jzchan->vchan.chan.dev->device,
614                          "address error (DCS=0x%x)\n", dcs);
615         }
616
617         if (dcs & JZ_DMA_DCS_HLT) {
618                 dev_warn(&jzchan->vchan.chan.dev->device,
619                          "channel halt (DCS=0x%x)\n", dcs);
620         }
621
622         if (jzchan->desc) {
623                 jzchan->desc->status = dcs;
624
625                 if ((dcs & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) == 0) {
626                         if (jzchan->desc->type == DMA_CYCLIC) {
627                                 vchan_cyclic_callback(&jzchan->desc->vdesc);
628                         } else {
629                                 vchan_cookie_complete(&jzchan->desc->vdesc);
630                                 jzchan->desc = NULL;
631                         }
632
633                         jz4780_dma_begin(jzchan);
634                 }
635         } else {
636                 dev_err(&jzchan->vchan.chan.dev->device,
637                         "channel IRQ with no active transfer\n");
638         }
639
640         spin_unlock(&jzchan->vchan.lock);
641 }
642
643 static irqreturn_t jz4780_dma_irq_handler(int irq, void *data)
644 {
645         struct jz4780_dma_dev *jzdma = data;
646         uint32_t pending, dmac;
647         int i;
648
649         pending = jz4780_dma_readl(jzdma, JZ_DMA_REG_DIRQP);
650
651         for (i = 0; i < JZ_DMA_NR_CHANNELS; i++) {
652                 if (!(pending & (1<<i)))
653                         continue;
654
655                 jz4780_dma_chan_irq(jzdma, &jzdma->chan[i]);
656         }
657
658         /* Clear halt and address error status of all channels. */
659         dmac = jz4780_dma_readl(jzdma, JZ_DMA_REG_DMAC);
660         dmac &= ~(JZ_DMA_DMAC_HLT | JZ_DMA_DMAC_AR);
661         jz4780_dma_writel(jzdma, JZ_DMA_REG_DMAC, dmac);
662
663         /* Clear interrupt pending status. */
664         jz4780_dma_writel(jzdma, JZ_DMA_REG_DIRQP, 0);
665
666         return IRQ_HANDLED;
667 }
668
669 static int jz4780_dma_alloc_chan_resources(struct dma_chan *chan)
670 {
671         struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
672
673         jzchan->desc_pool = dma_pool_create(dev_name(&chan->dev->device),
674                                             chan->device->dev,
675                                             JZ_DMA_DESC_BLOCK_SIZE,
676                                             PAGE_SIZE, 0);
677         if (!jzchan->desc_pool) {
678                 dev_err(&chan->dev->device,
679                         "failed to allocate descriptor pool\n");
680                 return -ENOMEM;
681         }
682
683         return 0;
684 }
685
686 static void jz4780_dma_free_chan_resources(struct dma_chan *chan)
687 {
688         struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
689
690         vchan_free_chan_resources(&jzchan->vchan);
691         dma_pool_destroy(jzchan->desc_pool);
692         jzchan->desc_pool = NULL;
693 }
694
695 static bool jz4780_dma_filter_fn(struct dma_chan *chan, void *param)
696 {
697         struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
698         struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan);
699         struct jz4780_dma_filter_data *data = param;
700
701         if (jzdma->dma_device.dev->of_node != data->of_node)
702                 return false;
703
704         if (data->channel > -1) {
705                 if (data->channel != jzchan->id)
706                         return false;
707         } else if (jzdma->chan_reserved & BIT(jzchan->id)) {
708                 return false;
709         }
710
711         jzchan->transfer_type = data->transfer_type;
712
713         return true;
714 }
715
716 static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec,
717         struct of_dma *ofdma)
718 {
719         struct jz4780_dma_dev *jzdma = ofdma->of_dma_data;
720         dma_cap_mask_t mask = jzdma->dma_device.cap_mask;
721         struct jz4780_dma_filter_data data;
722
723         if (dma_spec->args_count != 2)
724                 return NULL;
725
726         data.of_node = ofdma->of_node;
727         data.transfer_type = dma_spec->args[0];
728         data.channel = dma_spec->args[1];
729
730         if (data.channel > -1) {
731                 if (data.channel >= JZ_DMA_NR_CHANNELS) {
732                         dev_err(jzdma->dma_device.dev,
733                                 "device requested non-existent channel %u\n",
734                                 data.channel);
735                         return NULL;
736                 }
737
738                 /* Can only select a channel marked as reserved. */
739                 if (!(jzdma->chan_reserved & BIT(data.channel))) {
740                         dev_err(jzdma->dma_device.dev,
741                                 "device requested unreserved channel %u\n",
742                                 data.channel);
743                         return NULL;
744                 }
745
746                 jzdma->chan[data.channel].transfer_type = data.transfer_type;
747
748                 return dma_get_slave_channel(
749                         &jzdma->chan[data.channel].vchan.chan);
750         } else {
751                 return dma_request_channel(mask, jz4780_dma_filter_fn, &data);
752         }
753 }
754
755 static int jz4780_dma_probe(struct platform_device *pdev)
756 {
757         struct device *dev = &pdev->dev;
758         struct jz4780_dma_dev *jzdma;
759         struct jz4780_dma_chan *jzchan;
760         struct dma_device *dd;
761         struct resource *res;
762         int i, ret;
763
764         jzdma = devm_kzalloc(dev, sizeof(*jzdma), GFP_KERNEL);
765         if (!jzdma)
766                 return -ENOMEM;
767
768         platform_set_drvdata(pdev, jzdma);
769
770         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
771         if (!res) {
772                 dev_err(dev, "failed to get I/O memory\n");
773                 return -EINVAL;
774         }
775
776         jzdma->base = devm_ioremap_resource(dev, res);
777         if (IS_ERR(jzdma->base))
778                 return PTR_ERR(jzdma->base);
779
780         ret = platform_get_irq(pdev, 0);
781         if (ret < 0) {
782                 dev_err(dev, "failed to get IRQ: %d\n", ret);
783                 return ret;
784         }
785
786         jzdma->irq = ret;
787
788         ret = request_irq(jzdma->irq, jz4780_dma_irq_handler, 0, dev_name(dev),
789                           jzdma);
790         if (ret) {
791                 dev_err(dev, "failed to request IRQ %u!\n", jzdma->irq);
792                 return ret;
793         }
794
795         jzdma->clk = devm_clk_get(dev, NULL);
796         if (IS_ERR(jzdma->clk)) {
797                 dev_err(dev, "failed to get clock\n");
798                 ret = PTR_ERR(jzdma->clk);
799                 goto err_free_irq;
800         }
801
802         clk_prepare_enable(jzdma->clk);
803
804         /* Property is optional, if it doesn't exist the value will remain 0. */
805         of_property_read_u32_index(dev->of_node, "ingenic,reserved-channels",
806                                    0, &jzdma->chan_reserved);
807
808         dd = &jzdma->dma_device;
809
810         dma_cap_set(DMA_MEMCPY, dd->cap_mask);
811         dma_cap_set(DMA_SLAVE, dd->cap_mask);
812         dma_cap_set(DMA_CYCLIC, dd->cap_mask);
813
814         dd->dev = dev;
815         dd->copy_align = DMAENGINE_ALIGN_4_BYTES;
816         dd->device_alloc_chan_resources = jz4780_dma_alloc_chan_resources;
817         dd->device_free_chan_resources = jz4780_dma_free_chan_resources;
818         dd->device_prep_slave_sg = jz4780_dma_prep_slave_sg;
819         dd->device_prep_dma_cyclic = jz4780_dma_prep_dma_cyclic;
820         dd->device_prep_dma_memcpy = jz4780_dma_prep_dma_memcpy;
821         dd->device_config = jz4780_dma_config;
822         dd->device_terminate_all = jz4780_dma_terminate_all;
823         dd->device_synchronize = jz4780_dma_synchronize;
824         dd->device_tx_status = jz4780_dma_tx_status;
825         dd->device_issue_pending = jz4780_dma_issue_pending;
826         dd->src_addr_widths = JZ_DMA_BUSWIDTHS;
827         dd->dst_addr_widths = JZ_DMA_BUSWIDTHS;
828         dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
829         dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
830
831         /*
832          * Enable DMA controller, mark all channels as not programmable.
833          * Also set the FMSC bit - it increases MSC performance, so it makes
834          * little sense not to enable it.
835          */
836         jz4780_dma_writel(jzdma, JZ_DMA_REG_DMAC,
837                           JZ_DMA_DMAC_DMAE | JZ_DMA_DMAC_FMSC);
838         jz4780_dma_writel(jzdma, JZ_DMA_REG_DMACP, 0);
839
840         INIT_LIST_HEAD(&dd->channels);
841
842         for (i = 0; i < JZ_DMA_NR_CHANNELS; i++) {
843                 jzchan = &jzdma->chan[i];
844                 jzchan->id = i;
845
846                 vchan_init(&jzchan->vchan, dd);
847                 jzchan->vchan.desc_free = jz4780_dma_desc_free;
848         }
849
850         ret = dma_async_device_register(dd);
851         if (ret) {
852                 dev_err(dev, "failed to register device\n");
853                 goto err_disable_clk;
854         }
855
856         /* Register with OF DMA helpers. */
857         ret = of_dma_controller_register(dev->of_node, jz4780_of_dma_xlate,
858                                          jzdma);
859         if (ret) {
860                 dev_err(dev, "failed to register OF DMA controller\n");
861                 goto err_unregister_dev;
862         }
863
864         dev_info(dev, "JZ4780 DMA controller initialised\n");
865         return 0;
866
867 err_unregister_dev:
868         dma_async_device_unregister(dd);
869
870 err_disable_clk:
871         clk_disable_unprepare(jzdma->clk);
872
873 err_free_irq:
874         free_irq(jzdma->irq, jzdma);
875         return ret;
876 }
877
878 static int jz4780_dma_remove(struct platform_device *pdev)
879 {
880         struct jz4780_dma_dev *jzdma = platform_get_drvdata(pdev);
881         int i;
882
883         of_dma_controller_free(pdev->dev.of_node);
884
885         free_irq(jzdma->irq, jzdma);
886
887         for (i = 0; i < JZ_DMA_NR_CHANNELS; i++)
888                 tasklet_kill(&jzdma->chan[i].vchan.task);
889
890         dma_async_device_unregister(&jzdma->dma_device);
891         return 0;
892 }
893
894 static const struct of_device_id jz4780_dma_dt_match[] = {
895         { .compatible = "ingenic,jz4780-dma", .data = NULL },
896         {},
897 };
898 MODULE_DEVICE_TABLE(of, jz4780_dma_dt_match);
899
900 static struct platform_driver jz4780_dma_driver = {
901         .probe          = jz4780_dma_probe,
902         .remove         = jz4780_dma_remove,
903         .driver = {
904                 .name   = "jz4780-dma",
905                 .of_match_table = of_match_ptr(jz4780_dma_dt_match),
906         },
907 };
908
909 static int __init jz4780_dma_init(void)
910 {
911         return platform_driver_register(&jz4780_dma_driver);
912 }
913 subsys_initcall(jz4780_dma_init);
914
915 static void __exit jz4780_dma_exit(void)
916 {
917         platform_driver_unregister(&jz4780_dma_driver);
918 }
919 module_exit(jz4780_dma_exit);
920
921 MODULE_AUTHOR("Alex Smith <alex@alex-smith.me.uk>");
922 MODULE_DESCRIPTION("Ingenic JZ4780 DMA controller driver");
923 MODULE_LICENSE("GPL");