3 * Copyright (C) STMicroelectronics SA 2017
4 * Author(s): M'boumba Cedric Madianga <cedric.madianga@gmail.com>
5 * Pierre-Yves Mordret <pierre-yves.mordret@st.com>
7 * License terms: GPL V2.0.
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms of the GNU General Public License version 2 as published by
11 * the Free Software Foundation.
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
18 * Driver for STM32 MDMA controller
20 * Inspired by stm32-dma.c and dma-jz4780.c
24 #include <linux/clk.h>
25 #include <linux/delay.h>
26 #include <linux/dmaengine.h>
27 #include <linux/dma-mapping.h>
28 #include <linux/dmapool.h>
29 #include <linux/err.h>
30 #include <linux/init.h>
31 #include <linux/iopoll.h>
32 #include <linux/jiffies.h>
33 #include <linux/list.h>
34 #include <linux/log2.h>
35 #include <linux/module.h>
37 #include <linux/of_device.h>
38 #include <linux/of_dma.h>
39 #include <linux/platform_device.h>
40 #include <linux/pm_runtime.h>
41 #include <linux/reset.h>
42 #include <linux/slab.h>
46 /* MDMA Generic getter/setter */
47 #define STM32_MDMA_SHIFT(n) (ffs(n) - 1)
48 #define STM32_MDMA_SET(n, mask) (((n) << STM32_MDMA_SHIFT(mask)) & \
50 #define STM32_MDMA_GET(n, mask) (((n) & (mask)) >> \
51 STM32_MDMA_SHIFT(mask))
53 #define STM32_MDMA_GISR0 0x0000 /* MDMA Int Status Reg 1 */
54 #define STM32_MDMA_GISR1 0x0004 /* MDMA Int Status Reg 2 */
56 /* MDMA Channel x interrupt/status register */
57 #define STM32_MDMA_CISR(x) (0x40 + 0x40 * (x)) /* x = 0..62 */
58 #define STM32_MDMA_CISR_CRQA BIT(16)
59 #define STM32_MDMA_CISR_TCIF BIT(4)
60 #define STM32_MDMA_CISR_BTIF BIT(3)
61 #define STM32_MDMA_CISR_BRTIF BIT(2)
62 #define STM32_MDMA_CISR_CTCIF BIT(1)
63 #define STM32_MDMA_CISR_TEIF BIT(0)
65 /* MDMA Channel x interrupt flag clear register */
66 #define STM32_MDMA_CIFCR(x) (0x44 + 0x40 * (x))
67 #define STM32_MDMA_CIFCR_CLTCIF BIT(4)
68 #define STM32_MDMA_CIFCR_CBTIF BIT(3)
69 #define STM32_MDMA_CIFCR_CBRTIF BIT(2)
70 #define STM32_MDMA_CIFCR_CCTCIF BIT(1)
71 #define STM32_MDMA_CIFCR_CTEIF BIT(0)
72 #define STM32_MDMA_CIFCR_CLEAR_ALL (STM32_MDMA_CIFCR_CLTCIF \
73 | STM32_MDMA_CIFCR_CBTIF \
74 | STM32_MDMA_CIFCR_CBRTIF \
75 | STM32_MDMA_CIFCR_CCTCIF \
76 | STM32_MDMA_CIFCR_CTEIF)
78 /* MDMA Channel x error status register */
79 #define STM32_MDMA_CESR(x) (0x48 + 0x40 * (x))
80 #define STM32_MDMA_CESR_BSE BIT(11)
81 #define STM32_MDMA_CESR_ASR BIT(10)
82 #define STM32_MDMA_CESR_TEMD BIT(9)
83 #define STM32_MDMA_CESR_TELD BIT(8)
84 #define STM32_MDMA_CESR_TED BIT(7)
85 #define STM32_MDMA_CESR_TEA_MASK GENMASK(6, 0)
87 /* MDMA Channel x control register */
88 #define STM32_MDMA_CCR(x) (0x4C + 0x40 * (x))
89 #define STM32_MDMA_CCR_SWRQ BIT(16)
90 #define STM32_MDMA_CCR_WEX BIT(14)
91 #define STM32_MDMA_CCR_HEX BIT(13)
92 #define STM32_MDMA_CCR_BEX BIT(12)
93 #define STM32_MDMA_CCR_PL_MASK GENMASK(7, 6)
94 #define STM32_MDMA_CCR_PL(n) STM32_MDMA_SET(n, \
95 STM32_MDMA_CCR_PL_MASK)
96 #define STM32_MDMA_CCR_TCIE BIT(5)
97 #define STM32_MDMA_CCR_BTIE BIT(4)
98 #define STM32_MDMA_CCR_BRTIE BIT(3)
99 #define STM32_MDMA_CCR_CTCIE BIT(2)
100 #define STM32_MDMA_CCR_TEIE BIT(1)
101 #define STM32_MDMA_CCR_EN BIT(0)
102 #define STM32_MDMA_CCR_IRQ_MASK (STM32_MDMA_CCR_TCIE \
103 | STM32_MDMA_CCR_BTIE \
104 | STM32_MDMA_CCR_BRTIE \
105 | STM32_MDMA_CCR_CTCIE \
106 | STM32_MDMA_CCR_TEIE)
108 /* MDMA Channel x transfer configuration register */
109 #define STM32_MDMA_CTCR(x) (0x50 + 0x40 * (x))
110 #define STM32_MDMA_CTCR_BWM BIT(31)
111 #define STM32_MDMA_CTCR_SWRM BIT(30)
112 #define STM32_MDMA_CTCR_TRGM_MSK GENMASK(29, 28)
113 #define STM32_MDMA_CTCR_TRGM(n) STM32_MDMA_SET((n), \
114 STM32_MDMA_CTCR_TRGM_MSK)
115 #define STM32_MDMA_CTCR_TRGM_GET(n) STM32_MDMA_GET((n), \
116 STM32_MDMA_CTCR_TRGM_MSK)
117 #define STM32_MDMA_CTCR_PAM_MASK GENMASK(27, 26)
118 #define STM32_MDMA_CTCR_PAM(n) STM32_MDMA_SET(n, \
119 STM32_MDMA_CTCR_PAM_MASK)
120 #define STM32_MDMA_CTCR_PKE BIT(25)
121 #define STM32_MDMA_CTCR_TLEN_MSK GENMASK(24, 18)
122 #define STM32_MDMA_CTCR_TLEN(n) STM32_MDMA_SET((n), \
123 STM32_MDMA_CTCR_TLEN_MSK)
124 #define STM32_MDMA_CTCR_TLEN_GET(n) STM32_MDMA_GET((n), \
125 STM32_MDMA_CTCR_TLEN_MSK)
126 #define STM32_MDMA_CTCR_LEN2_MSK GENMASK(25, 18)
127 #define STM32_MDMA_CTCR_LEN2(n) STM32_MDMA_SET((n), \
128 STM32_MDMA_CTCR_LEN2_MSK)
129 #define STM32_MDMA_CTCR_LEN2_GET(n) STM32_MDMA_GET((n), \
130 STM32_MDMA_CTCR_LEN2_MSK)
131 #define STM32_MDMA_CTCR_DBURST_MASK GENMASK(17, 15)
132 #define STM32_MDMA_CTCR_DBURST(n) STM32_MDMA_SET(n, \
133 STM32_MDMA_CTCR_DBURST_MASK)
134 #define STM32_MDMA_CTCR_SBURST_MASK GENMASK(14, 12)
135 #define STM32_MDMA_CTCR_SBURST(n) STM32_MDMA_SET(n, \
136 STM32_MDMA_CTCR_SBURST_MASK)
137 #define STM32_MDMA_CTCR_DINCOS_MASK GENMASK(11, 10)
138 #define STM32_MDMA_CTCR_DINCOS(n) STM32_MDMA_SET((n), \
139 STM32_MDMA_CTCR_DINCOS_MASK)
140 #define STM32_MDMA_CTCR_SINCOS_MASK GENMASK(9, 8)
141 #define STM32_MDMA_CTCR_SINCOS(n) STM32_MDMA_SET((n), \
142 STM32_MDMA_CTCR_SINCOS_MASK)
143 #define STM32_MDMA_CTCR_DSIZE_MASK GENMASK(7, 6)
144 #define STM32_MDMA_CTCR_DSIZE(n) STM32_MDMA_SET(n, \
145 STM32_MDMA_CTCR_DSIZE_MASK)
146 #define STM32_MDMA_CTCR_SSIZE_MASK GENMASK(5, 4)
147 #define STM32_MDMA_CTCR_SSIZE(n) STM32_MDMA_SET(n, \
148 STM32_MDMA_CTCR_SSIZE_MASK)
149 #define STM32_MDMA_CTCR_DINC_MASK GENMASK(3, 2)
150 #define STM32_MDMA_CTCR_DINC(n) STM32_MDMA_SET((n), \
151 STM32_MDMA_CTCR_DINC_MASK)
152 #define STM32_MDMA_CTCR_SINC_MASK GENMASK(1, 0)
153 #define STM32_MDMA_CTCR_SINC(n) STM32_MDMA_SET((n), \
154 STM32_MDMA_CTCR_SINC_MASK)
155 #define STM32_MDMA_CTCR_CFG_MASK (STM32_MDMA_CTCR_SINC_MASK \
156 | STM32_MDMA_CTCR_DINC_MASK \
157 | STM32_MDMA_CTCR_SINCOS_MASK \
158 | STM32_MDMA_CTCR_DINCOS_MASK \
159 | STM32_MDMA_CTCR_LEN2_MSK \
160 | STM32_MDMA_CTCR_TRGM_MSK)
162 /* MDMA Channel x block number of data register */
163 #define STM32_MDMA_CBNDTR(x) (0x54 + 0x40 * (x))
164 #define STM32_MDMA_CBNDTR_BRC_MK GENMASK(31, 20)
165 #define STM32_MDMA_CBNDTR_BRC(n) STM32_MDMA_SET(n, \
166 STM32_MDMA_CBNDTR_BRC_MK)
167 #define STM32_MDMA_CBNDTR_BRC_GET(n) STM32_MDMA_GET((n), \
168 STM32_MDMA_CBNDTR_BRC_MK)
170 #define STM32_MDMA_CBNDTR_BRDUM BIT(19)
171 #define STM32_MDMA_CBNDTR_BRSUM BIT(18)
172 #define STM32_MDMA_CBNDTR_BNDT_MASK GENMASK(16, 0)
173 #define STM32_MDMA_CBNDTR_BNDT(n) STM32_MDMA_SET(n, \
174 STM32_MDMA_CBNDTR_BNDT_MASK)
176 /* MDMA Channel x source address register */
177 #define STM32_MDMA_CSAR(x) (0x58 + 0x40 * (x))
179 /* MDMA Channel x destination address register */
180 #define STM32_MDMA_CDAR(x) (0x5C + 0x40 * (x))
182 /* MDMA Channel x block repeat address update register */
183 #define STM32_MDMA_CBRUR(x) (0x60 + 0x40 * (x))
184 #define STM32_MDMA_CBRUR_DUV_MASK GENMASK(31, 16)
185 #define STM32_MDMA_CBRUR_DUV(n) STM32_MDMA_SET(n, \
186 STM32_MDMA_CBRUR_DUV_MASK)
187 #define STM32_MDMA_CBRUR_SUV_MASK GENMASK(15, 0)
188 #define STM32_MDMA_CBRUR_SUV(n) STM32_MDMA_SET(n, \
189 STM32_MDMA_CBRUR_SUV_MASK)
191 /* MDMA Channel x link address register */
192 #define STM32_MDMA_CLAR(x) (0x64 + 0x40 * (x))
194 /* MDMA Channel x trigger and bus selection register */
195 #define STM32_MDMA_CTBR(x) (0x68 + 0x40 * (x))
196 #define STM32_MDMA_CTBR_DBUS BIT(17)
197 #define STM32_MDMA_CTBR_SBUS BIT(16)
198 #define STM32_MDMA_CTBR_TSEL_MASK GENMASK(7, 0)
199 #define STM32_MDMA_CTBR_TSEL(n) STM32_MDMA_SET(n, \
200 STM32_MDMA_CTBR_TSEL_MASK)
202 /* MDMA Channel x mask address register */
203 #define STM32_MDMA_CMAR(x) (0x70 + 0x40 * (x))
205 /* MDMA Channel x mask data register */
206 #define STM32_MDMA_CMDR(x) (0x74 + 0x40 * (x))
208 #define STM32_MDMA_MAX_BUF_LEN 128
209 #define STM32_MDMA_MAX_BLOCK_LEN 65536
210 #define STM32_MDMA_MAX_CHANNELS 63
211 #define STM32_MDMA_MAX_REQUESTS 256
212 #define STM32_MDMA_MAX_BURST 128
213 #define STM32_MDMA_VERY_HIGH_PRIORITY 0x11
215 enum stm32_mdma_trigger_mode {
218 STM32_MDMA_BLOCK_REP,
219 STM32_MDMA_LINKED_LIST,
222 enum stm32_mdma_width {
224 STM32_MDMA_HALF_WORD,
226 STM32_MDMA_DOUBLE_WORD,
229 enum stm32_mdma_inc_mode {
230 STM32_MDMA_FIXED = 0,
235 struct stm32_mdma_chan_config {
243 struct stm32_mdma_hwdesc {
256 struct stm32_mdma_desc_node {
257 struct stm32_mdma_hwdesc *hwdesc;
258 dma_addr_t hwdesc_phys;
261 struct stm32_mdma_desc {
262 struct virt_dma_desc vdesc;
266 struct stm32_mdma_desc_node node[];
269 struct stm32_mdma_chan {
270 struct virt_dma_chan vchan;
271 struct dma_pool *desc_pool;
273 struct stm32_mdma_desc *desc;
275 struct dma_slave_config dma_config;
276 struct stm32_mdma_chan_config chan_config;
282 struct stm32_mdma_device {
283 struct dma_device ddev;
287 struct reset_control *rst;
290 u32 nr_ahb_addr_masks;
291 struct stm32_mdma_chan chan[STM32_MDMA_MAX_CHANNELS];
292 u32 ahb_addr_masks[];
295 static struct stm32_mdma_device *stm32_mdma_get_dev(
296 struct stm32_mdma_chan *chan)
298 return container_of(chan->vchan.chan.device, struct stm32_mdma_device,
302 static struct stm32_mdma_chan *to_stm32_mdma_chan(struct dma_chan *c)
304 return container_of(c, struct stm32_mdma_chan, vchan.chan);
307 static struct stm32_mdma_desc *to_stm32_mdma_desc(struct virt_dma_desc *vdesc)
309 return container_of(vdesc, struct stm32_mdma_desc, vdesc);
312 static struct device *chan2dev(struct stm32_mdma_chan *chan)
314 return &chan->vchan.chan.dev->device;
317 static struct device *mdma2dev(struct stm32_mdma_device *mdma_dev)
319 return mdma_dev->ddev.dev;
322 static u32 stm32_mdma_read(struct stm32_mdma_device *dmadev, u32 reg)
324 return readl_relaxed(dmadev->base + reg);
327 static void stm32_mdma_write(struct stm32_mdma_device *dmadev, u32 reg, u32 val)
329 writel_relaxed(val, dmadev->base + reg);
332 static void stm32_mdma_set_bits(struct stm32_mdma_device *dmadev, u32 reg,
335 void __iomem *addr = dmadev->base + reg;
337 writel_relaxed(readl_relaxed(addr) | mask, addr);
340 static void stm32_mdma_clr_bits(struct stm32_mdma_device *dmadev, u32 reg,
343 void __iomem *addr = dmadev->base + reg;
345 writel_relaxed(readl_relaxed(addr) & ~mask, addr);
348 static struct stm32_mdma_desc *stm32_mdma_alloc_desc(
349 struct stm32_mdma_chan *chan, u32 count)
351 struct stm32_mdma_desc *desc;
354 desc = kzalloc(offsetof(typeof(*desc), node[count]), GFP_NOWAIT);
358 for (i = 0; i < count; i++) {
359 desc->node[i].hwdesc =
360 dma_pool_alloc(chan->desc_pool, GFP_NOWAIT,
361 &desc->node[i].hwdesc_phys);
362 if (!desc->node[i].hwdesc)
371 dev_err(chan2dev(chan), "Failed to allocate descriptor\n");
373 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,
374 desc->node[i].hwdesc_phys);
379 static void stm32_mdma_desc_free(struct virt_dma_desc *vdesc)
381 struct stm32_mdma_desc *desc = to_stm32_mdma_desc(vdesc);
382 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(vdesc->tx.chan);
385 for (i = 0; i < desc->count; i++)
386 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,
387 desc->node[i].hwdesc_phys);
391 static int stm32_mdma_get_width(struct stm32_mdma_chan *chan,
392 enum dma_slave_buswidth width)
395 case DMA_SLAVE_BUSWIDTH_1_BYTE:
396 case DMA_SLAVE_BUSWIDTH_2_BYTES:
397 case DMA_SLAVE_BUSWIDTH_4_BYTES:
398 case DMA_SLAVE_BUSWIDTH_8_BYTES:
399 return ffs(width) - 1;
401 dev_err(chan2dev(chan), "Dma bus width %i not supported\n",
407 static enum dma_slave_buswidth stm32_mdma_get_max_width(dma_addr_t addr,
408 u32 buf_len, u32 tlen)
410 enum dma_slave_buswidth max_width = DMA_SLAVE_BUSWIDTH_8_BYTES;
412 for (max_width = DMA_SLAVE_BUSWIDTH_8_BYTES;
413 max_width > DMA_SLAVE_BUSWIDTH_1_BYTE;
416 * Address and buffer length both have to be aligned on
419 if ((((buf_len | addr) & (max_width - 1)) == 0) &&
427 static u32 stm32_mdma_get_best_burst(u32 buf_len, u32 tlen, u32 max_burst,
428 enum dma_slave_buswidth width)
432 best_burst = min((u32)1 << __ffs(tlen | buf_len),
433 max_burst * width) / width;
435 return (best_burst > 0) ? best_burst : 1;
438 static int stm32_mdma_disable_chan(struct stm32_mdma_chan *chan)
440 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
441 u32 ccr, cisr, id, reg;
445 reg = STM32_MDMA_CCR(id);
447 /* Disable interrupts */
448 stm32_mdma_clr_bits(dmadev, reg, STM32_MDMA_CCR_IRQ_MASK);
450 ccr = stm32_mdma_read(dmadev, reg);
451 if (ccr & STM32_MDMA_CCR_EN) {
452 stm32_mdma_clr_bits(dmadev, reg, STM32_MDMA_CCR_EN);
454 /* Ensure that any ongoing transfer has been completed */
455 ret = readl_relaxed_poll_timeout_atomic(
456 dmadev->base + STM32_MDMA_CISR(id), cisr,
457 (cisr & STM32_MDMA_CISR_CTCIF), 10, 1000);
459 dev_err(chan2dev(chan), "%s: timeout!\n", __func__);
467 static void stm32_mdma_stop(struct stm32_mdma_chan *chan)
469 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
474 ret = stm32_mdma_disable_chan(chan);
478 /* Clear interrupt status if it is there */
479 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id));
481 dev_dbg(chan2dev(chan), "%s(): clearing interrupt: 0x%08x\n",
483 stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(chan->id), status);
489 static void stm32_mdma_set_bus(struct stm32_mdma_device *dmadev, u32 *ctbr,
490 u32 ctbr_mask, u32 src_addr)
495 /* Check if memory device is on AHB or AXI */
497 mask = src_addr & 0xF0000000;
498 for (i = 0; i < dmadev->nr_ahb_addr_masks; i++) {
499 if (mask == dmadev->ahb_addr_masks[i]) {
506 static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan,
507 enum dma_transfer_direction direction,
508 u32 *mdma_ccr, u32 *mdma_ctcr,
509 u32 *mdma_ctbr, dma_addr_t addr,
512 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
513 struct stm32_mdma_chan_config *chan_config = &chan->chan_config;
514 enum dma_slave_buswidth src_addr_width, dst_addr_width;
515 phys_addr_t src_addr, dst_addr;
516 int src_bus_width, dst_bus_width;
517 u32 src_maxburst, dst_maxburst, src_best_burst, dst_best_burst;
518 u32 ccr, ctcr, ctbr, tlen;
520 src_addr_width = chan->dma_config.src_addr_width;
521 dst_addr_width = chan->dma_config.dst_addr_width;
522 src_maxburst = chan->dma_config.src_maxburst;
523 dst_maxburst = chan->dma_config.dst_maxburst;
525 ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id));
526 ctcr = stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id));
527 ctbr = stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id));
529 /* Enable HW request mode */
530 ctcr &= ~STM32_MDMA_CTCR_SWRM;
532 /* Set DINC, SINC, DINCOS, SINCOS, TRGM and TLEN retrieve from DT */
533 ctcr &= ~STM32_MDMA_CTCR_CFG_MASK;
534 ctcr |= chan_config->transfer_config & STM32_MDMA_CTCR_CFG_MASK;
537 * For buffer transfer length (TLEN) we have to set
538 * the number of bytes - 1 in CTCR register
540 tlen = STM32_MDMA_CTCR_LEN2_GET(ctcr);
541 ctcr &= ~STM32_MDMA_CTCR_LEN2_MSK;
542 ctcr |= STM32_MDMA_CTCR_TLEN((tlen - 1));
544 /* Disable Pack Enable */
545 ctcr &= ~STM32_MDMA_CTCR_PKE;
547 /* Check burst size constraints */
548 if (src_maxburst * src_addr_width > STM32_MDMA_MAX_BURST ||
549 dst_maxburst * dst_addr_width > STM32_MDMA_MAX_BURST) {
550 dev_err(chan2dev(chan),
551 "burst size * bus width higher than %d bytes\n",
552 STM32_MDMA_MAX_BURST);
556 if ((!is_power_of_2(src_maxburst) && src_maxburst > 0) ||
557 (!is_power_of_2(dst_maxburst) && dst_maxburst > 0)) {
558 dev_err(chan2dev(chan), "burst size must be a power of 2\n");
563 * Configure channel control:
564 * - Clear SW request as in this case this is a HW one
565 * - Clear WEX, HEX and BEX bits
566 * - Set priority level
568 ccr &= ~(STM32_MDMA_CCR_SWRQ | STM32_MDMA_CCR_WEX | STM32_MDMA_CCR_HEX |
569 STM32_MDMA_CCR_BEX | STM32_MDMA_CCR_PL_MASK);
570 ccr |= STM32_MDMA_CCR_PL(chan_config->priority_level);
572 /* Configure Trigger selection */
573 ctbr &= ~STM32_MDMA_CTBR_TSEL_MASK;
574 ctbr |= STM32_MDMA_CTBR_TSEL(chan_config->request);
578 dst_addr = chan->dma_config.dst_addr;
580 /* Set device data size */
581 dst_bus_width = stm32_mdma_get_width(chan, dst_addr_width);
582 if (dst_bus_width < 0)
583 return dst_bus_width;
584 ctcr &= ~STM32_MDMA_CTCR_DSIZE_MASK;
585 ctcr |= STM32_MDMA_CTCR_DSIZE(dst_bus_width);
587 /* Set device burst value */
588 dst_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,
591 chan->mem_burst = dst_best_burst;
592 ctcr &= ~STM32_MDMA_CTCR_DBURST_MASK;
593 ctcr |= STM32_MDMA_CTCR_DBURST((ilog2(dst_best_burst)));
595 /* Set memory data size */
596 src_addr_width = stm32_mdma_get_max_width(addr, buf_len, tlen);
597 chan->mem_width = src_addr_width;
598 src_bus_width = stm32_mdma_get_width(chan, src_addr_width);
599 if (src_bus_width < 0)
600 return src_bus_width;
601 ctcr &= ~STM32_MDMA_CTCR_SSIZE_MASK |
602 STM32_MDMA_CTCR_SINCOS_MASK;
603 ctcr |= STM32_MDMA_CTCR_SSIZE(src_bus_width) |
604 STM32_MDMA_CTCR_SINCOS(src_bus_width);
606 /* Set memory burst value */
607 src_maxburst = STM32_MDMA_MAX_BUF_LEN / src_addr_width;
608 src_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,
611 chan->mem_burst = src_best_burst;
612 ctcr &= ~STM32_MDMA_CTCR_SBURST_MASK;
613 ctcr |= STM32_MDMA_CTCR_SBURST((ilog2(src_best_burst)));
616 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS,
619 if (dst_bus_width != src_bus_width)
620 ctcr |= STM32_MDMA_CTCR_PKE;
622 /* Set destination address */
623 stm32_mdma_write(dmadev, STM32_MDMA_CDAR(chan->id), dst_addr);
627 src_addr = chan->dma_config.src_addr;
629 /* Set device data size */
630 src_bus_width = stm32_mdma_get_width(chan, src_addr_width);
631 if (src_bus_width < 0)
632 return src_bus_width;
633 ctcr &= ~STM32_MDMA_CTCR_SSIZE_MASK;
634 ctcr |= STM32_MDMA_CTCR_SSIZE(src_bus_width);
636 /* Set device burst value */
637 src_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,
640 ctcr &= ~STM32_MDMA_CTCR_SBURST_MASK;
641 ctcr |= STM32_MDMA_CTCR_SBURST((ilog2(src_best_burst)));
643 /* Set memory data size */
644 dst_addr_width = stm32_mdma_get_max_width(addr, buf_len, tlen);
645 chan->mem_width = dst_addr_width;
646 dst_bus_width = stm32_mdma_get_width(chan, dst_addr_width);
647 if (dst_bus_width < 0)
648 return dst_bus_width;
649 ctcr &= ~(STM32_MDMA_CTCR_DSIZE_MASK |
650 STM32_MDMA_CTCR_DINCOS_MASK);
651 ctcr |= STM32_MDMA_CTCR_DSIZE(dst_bus_width) |
652 STM32_MDMA_CTCR_DINCOS(dst_bus_width);
654 /* Set memory burst value */
655 dst_maxburst = STM32_MDMA_MAX_BUF_LEN / dst_addr_width;
656 dst_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,
659 ctcr &= ~STM32_MDMA_CTCR_DBURST_MASK;
660 ctcr |= STM32_MDMA_CTCR_DBURST((ilog2(dst_best_burst)));
663 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS,
666 if (dst_bus_width != src_bus_width)
667 ctcr |= STM32_MDMA_CTCR_PKE;
669 /* Set source address */
670 stm32_mdma_write(dmadev, STM32_MDMA_CSAR(chan->id), src_addr);
674 dev_err(chan2dev(chan), "Dma direction is not supported\n");
685 static void stm32_mdma_dump_hwdesc(struct stm32_mdma_chan *chan,
686 struct stm32_mdma_desc_node *node)
688 dev_dbg(chan2dev(chan), "hwdesc: %pad\n", &node->hwdesc_phys);
689 dev_dbg(chan2dev(chan), "CTCR: 0x%08x\n", node->hwdesc->ctcr);
690 dev_dbg(chan2dev(chan), "CBNDTR: 0x%08x\n", node->hwdesc->cbndtr);
691 dev_dbg(chan2dev(chan), "CSAR: 0x%08x\n", node->hwdesc->csar);
692 dev_dbg(chan2dev(chan), "CDAR: 0x%08x\n", node->hwdesc->cdar);
693 dev_dbg(chan2dev(chan), "CBRUR: 0x%08x\n", node->hwdesc->cbrur);
694 dev_dbg(chan2dev(chan), "CLAR: 0x%08x\n", node->hwdesc->clar);
695 dev_dbg(chan2dev(chan), "CTBR: 0x%08x\n", node->hwdesc->ctbr);
696 dev_dbg(chan2dev(chan), "CMAR: 0x%08x\n", node->hwdesc->cmar);
697 dev_dbg(chan2dev(chan), "CMDR: 0x%08x\n\n", node->hwdesc->cmdr);
700 static void stm32_mdma_setup_hwdesc(struct stm32_mdma_chan *chan,
701 struct stm32_mdma_desc *desc,
702 enum dma_transfer_direction dir, u32 count,
703 dma_addr_t src_addr, dma_addr_t dst_addr,
704 u32 len, u32 ctcr, u32 ctbr, bool is_last,
705 bool is_first, bool is_cyclic)
707 struct stm32_mdma_chan_config *config = &chan->chan_config;
708 struct stm32_mdma_hwdesc *hwdesc;
709 u32 next = count + 1;
711 hwdesc = desc->node[count].hwdesc;
713 hwdesc->cbndtr &= ~(STM32_MDMA_CBNDTR_BRC_MK |
714 STM32_MDMA_CBNDTR_BRDUM |
715 STM32_MDMA_CBNDTR_BRSUM |
716 STM32_MDMA_CBNDTR_BNDT_MASK);
717 hwdesc->cbndtr |= STM32_MDMA_CBNDTR_BNDT(len);
718 hwdesc->csar = src_addr;
719 hwdesc->cdar = dst_addr;
722 hwdesc->cmar = config->mask_addr;
723 hwdesc->cmdr = config->mask_data;
727 hwdesc->clar = desc->node[0].hwdesc_phys;
731 hwdesc->clar = desc->node[next].hwdesc_phys;
734 stm32_mdma_dump_hwdesc(chan, &desc->node[count]);
737 static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan,
738 struct stm32_mdma_desc *desc,
739 struct scatterlist *sgl, u32 sg_len,
740 enum dma_transfer_direction direction)
742 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
743 struct dma_slave_config *dma_config = &chan->dma_config;
744 struct scatterlist *sg;
745 dma_addr_t src_addr, dst_addr;
749 for_each_sg(sgl, sg, sg_len, i) {
750 if (sg_dma_len(sg) > STM32_MDMA_MAX_BLOCK_LEN) {
751 dev_err(chan2dev(chan), "Invalid block len\n");
755 if (direction == DMA_MEM_TO_DEV) {
756 src_addr = sg_dma_address(sg);
757 dst_addr = dma_config->dst_addr;
758 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr,
759 &ctcr, &ctbr, src_addr,
761 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS,
764 src_addr = dma_config->src_addr;
765 dst_addr = sg_dma_address(sg);
766 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr,
767 &ctcr, &ctbr, dst_addr,
769 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS,
776 stm32_mdma_setup_hwdesc(chan, desc, direction, i, src_addr,
777 dst_addr, sg_dma_len(sg), ctcr, ctbr,
778 i == sg_len - 1, i == 0, false);
781 /* Enable interrupts */
782 ccr &= ~STM32_MDMA_CCR_IRQ_MASK;
783 ccr |= STM32_MDMA_CCR_TEIE | STM32_MDMA_CCR_CTCIE;
785 ccr |= STM32_MDMA_CCR_BTIE;
791 static struct dma_async_tx_descriptor *
792 stm32_mdma_prep_slave_sg(struct dma_chan *c, struct scatterlist *sgl,
793 u32 sg_len, enum dma_transfer_direction direction,
794 unsigned long flags, void *context)
796 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
797 struct stm32_mdma_desc *desc;
801 * Once DMA is in setup cyclic mode the channel we cannot assign this
802 * channel anymore. The DMA channel needs to be aborted or terminated
803 * for allowing another request.
805 if (chan->desc && chan->desc->cyclic) {
806 dev_err(chan2dev(chan),
807 "Request not allowed when dma in cyclic mode\n");
811 desc = stm32_mdma_alloc_desc(chan, sg_len);
815 ret = stm32_mdma_setup_xfer(chan, desc, sgl, sg_len, direction);
819 desc->cyclic = false;
821 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
824 for (i = 0; i < desc->count; i++)
825 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,
826 desc->node[i].hwdesc_phys);
831 static struct dma_async_tx_descriptor *
832 stm32_mdma_prep_dma_cyclic(struct dma_chan *c, dma_addr_t buf_addr,
833 size_t buf_len, size_t period_len,
834 enum dma_transfer_direction direction,
837 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
838 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
839 struct dma_slave_config *dma_config = &chan->dma_config;
840 struct stm32_mdma_desc *desc;
841 dma_addr_t src_addr, dst_addr;
842 u32 ccr, ctcr, ctbr, count;
846 * Once DMA is in setup cyclic mode the channel we cannot assign this
847 * channel anymore. The DMA channel needs to be aborted or terminated
848 * for allowing another request.
850 if (chan->desc && chan->desc->cyclic) {
851 dev_err(chan2dev(chan),
852 "Request not allowed when dma in cyclic mode\n");
856 if (!buf_len || !period_len || period_len > STM32_MDMA_MAX_BLOCK_LEN) {
857 dev_err(chan2dev(chan), "Invalid buffer/period len\n");
861 if (buf_len % period_len) {
862 dev_err(chan2dev(chan), "buf_len not multiple of period_len\n");
866 count = buf_len / period_len;
868 desc = stm32_mdma_alloc_desc(chan, count);
873 if (direction == DMA_MEM_TO_DEV) {
875 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr, &ctcr,
876 &ctbr, src_addr, period_len);
877 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS,
881 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr, &ctcr,
882 &ctbr, dst_addr, period_len);
883 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS,
890 /* Enable interrupts */
891 ccr &= ~STM32_MDMA_CCR_IRQ_MASK;
892 ccr |= STM32_MDMA_CCR_TEIE | STM32_MDMA_CCR_CTCIE | STM32_MDMA_CCR_BTIE;
895 /* Configure hwdesc list */
896 for (i = 0; i < count; i++) {
897 if (direction == DMA_MEM_TO_DEV) {
898 src_addr = buf_addr + i * period_len;
899 dst_addr = dma_config->dst_addr;
901 src_addr = dma_config->src_addr;
902 dst_addr = buf_addr + i * period_len;
905 stm32_mdma_setup_hwdesc(chan, desc, direction, i, src_addr,
906 dst_addr, period_len, ctcr, ctbr,
907 i == count - 1, i == 0, true);
912 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
915 for (i = 0; i < desc->count; i++)
916 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,
917 desc->node[i].hwdesc_phys);
922 static struct dma_async_tx_descriptor *
923 stm32_mdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, dma_addr_t src,
924 size_t len, unsigned long flags)
926 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
927 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
928 enum dma_slave_buswidth max_width;
929 struct stm32_mdma_desc *desc;
930 struct stm32_mdma_hwdesc *hwdesc;
931 u32 ccr, ctcr, ctbr, cbndtr, count, max_burst, mdma_burst;
932 u32 best_burst, tlen;
933 size_t xfer_count, offset;
934 int src_bus_width, dst_bus_width;
938 * Once DMA is in setup cyclic mode the channel we cannot assign this
939 * channel anymore. The DMA channel needs to be aborted or terminated
940 * to allow another request
942 if (chan->desc && chan->desc->cyclic) {
943 dev_err(chan2dev(chan),
944 "Request not allowed when dma in cyclic mode\n");
948 count = DIV_ROUND_UP(len, STM32_MDMA_MAX_BLOCK_LEN);
949 desc = stm32_mdma_alloc_desc(chan, count);
953 ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id));
954 ctcr = stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id));
955 ctbr = stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id));
956 cbndtr = stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id));
958 /* Enable sw req, some interrupts and clear other bits */
959 ccr &= ~(STM32_MDMA_CCR_WEX | STM32_MDMA_CCR_HEX |
960 STM32_MDMA_CCR_BEX | STM32_MDMA_CCR_PL_MASK |
961 STM32_MDMA_CCR_IRQ_MASK);
962 ccr |= STM32_MDMA_CCR_TEIE;
964 /* Enable SW request mode, dest/src inc and clear other bits */
965 ctcr &= ~(STM32_MDMA_CTCR_BWM | STM32_MDMA_CTCR_TRGM_MSK |
966 STM32_MDMA_CTCR_PAM_MASK | STM32_MDMA_CTCR_PKE |
967 STM32_MDMA_CTCR_TLEN_MSK | STM32_MDMA_CTCR_DBURST_MASK |
968 STM32_MDMA_CTCR_SBURST_MASK | STM32_MDMA_CTCR_DINCOS_MASK |
969 STM32_MDMA_CTCR_SINCOS_MASK | STM32_MDMA_CTCR_DSIZE_MASK |
970 STM32_MDMA_CTCR_SSIZE_MASK | STM32_MDMA_CTCR_DINC_MASK |
971 STM32_MDMA_CTCR_SINC_MASK);
972 ctcr |= STM32_MDMA_CTCR_SWRM | STM32_MDMA_CTCR_SINC(STM32_MDMA_INC) |
973 STM32_MDMA_CTCR_DINC(STM32_MDMA_INC);
975 /* Reset HW request */
976 ctbr &= ~STM32_MDMA_CTBR_TSEL_MASK;
979 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS, src);
980 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS, dest);
982 /* Clear CBNDTR registers */
983 cbndtr &= ~(STM32_MDMA_CBNDTR_BRC_MK | STM32_MDMA_CBNDTR_BRDUM |
984 STM32_MDMA_CBNDTR_BRSUM | STM32_MDMA_CBNDTR_BNDT_MASK);
986 if (len <= STM32_MDMA_MAX_BLOCK_LEN) {
987 cbndtr |= STM32_MDMA_CBNDTR_BNDT(len);
988 if (len <= STM32_MDMA_MAX_BUF_LEN) {
989 /* Setup a buffer transfer */
990 ccr |= STM32_MDMA_CCR_TCIE | STM32_MDMA_CCR_CTCIE;
991 ctcr |= STM32_MDMA_CTCR_TRGM(STM32_MDMA_BUFFER);
993 /* Setup a block transfer */
994 ccr |= STM32_MDMA_CCR_BTIE | STM32_MDMA_CCR_CTCIE;
995 ctcr |= STM32_MDMA_CTCR_TRGM(STM32_MDMA_BLOCK);
998 tlen = STM32_MDMA_MAX_BUF_LEN;
999 ctcr |= STM32_MDMA_CTCR_TLEN((tlen - 1));
1001 /* Set source best burst size */
1002 max_width = stm32_mdma_get_max_width(src, len, tlen);
1003 src_bus_width = stm32_mdma_get_width(chan, max_width);
1005 max_burst = tlen / max_width;
1006 best_burst = stm32_mdma_get_best_burst(len, tlen, max_burst,
1008 mdma_burst = ilog2(best_burst);
1010 ctcr |= STM32_MDMA_CTCR_SBURST(mdma_burst) |
1011 STM32_MDMA_CTCR_SSIZE(src_bus_width) |
1012 STM32_MDMA_CTCR_SINCOS(src_bus_width);
1014 /* Set destination best burst size */
1015 max_width = stm32_mdma_get_max_width(dest, len, tlen);
1016 dst_bus_width = stm32_mdma_get_width(chan, max_width);
1018 max_burst = tlen / max_width;
1019 best_burst = stm32_mdma_get_best_burst(len, tlen, max_burst,
1021 mdma_burst = ilog2(best_burst);
1023 ctcr |= STM32_MDMA_CTCR_DBURST(mdma_burst) |
1024 STM32_MDMA_CTCR_DSIZE(dst_bus_width) |
1025 STM32_MDMA_CTCR_DINCOS(dst_bus_width);
1027 if (dst_bus_width != src_bus_width)
1028 ctcr |= STM32_MDMA_CTCR_PKE;
1030 /* Prepare hardware descriptor */
1031 hwdesc = desc->node[0].hwdesc;
1032 hwdesc->ctcr = ctcr;
1033 hwdesc->cbndtr = cbndtr;
1035 hwdesc->cdar = dest;
1038 hwdesc->ctbr = ctbr;
1042 stm32_mdma_dump_hwdesc(chan, &desc->node[0]);
1044 /* Setup a LLI transfer */
1045 ctcr |= STM32_MDMA_CTCR_TRGM(STM32_MDMA_LINKED_LIST) |
1046 STM32_MDMA_CTCR_TLEN((STM32_MDMA_MAX_BUF_LEN - 1));
1047 ccr |= STM32_MDMA_CCR_BTIE | STM32_MDMA_CCR_CTCIE;
1048 tlen = STM32_MDMA_MAX_BUF_LEN;
1050 for (i = 0, offset = 0; offset < len;
1051 i++, offset += xfer_count) {
1052 xfer_count = min_t(size_t, len - offset,
1053 STM32_MDMA_MAX_BLOCK_LEN);
1055 /* Set source best burst size */
1056 max_width = stm32_mdma_get_max_width(src, len, tlen);
1057 src_bus_width = stm32_mdma_get_width(chan, max_width);
1059 max_burst = tlen / max_width;
1060 best_burst = stm32_mdma_get_best_burst(len, tlen,
1063 mdma_burst = ilog2(best_burst);
1065 ctcr |= STM32_MDMA_CTCR_SBURST(mdma_burst) |
1066 STM32_MDMA_CTCR_SSIZE(src_bus_width) |
1067 STM32_MDMA_CTCR_SINCOS(src_bus_width);
1069 /* Set destination best burst size */
1070 max_width = stm32_mdma_get_max_width(dest, len, tlen);
1071 dst_bus_width = stm32_mdma_get_width(chan, max_width);
1073 max_burst = tlen / max_width;
1074 best_burst = stm32_mdma_get_best_burst(len, tlen,
1077 mdma_burst = ilog2(best_burst);
1079 ctcr |= STM32_MDMA_CTCR_DBURST(mdma_burst) |
1080 STM32_MDMA_CTCR_DSIZE(dst_bus_width) |
1081 STM32_MDMA_CTCR_DINCOS(dst_bus_width);
1083 if (dst_bus_width != src_bus_width)
1084 ctcr |= STM32_MDMA_CTCR_PKE;
1086 /* Prepare hardware descriptor */
1087 stm32_mdma_setup_hwdesc(chan, desc, DMA_MEM_TO_MEM, i,
1088 src + offset, dest + offset,
1089 xfer_count, ctcr, ctbr,
1090 i == count - 1, i == 0, false);
1096 desc->cyclic = false;
1098 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
1101 static void stm32_mdma_dump_reg(struct stm32_mdma_chan *chan)
1103 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1105 dev_dbg(chan2dev(chan), "CCR: 0x%08x\n",
1106 stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)));
1107 dev_dbg(chan2dev(chan), "CTCR: 0x%08x\n",
1108 stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id)));
1109 dev_dbg(chan2dev(chan), "CBNDTR: 0x%08x\n",
1110 stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id)));
1111 dev_dbg(chan2dev(chan), "CSAR: 0x%08x\n",
1112 stm32_mdma_read(dmadev, STM32_MDMA_CSAR(chan->id)));
1113 dev_dbg(chan2dev(chan), "CDAR: 0x%08x\n",
1114 stm32_mdma_read(dmadev, STM32_MDMA_CDAR(chan->id)));
1115 dev_dbg(chan2dev(chan), "CBRUR: 0x%08x\n",
1116 stm32_mdma_read(dmadev, STM32_MDMA_CBRUR(chan->id)));
1117 dev_dbg(chan2dev(chan), "CLAR: 0x%08x\n",
1118 stm32_mdma_read(dmadev, STM32_MDMA_CLAR(chan->id)));
1119 dev_dbg(chan2dev(chan), "CTBR: 0x%08x\n",
1120 stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id)));
1121 dev_dbg(chan2dev(chan), "CMAR: 0x%08x\n",
1122 stm32_mdma_read(dmadev, STM32_MDMA_CMAR(chan->id)));
1123 dev_dbg(chan2dev(chan), "CMDR: 0x%08x\n",
1124 stm32_mdma_read(dmadev, STM32_MDMA_CMDR(chan->id)));
1127 static void stm32_mdma_start_transfer(struct stm32_mdma_chan *chan)
1129 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1130 struct virt_dma_desc *vdesc;
1131 struct stm32_mdma_hwdesc *hwdesc;
1135 vdesc = vchan_next_desc(&chan->vchan);
1141 chan->desc = to_stm32_mdma_desc(vdesc);
1142 hwdesc = chan->desc->node[0].hwdesc;
1143 chan->curr_hwdesc = 0;
1145 stm32_mdma_write(dmadev, STM32_MDMA_CCR(id), chan->desc->ccr);
1146 stm32_mdma_write(dmadev, STM32_MDMA_CTCR(id), hwdesc->ctcr);
1147 stm32_mdma_write(dmadev, STM32_MDMA_CBNDTR(id), hwdesc->cbndtr);
1148 stm32_mdma_write(dmadev, STM32_MDMA_CSAR(id), hwdesc->csar);
1149 stm32_mdma_write(dmadev, STM32_MDMA_CDAR(id), hwdesc->cdar);
1150 stm32_mdma_write(dmadev, STM32_MDMA_CBRUR(id), hwdesc->cbrur);
1151 stm32_mdma_write(dmadev, STM32_MDMA_CLAR(id), hwdesc->clar);
1152 stm32_mdma_write(dmadev, STM32_MDMA_CTBR(id), hwdesc->ctbr);
1153 stm32_mdma_write(dmadev, STM32_MDMA_CMAR(id), hwdesc->cmar);
1154 stm32_mdma_write(dmadev, STM32_MDMA_CMDR(id), hwdesc->cmdr);
1156 /* Clear interrupt status if it is there */
1157 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(id));
1159 stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(id), status);
1161 stm32_mdma_dump_reg(chan);
1164 stm32_mdma_set_bits(dmadev, STM32_MDMA_CCR(id), STM32_MDMA_CCR_EN);
1166 /* Set SW request in case of MEM2MEM transfer */
1167 if (hwdesc->ctcr & STM32_MDMA_CTCR_SWRM) {
1168 reg = STM32_MDMA_CCR(id);
1169 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CCR_SWRQ);
1174 dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan);
1177 static void stm32_mdma_issue_pending(struct dma_chan *c)
1179 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1180 unsigned long flags;
1182 spin_lock_irqsave(&chan->vchan.lock, flags);
1184 if (!vchan_issue_pending(&chan->vchan))
1187 dev_dbg(chan2dev(chan), "vchan %pK: issued\n", &chan->vchan);
1189 if (!chan->desc && !chan->busy)
1190 stm32_mdma_start_transfer(chan);
1193 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1196 static int stm32_mdma_pause(struct dma_chan *c)
1198 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1199 unsigned long flags;
1202 spin_lock_irqsave(&chan->vchan.lock, flags);
1203 ret = stm32_mdma_disable_chan(chan);
1204 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1207 dev_dbg(chan2dev(chan), "vchan %pK: pause\n", &chan->vchan);
1212 static int stm32_mdma_resume(struct dma_chan *c)
1214 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1215 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1216 struct stm32_mdma_hwdesc *hwdesc;
1217 unsigned long flags;
1220 hwdesc = chan->desc->node[chan->curr_hwdesc].hwdesc;
1222 spin_lock_irqsave(&chan->vchan.lock, flags);
1224 /* Re-configure control register */
1225 stm32_mdma_write(dmadev, STM32_MDMA_CCR(chan->id), chan->desc->ccr);
1227 /* Clear interrupt status if it is there */
1228 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id));
1230 stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(chan->id), status);
1232 stm32_mdma_dump_reg(chan);
1235 reg = STM32_MDMA_CCR(chan->id);
1236 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CCR_EN);
1238 /* Set SW request in case of MEM2MEM transfer */
1239 if (hwdesc->ctcr & STM32_MDMA_CTCR_SWRM)
1240 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CCR_SWRQ);
1242 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1244 dev_dbg(chan2dev(chan), "vchan %pK: resume\n", &chan->vchan);
1249 static int stm32_mdma_terminate_all(struct dma_chan *c)
1251 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1252 unsigned long flags;
1255 spin_lock_irqsave(&chan->vchan.lock, flags);
1257 stm32_mdma_stop(chan);
1260 vchan_get_all_descriptors(&chan->vchan, &head);
1261 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1263 vchan_dma_desc_free_list(&chan->vchan, &head);
1268 static void stm32_mdma_synchronize(struct dma_chan *c)
1270 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1272 vchan_synchronize(&chan->vchan);
1275 static int stm32_mdma_slave_config(struct dma_chan *c,
1276 struct dma_slave_config *config)
1278 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1280 memcpy(&chan->dma_config, config, sizeof(*config));
1285 static size_t stm32_mdma_desc_residue(struct stm32_mdma_chan *chan,
1286 struct stm32_mdma_desc *desc,
1289 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1290 struct stm32_mdma_hwdesc *hwdesc = desc->node[0].hwdesc;
1291 u32 cbndtr, residue, modulo, burst_size;
1295 for (i = curr_hwdesc + 1; i < desc->count; i++) {
1296 hwdesc = desc->node[i].hwdesc;
1297 residue += STM32_MDMA_CBNDTR_BNDT(hwdesc->cbndtr);
1299 cbndtr = stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id));
1300 residue += cbndtr & STM32_MDMA_CBNDTR_BNDT_MASK;
1302 if (!chan->mem_burst)
1305 burst_size = chan->mem_burst * chan->mem_width;
1306 modulo = residue % burst_size;
1308 residue = residue - modulo + burst_size;
1313 static enum dma_status stm32_mdma_tx_status(struct dma_chan *c,
1314 dma_cookie_t cookie,
1315 struct dma_tx_state *state)
1317 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1318 struct virt_dma_desc *vdesc;
1319 enum dma_status status;
1320 unsigned long flags;
1323 status = dma_cookie_status(c, cookie, state);
1324 if ((status == DMA_COMPLETE) || (!state))
1327 spin_lock_irqsave(&chan->vchan.lock, flags);
1329 vdesc = vchan_find_desc(&chan->vchan, cookie);
1330 if (chan->desc && cookie == chan->desc->vdesc.tx.cookie)
1331 residue = stm32_mdma_desc_residue(chan, chan->desc,
1334 residue = stm32_mdma_desc_residue(chan,
1335 to_stm32_mdma_desc(vdesc), 0);
1336 dma_set_residue(state, residue);
1338 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1343 static void stm32_mdma_xfer_end(struct stm32_mdma_chan *chan)
1345 list_del(&chan->desc->vdesc.node);
1346 vchan_cookie_complete(&chan->desc->vdesc);
1350 /* Start the next transfer if this driver has a next desc */
1351 stm32_mdma_start_transfer(chan);
1354 static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid)
1356 struct stm32_mdma_device *dmadev = devid;
1357 struct stm32_mdma_chan *chan = devid;
1358 u32 reg, id, ien, status, flag;
1360 /* Find out which channel generates the interrupt */
1361 status = readl_relaxed(dmadev->base + STM32_MDMA_GISR0);
1365 status = readl_relaxed(dmadev->base + STM32_MDMA_GISR1);
1367 dev_dbg(mdma2dev(dmadev), "spurious it\n");
1372 * As GISR0 provides status for channel id from 0 to 31,
1373 * so GISR1 provides status for channel id from 32 to 62
1378 chan = &dmadev->chan[id];
1380 dev_err(chan2dev(chan), "MDMA channel not initialized\n");
1384 /* Handle interrupt for the channel */
1385 spin_lock(&chan->vchan.lock);
1386 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id));
1387 ien = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id));
1388 ien &= STM32_MDMA_CCR_IRQ_MASK;
1391 if (!(status & ien)) {
1392 spin_unlock(&chan->vchan.lock);
1393 dev_dbg(chan2dev(chan),
1394 "spurious it (status=0x%04x, ien=0x%04x)\n",
1399 flag = __ffs(status & ien);
1400 reg = STM32_MDMA_CIFCR(chan->id);
1402 switch (1 << flag) {
1403 case STM32_MDMA_CISR_TEIF:
1405 status = readl_relaxed(dmadev->base + STM32_MDMA_CESR(id));
1406 dev_err(chan2dev(chan), "Transfer Err: stat=0x%08x\n", status);
1407 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CTEIF);
1410 case STM32_MDMA_CISR_CTCIF:
1411 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CCTCIF);
1412 stm32_mdma_xfer_end(chan);
1415 case STM32_MDMA_CISR_BRTIF:
1416 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CBRTIF);
1419 case STM32_MDMA_CISR_BTIF:
1420 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CBTIF);
1421 chan->curr_hwdesc++;
1422 if (chan->desc && chan->desc->cyclic) {
1423 if (chan->curr_hwdesc == chan->desc->count)
1424 chan->curr_hwdesc = 0;
1425 vchan_cyclic_callback(&chan->desc->vdesc);
1429 case STM32_MDMA_CISR_TCIF:
1430 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CLTCIF);
1434 dev_err(chan2dev(chan), "it %d unhandled (status=0x%04x)\n",
1438 spin_unlock(&chan->vchan.lock);
1444 static int stm32_mdma_alloc_chan_resources(struct dma_chan *c)
1446 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1447 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1450 chan->desc_pool = dmam_pool_create(dev_name(&c->dev->device),
1452 sizeof(struct stm32_mdma_hwdesc),
1453 __alignof__(struct stm32_mdma_hwdesc),
1455 if (!chan->desc_pool) {
1456 dev_err(chan2dev(chan), "failed to allocate descriptor pool\n");
1460 ret = pm_runtime_get_sync(dmadev->ddev.dev);
1464 ret = stm32_mdma_disable_chan(chan);
1466 pm_runtime_put(dmadev->ddev.dev);
1471 static void stm32_mdma_free_chan_resources(struct dma_chan *c)
1473 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1474 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1475 unsigned long flags;
1477 dev_dbg(chan2dev(chan), "Freeing channel %d\n", chan->id);
1480 spin_lock_irqsave(&chan->vchan.lock, flags);
1481 stm32_mdma_stop(chan);
1483 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1486 pm_runtime_put(dmadev->ddev.dev);
1487 vchan_free_chan_resources(to_virt_chan(c));
1488 dmam_pool_destroy(chan->desc_pool);
1489 chan->desc_pool = NULL;
1492 static struct dma_chan *stm32_mdma_of_xlate(struct of_phandle_args *dma_spec,
1493 struct of_dma *ofdma)
1495 struct stm32_mdma_device *dmadev = ofdma->of_dma_data;
1496 struct stm32_mdma_chan *chan;
1498 struct stm32_mdma_chan_config config;
1500 if (dma_spec->args_count < 5) {
1501 dev_err(mdma2dev(dmadev), "Bad number of args\n");
1505 config.request = dma_spec->args[0];
1506 config.priority_level = dma_spec->args[1];
1507 config.transfer_config = dma_spec->args[2];
1508 config.mask_addr = dma_spec->args[3];
1509 config.mask_data = dma_spec->args[4];
1511 if (config.request >= dmadev->nr_requests) {
1512 dev_err(mdma2dev(dmadev), "Bad request line\n");
1516 if (config.priority_level > STM32_MDMA_VERY_HIGH_PRIORITY) {
1517 dev_err(mdma2dev(dmadev), "Priority level not supported\n");
1521 c = dma_get_any_slave_channel(&dmadev->ddev);
1523 dev_err(mdma2dev(dmadev), "No more channels available\n");
1527 chan = to_stm32_mdma_chan(c);
1528 chan->chan_config = config;
1533 static const struct of_device_id stm32_mdma_of_match[] = {
1534 { .compatible = "st,stm32h7-mdma", },
1537 MODULE_DEVICE_TABLE(of, stm32_mdma_of_match);
1539 static int stm32_mdma_probe(struct platform_device *pdev)
1541 struct stm32_mdma_chan *chan;
1542 struct stm32_mdma_device *dmadev;
1543 struct dma_device *dd;
1544 struct device_node *of_node;
1545 struct resource *res;
1546 u32 nr_channels, nr_requests;
1549 of_node = pdev->dev.of_node;
1553 ret = device_property_read_u32(&pdev->dev, "dma-channels",
1556 nr_channels = STM32_MDMA_MAX_CHANNELS;
1557 dev_warn(&pdev->dev, "MDMA defaulting on %i channels\n",
1561 ret = device_property_read_u32(&pdev->dev, "dma-requests",
1564 nr_requests = STM32_MDMA_MAX_REQUESTS;
1565 dev_warn(&pdev->dev, "MDMA defaulting on %i request lines\n",
1569 count = device_property_read_u32_array(&pdev->dev, "st,ahb-addr-masks",
1574 dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev) + sizeof(u32) * count,
1579 dmadev->nr_channels = nr_channels;
1580 dmadev->nr_requests = nr_requests;
1581 device_property_read_u32_array(&pdev->dev, "st,ahb-addr-masks",
1582 dmadev->ahb_addr_masks,
1584 dmadev->nr_ahb_addr_masks = count;
1586 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1587 dmadev->base = devm_ioremap_resource(&pdev->dev, res);
1588 if (IS_ERR(dmadev->base))
1589 return PTR_ERR(dmadev->base);
1591 dmadev->clk = devm_clk_get(&pdev->dev, NULL);
1592 if (IS_ERR(dmadev->clk)) {
1593 ret = PTR_ERR(dmadev->clk);
1594 if (ret == -EPROBE_DEFER)
1595 dev_info(&pdev->dev, "Missing controller clock\n");
1599 ret = clk_prepare_enable(dmadev->clk);
1601 dev_err(&pdev->dev, "clk_prep_enable error: %d\n", ret);
1605 dmadev->rst = devm_reset_control_get(&pdev->dev, NULL);
1606 if (!IS_ERR(dmadev->rst)) {
1607 reset_control_assert(dmadev->rst);
1609 reset_control_deassert(dmadev->rst);
1613 dma_cap_set(DMA_SLAVE, dd->cap_mask);
1614 dma_cap_set(DMA_PRIVATE, dd->cap_mask);
1615 dma_cap_set(DMA_CYCLIC, dd->cap_mask);
1616 dma_cap_set(DMA_MEMCPY, dd->cap_mask);
1617 dd->device_alloc_chan_resources = stm32_mdma_alloc_chan_resources;
1618 dd->device_free_chan_resources = stm32_mdma_free_chan_resources;
1619 dd->device_tx_status = stm32_mdma_tx_status;
1620 dd->device_issue_pending = stm32_mdma_issue_pending;
1621 dd->device_prep_slave_sg = stm32_mdma_prep_slave_sg;
1622 dd->device_prep_dma_cyclic = stm32_mdma_prep_dma_cyclic;
1623 dd->device_prep_dma_memcpy = stm32_mdma_prep_dma_memcpy;
1624 dd->device_config = stm32_mdma_slave_config;
1625 dd->device_pause = stm32_mdma_pause;
1626 dd->device_resume = stm32_mdma_resume;
1627 dd->device_terminate_all = stm32_mdma_terminate_all;
1628 dd->device_synchronize = stm32_mdma_synchronize;
1629 dd->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1630 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1631 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1632 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1633 dd->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1634 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1635 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1636 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1637 dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) |
1638 BIT(DMA_MEM_TO_MEM);
1639 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1640 dd->max_burst = STM32_MDMA_MAX_BURST;
1641 dd->dev = &pdev->dev;
1642 INIT_LIST_HEAD(&dd->channels);
1644 for (i = 0; i < dmadev->nr_channels; i++) {
1645 chan = &dmadev->chan[i];
1647 chan->vchan.desc_free = stm32_mdma_desc_free;
1648 vchan_init(&chan->vchan, dd);
1651 dmadev->irq = platform_get_irq(pdev, 0);
1652 if (dmadev->irq < 0) {
1653 dev_err(&pdev->dev, "failed to get IRQ\n");
1657 ret = devm_request_irq(&pdev->dev, dmadev->irq, stm32_mdma_irq_handler,
1658 0, dev_name(&pdev->dev), dmadev);
1660 dev_err(&pdev->dev, "failed to request IRQ\n");
1664 ret = dmaenginem_async_device_register(dd);
1668 ret = of_dma_controller_register(of_node, stm32_mdma_of_xlate, dmadev);
1671 "STM32 MDMA DMA OF registration failed %d\n", ret);
1672 goto err_unregister;
1675 platform_set_drvdata(pdev, dmadev);
1676 pm_runtime_set_active(&pdev->dev);
1677 pm_runtime_enable(&pdev->dev);
1678 pm_runtime_get_noresume(&pdev->dev);
1679 pm_runtime_put(&pdev->dev);
1681 dev_info(&pdev->dev, "STM32 MDMA driver registered\n");
1690 static int stm32_mdma_runtime_suspend(struct device *dev)
1692 struct stm32_mdma_device *dmadev = dev_get_drvdata(dev);
1694 clk_disable_unprepare(dmadev->clk);
1699 static int stm32_mdma_runtime_resume(struct device *dev)
1701 struct stm32_mdma_device *dmadev = dev_get_drvdata(dev);
1704 ret = clk_prepare_enable(dmadev->clk);
1706 dev_err(dev, "failed to prepare_enable clock\n");
1714 static const struct dev_pm_ops stm32_mdma_pm_ops = {
1715 SET_RUNTIME_PM_OPS(stm32_mdma_runtime_suspend,
1716 stm32_mdma_runtime_resume, NULL)
1719 static struct platform_driver stm32_mdma_driver = {
1720 .probe = stm32_mdma_probe,
1722 .name = "stm32-mdma",
1723 .of_match_table = stm32_mdma_of_match,
1724 .pm = &stm32_mdma_pm_ops,
1728 static int __init stm32_mdma_init(void)
1730 return platform_driver_register(&stm32_mdma_driver);
1733 subsys_initcall(stm32_mdma_init);
1735 MODULE_DESCRIPTION("Driver for STM32 MDMA controller");
1736 MODULE_AUTHOR("M'boumba Cedric Madianga <cedric.madianga@gmail.com>");
1737 MODULE_AUTHOR("Pierre-Yves Mordret <pierre-yves.mordret@st.com>");
1738 MODULE_LICENSE("GPL v2");