drivers/dma/uniphier-xdmac.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * External DMA controller driver for UniPhier SoCs
   4  * Copyright 2019 Socionext Inc.
   5  * Author: Kunihiko Hayashi <hayashi.kunihiko@socionext.com>
   6  */
   7
   8 #include <linux/bitops.h>
   9 #include <linux/bitfield.h>
  10 #include <linux/iopoll.h>
  11 #include <linux/module.h>
  12 #include <linux/of.h>
  13 #include <linux/of_dma.h>
  14 #include <linux/platform_device.h>
  15
  16 #include "dmaengine.h"
  17 #include "virt-dma.h"
  18
  19 #define XDMAC_CH_WIDTH          0x100
  20
  21 #define XDMAC_TFA               0x08
  22 #define XDMAC_TFA_MCNT_MASK     GENMASK(23, 16)
  23 #define XDMAC_TFA_MASK          GENMASK(5, 0)
  24 #define XDMAC_SADM              0x10
  25 #define XDMAC_SADM_STW_MASK     GENMASK(25, 24)
  26 #define XDMAC_SADM_SAM          BIT(4)
  27 #define XDMAC_SADM_SAM_FIXED    XDMAC_SADM_SAM
  28 #define XDMAC_SADM_SAM_INC      0
  29 #define XDMAC_DADM              0x14
  30 #define XDMAC_DADM_DTW_MASK     XDMAC_SADM_STW_MASK
  31 #define XDMAC_DADM_DAM          XDMAC_SADM_SAM
  32 #define XDMAC_DADM_DAM_FIXED    XDMAC_SADM_SAM_FIXED
  33 #define XDMAC_DADM_DAM_INC      XDMAC_SADM_SAM_INC
  34 #define XDMAC_EXSAD             0x18
  35 #define XDMAC_EXDAD             0x1c
  36 #define XDMAC_SAD               0x20
  37 #define XDMAC_DAD               0x24
  38 #define XDMAC_ITS               0x28
  39 #define XDMAC_ITS_MASK          GENMASK(25, 0)
  40 #define XDMAC_TNUM              0x2c
  41 #define XDMAC_TNUM_MASK         GENMASK(15, 0)
  42 #define XDMAC_TSS               0x30
  43 #define XDMAC_TSS_REQ           BIT(0)
  44 #define XDMAC_IEN               0x34
  45 #define XDMAC_IEN_ERRIEN        BIT(1)
  46 #define XDMAC_IEN_ENDIEN        BIT(0)
  47 #define XDMAC_STAT              0x40
  48 #define XDMAC_STAT_TENF         BIT(0)
  49 #define XDMAC_IR                0x44
  50 #define XDMAC_IR_ERRF           BIT(1)
  51 #define XDMAC_IR_ENDF           BIT(0)
  52 #define XDMAC_ID                0x48
  53 #define XDMAC_ID_ERRIDF         BIT(1)
  54 #define XDMAC_ID_ENDIDF         BIT(0)
  55
  56 #define XDMAC_MAX_CHANS         16
  57 #define XDMAC_INTERVAL_CLKS     20
  58 #define XDMAC_MAX_WORDS         XDMAC_TNUM_MASK
  59
  60 /* cut lower bit for maintain alignment of maximum transfer size */
  61 #define XDMAC_MAX_WORD_SIZE     (XDMAC_ITS_MASK & ~GENMASK(3, 0))
  62
  63 #define UNIPHIER_XDMAC_BUSWIDTHS \
  64         (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
  65          BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
  66          BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
  67          BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
  68
  69 struct uniphier_xdmac_desc_node {
  70         dma_addr_t src;
  71         dma_addr_t dst;
  72         u32 burst_size;
  73         u32 nr_burst;
  74 };
  75
  76 struct uniphier_xdmac_desc {
  77         struct virt_dma_desc vd;
  78
  79         unsigned int nr_node;
  80         unsigned int cur_node;
  81         enum dma_transfer_direction dir;
  82         struct uniphier_xdmac_desc_node nodes[];
  83 };
  84
  85 struct uniphier_xdmac_chan {
  86         struct virt_dma_chan vc;
  87         struct uniphier_xdmac_device *xdev;
  88         struct uniphier_xdmac_desc *xd;
  89         void __iomem *reg_ch_base;
  90         struct dma_slave_config sconfig;
  91         int id;
  92         unsigned int req_factor;
  93 };
  94
  95 struct uniphier_xdmac_device {
  96         struct dma_device ddev;
  97         void __iomem *reg_base;
  98         int nr_chans;
  99         struct uniphier_xdmac_chan channels[];
 100 };
 101
 102 static struct uniphier_xdmac_chan *
 103 to_uniphier_xdmac_chan(struct virt_dma_chan *vc)
 104 {
 105         return container_of(vc, struct uniphier_xdmac_chan, vc);
 106 }
 107
 108 static struct uniphier_xdmac_desc *
 109 to_uniphier_xdmac_desc(struct virt_dma_desc *vd)
 110 {
 111         return container_of(vd, struct uniphier_xdmac_desc, vd);
 112 }
 113
 114 /* xc->vc.lock must be held by caller */
 115 static struct uniphier_xdmac_desc *
 116 uniphier_xdmac_next_desc(struct uniphier_xdmac_chan *xc)
 117 {
 118         struct virt_dma_desc *vd;
 119
 120         vd = vchan_next_desc(&xc->vc);
 121         if (!vd)
 122                 return NULL;
 123
 124         list_del(&vd->node);
 125
 126         return to_uniphier_xdmac_desc(vd);
 127 }
 128
 129 /* xc->vc.lock must be held by caller */
 130 static void uniphier_xdmac_chan_start(struct uniphier_xdmac_chan *xc,
 131                                       struct uniphier_xdmac_desc *xd)
 132 {
 133         u32 src_mode, src_addr, src_width;
 134         u32 dst_mode, dst_addr, dst_width;
 135         u32 val, its, tnum;
 136         enum dma_slave_buswidth buswidth;
 137
 138         src_addr = xd->nodes[xd->cur_node].src;
 139         dst_addr = xd->nodes[xd->cur_node].dst;
 140         its      = xd->nodes[xd->cur_node].burst_size;
 141         tnum     = xd->nodes[xd->cur_node].nr_burst;
 142
 143         /*
 144          * The width of MEM side must be 4 or 8 bytes, that does not
 145          * affect that of DEV side and transfer size.
 146          */
 147         if (xd->dir == DMA_DEV_TO_MEM) {
 148                 src_mode = XDMAC_SADM_SAM_FIXED;
 149                 buswidth = xc->sconfig.src_addr_width;
 150         } else {
 151                 src_mode = XDMAC_SADM_SAM_INC;
 152                 buswidth = DMA_SLAVE_BUSWIDTH_8_BYTES;
 153         }
 154         src_width = FIELD_PREP(XDMAC_SADM_STW_MASK, __ffs(buswidth));
 155
 156         if (xd->dir == DMA_MEM_TO_DEV) {
 157                 dst_mode = XDMAC_DADM_DAM_FIXED;
 158                 buswidth = xc->sconfig.dst_addr_width;
 159         } else {
 160                 dst_mode = XDMAC_DADM_DAM_INC;
 161                 buswidth = DMA_SLAVE_BUSWIDTH_8_BYTES;
 162         }
 163         dst_width = FIELD_PREP(XDMAC_DADM_DTW_MASK, __ffs(buswidth));
 164
 165         /* setup transfer factor */
 166         val = FIELD_PREP(XDMAC_TFA_MCNT_MASK, XDMAC_INTERVAL_CLKS);
 167         val |= FIELD_PREP(XDMAC_TFA_MASK, xc->req_factor);
 168         writel(val, xc->reg_ch_base + XDMAC_TFA);
 169
 170         /* setup the channel */
 171         writel(lower_32_bits(src_addr), xc->reg_ch_base + XDMAC_SAD);
 172         writel(upper_32_bits(src_addr), xc->reg_ch_base + XDMAC_EXSAD);
 173
 174         writel(lower_32_bits(dst_addr), xc->reg_ch_base + XDMAC_DAD);
 175         writel(upper_32_bits(dst_addr), xc->reg_ch_base + XDMAC_EXDAD);
 176
 177         src_mode |= src_width;
 178         dst_mode |= dst_width;
 179         writel(src_mode, xc->reg_ch_base + XDMAC_SADM);
 180         writel(dst_mode, xc->reg_ch_base + XDMAC_DADM);
 181
 182         writel(its, xc->reg_ch_base + XDMAC_ITS);
 183         writel(tnum, xc->reg_ch_base + XDMAC_TNUM);
 184
 185         /* enable interrupt */
 186         writel(XDMAC_IEN_ENDIEN | XDMAC_IEN_ERRIEN,
 187                xc->reg_ch_base + XDMAC_IEN);
 188
 189         /* start XDMAC */
 190         val = readl(xc->reg_ch_base + XDMAC_TSS);
 191         val |= XDMAC_TSS_REQ;
 192         writel(val, xc->reg_ch_base + XDMAC_TSS);
 193 }
 194
 195 /* xc->vc.lock must be held by caller */
 196 static int uniphier_xdmac_chan_stop(struct uniphier_xdmac_chan *xc)
 197 {
 198         u32 val;
 199
 200         /* disable interrupt */
 201         val = readl(xc->reg_ch_base + XDMAC_IEN);
 202         val &= ~(XDMAC_IEN_ENDIEN | XDMAC_IEN_ERRIEN);
 203         writel(val, xc->reg_ch_base + XDMAC_IEN);
 204
 205         /* stop XDMAC */
 206         val = readl(xc->reg_ch_base + XDMAC_TSS);
 207         val &= ~XDMAC_TSS_REQ;
 208         writel(0, xc->reg_ch_base + XDMAC_TSS);
 209
 210         /* wait until transfer is stopped */
 211         return readl_poll_timeout(xc->reg_ch_base + XDMAC_STAT, val,
 212                                   !(val & XDMAC_STAT_TENF), 100, 1000);
 213 }
 214
 215 /* xc->vc.lock must be held by caller */
 216 static void uniphier_xdmac_start(struct uniphier_xdmac_chan *xc)
 217 {
 218         struct uniphier_xdmac_desc *xd;
 219
 220         xd = uniphier_xdmac_next_desc(xc);
 221         if (xd)
 222                 uniphier_xdmac_chan_start(xc, xd);
 223
 224         /* set desc to chan regardless of xd is null */
 225         xc->xd = xd;
 226 }
 227
 228 static void uniphier_xdmac_chan_irq(struct uniphier_xdmac_chan *xc)
 229 {
 230         u32 stat;
 231         int ret;
 232
 233         spin_lock(&xc->vc.lock);
 234
 235         stat = readl(xc->reg_ch_base + XDMAC_ID);
 236
 237         if (stat & XDMAC_ID_ERRIDF) {
 238                 ret = uniphier_xdmac_chan_stop(xc);
 239                 if (ret)
 240                         dev_err(xc->xdev->ddev.dev,
 241                                 "DMA transfer error with aborting issue\n");
 242                 else
 243                         dev_err(xc->xdev->ddev.dev,
 244                                 "DMA transfer error\n");
 245
 246         } else if ((stat & XDMAC_ID_ENDIDF) && xc->xd) {
 247                 xc->xd->cur_node++;
 248                 if (xc->xd->cur_node >= xc->xd->nr_node) {
 249                         vchan_cookie_complete(&xc->xd->vd);
 250                         uniphier_xdmac_start(xc);
 251                 } else {
 252                         uniphier_xdmac_chan_start(xc, xc->xd);
 253                 }
 254         }
 255
 256         /* write bits to clear */
 257         writel(stat, xc->reg_ch_base + XDMAC_IR);
 258
 259         spin_unlock(&xc->vc.lock);
 260 }
 261
 262 static irqreturn_t uniphier_xdmac_irq_handler(int irq, void *dev_id)
 263 {
 264         struct uniphier_xdmac_device *xdev = dev_id;
 265         int i;
 266
 267         for (i = 0; i < xdev->nr_chans; i++)
 268                 uniphier_xdmac_chan_irq(&xdev->channels[i]);
 269
 270         return IRQ_HANDLED;
 271 }
 272
 273 static void uniphier_xdmac_free_chan_resources(struct dma_chan *chan)
 274 {
 275         vchan_free_chan_resources(to_virt_chan(chan));
 276 }
 277
 278 static struct dma_async_tx_descriptor *
 279 uniphier_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst,
 280                                dma_addr_t src, size_t len, unsigned long flags)
 281 {
 282         struct virt_dma_chan *vc = to_virt_chan(chan);
 283         struct uniphier_xdmac_desc *xd;
 284         unsigned int nr;
 285         size_t burst_size, tlen;
 286         int i;
 287
 288         if (len > XDMAC_MAX_WORD_SIZE * XDMAC_MAX_WORDS)
 289                 return NULL;
 290
 291         nr = 1 + len / XDMAC_MAX_WORD_SIZE;
 292
 293         xd = kzalloc(struct_size(xd, nodes, nr), GFP_NOWAIT);
 294         if (!xd)
 295                 return NULL;
 296
 297         for (i = 0; i < nr; i++) {
 298                 burst_size = min_t(size_t, len, XDMAC_MAX_WORD_SIZE);
 299                 xd->nodes[i].src = src;
 300                 xd->nodes[i].dst = dst;
 301                 xd->nodes[i].burst_size = burst_size;
 302                 xd->nodes[i].nr_burst = len / burst_size;
 303                 tlen = rounddown(len, burst_size);
 304                 src += tlen;
 305                 dst += tlen;
 306                 len -= tlen;
 307         }
 308
 309         xd->dir = DMA_MEM_TO_MEM;
 310         xd->nr_node = nr;
 311         xd->cur_node = 0;
 312
 313         return vchan_tx_prep(vc, &xd->vd, flags);
 314 }
 315
 316 static struct dma_async_tx_descriptor *
 317 uniphier_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 318                              unsigned int sg_len,
 319                              enum dma_transfer_direction direction,
 320                              unsigned long flags, void *context)
 321 {
 322         struct virt_dma_chan *vc = to_virt_chan(chan);
 323         struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc);
 324         struct uniphier_xdmac_desc *xd;
 325         struct scatterlist *sg;
 326         enum dma_slave_buswidth buswidth;
 327         u32 maxburst;
 328         int i;
 329
 330         if (!is_slave_direction(direction))
 331                 return NULL;
 332
 333         if (direction == DMA_DEV_TO_MEM) {
 334                 buswidth = xc->sconfig.src_addr_width;
 335                 maxburst = xc->sconfig.src_maxburst;
 336         } else {
 337                 buswidth = xc->sconfig.dst_addr_width;
 338                 maxburst = xc->sconfig.dst_maxburst;
 339         }
 340
 341         if (!maxburst)
 342                 maxburst = 1;
 343         if (maxburst > xc->xdev->ddev.max_burst) {
 344                 dev_err(xc->xdev->ddev.dev,
 345                         "Exceed maximum number of burst words\n");
 346                 return NULL;
 347         }
 348
 349         xd = kzalloc(struct_size(xd, nodes, sg_len), GFP_NOWAIT);
 350         if (!xd)
 351                 return NULL;
 352
 353         for_each_sg(sgl, sg, sg_len, i) {
 354                 xd->nodes[i].src = (direction == DMA_DEV_TO_MEM)
 355                         ? xc->sconfig.src_addr : sg_dma_address(sg);
 356                 xd->nodes[i].dst = (direction == DMA_MEM_TO_DEV)
 357                         ? xc->sconfig.dst_addr : sg_dma_address(sg);
 358                 xd->nodes[i].burst_size = maxburst * buswidth;
 359                 xd->nodes[i].nr_burst =
 360                         sg_dma_len(sg) / xd->nodes[i].burst_size;
 361
 362                 /*
 363                  * Currently transfer that size doesn't align the unit size
 364                  * (the number of burst words * bus-width) is not allowed,
 365                  * because the driver does not support the way to transfer
 366                  * residue size. As a matter of fact, in order to transfer
 367                  * arbitrary size, 'src_maxburst' or 'dst_maxburst' of
 368                  * dma_slave_config must be 1.
 369                  */
 370                 if (sg_dma_len(sg) % xd->nodes[i].burst_size) {
 371                         dev_err(xc->xdev->ddev.dev,
 372                                 "Unaligned transfer size: %d", sg_dma_len(sg));
 373                         kfree(xd);
 374                         return NULL;
 375                 }
 376
 377                 if (xd->nodes[i].nr_burst > XDMAC_MAX_WORDS) {
 378                         dev_err(xc->xdev->ddev.dev,
 379                                 "Exceed maximum transfer size");
 380                         kfree(xd);
 381                         return NULL;
 382                 }
 383         }
 384
 385         xd->dir = direction;
 386         xd->nr_node = sg_len;
 387         xd->cur_node = 0;
 388
 389         return vchan_tx_prep(vc, &xd->vd, flags);
 390 }
 391
 392 static int uniphier_xdmac_slave_config(struct dma_chan *chan,
 393                                        struct dma_slave_config *config)
 394 {
 395         struct virt_dma_chan *vc = to_virt_chan(chan);
 396         struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc);
 397
 398         memcpy(&xc->sconfig, config, sizeof(*config));
 399
 400         return 0;
 401 }
 402
 403 static int uniphier_xdmac_terminate_all(struct dma_chan *chan)
 404 {
 405         struct virt_dma_chan *vc = to_virt_chan(chan);
 406         struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc);
 407         unsigned long flags;
 408         int ret = 0;
 409         LIST_HEAD(head);
 410
 411         spin_lock_irqsave(&vc->lock, flags);
 412
 413         if (xc->xd) {
 414                 vchan_terminate_vdesc(&xc->xd->vd);
 415                 xc->xd = NULL;
 416                 ret = uniphier_xdmac_chan_stop(xc);
 417         }
 418
 419         vchan_get_all_descriptors(vc, &head);
 420
 421         spin_unlock_irqrestore(&vc->lock, flags);
 422
 423         vchan_dma_desc_free_list(vc, &head);
 424
 425         return ret;
 426 }
 427
 428 static void uniphier_xdmac_synchronize(struct dma_chan *chan)
 429 {
 430         vchan_synchronize(to_virt_chan(chan));
 431 }
 432
 433 static void uniphier_xdmac_issue_pending(struct dma_chan *chan)
 434 {
 435         struct virt_dma_chan *vc = to_virt_chan(chan);
 436         struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc);
 437         unsigned long flags;
 438
 439         spin_lock_irqsave(&vc->lock, flags);
 440
 441         if (vchan_issue_pending(vc) && !xc->xd)
 442                 uniphier_xdmac_start(xc);
 443
 444         spin_unlock_irqrestore(&vc->lock, flags);
 445 }
 446
 447 static void uniphier_xdmac_desc_free(struct virt_dma_desc *vd)
 448 {
 449         kfree(to_uniphier_xdmac_desc(vd));
 450 }
 451
 452 static void uniphier_xdmac_chan_init(struct uniphier_xdmac_device *xdev,
 453                                      int ch)
 454 {
 455         struct uniphier_xdmac_chan *xc = &xdev->channels[ch];
 456
 457         xc->xdev = xdev;
 458         xc->reg_ch_base = xdev->reg_base + XDMAC_CH_WIDTH * ch;
 459         xc->vc.desc_free = uniphier_xdmac_desc_free;
 460
 461         vchan_init(&xc->vc, &xdev->ddev);
 462 }
 463
 464 static struct dma_chan *of_dma_uniphier_xlate(struct of_phandle_args *dma_spec,
 465                                               struct of_dma *ofdma)
 466 {
 467         struct uniphier_xdmac_device *xdev = ofdma->of_dma_data;
 468         int chan_id = dma_spec->args[0];
 469
 470         if (chan_id >= xdev->nr_chans)
 471                 return NULL;
 472
 473         xdev->channels[chan_id].id = chan_id;
 474         xdev->channels[chan_id].req_factor = dma_spec->args[1];
 475
 476         return dma_get_slave_channel(&xdev->channels[chan_id].vc.chan);
 477 }
 478
 479 static int uniphier_xdmac_probe(struct platform_device *pdev)
 480 {
 481         struct uniphier_xdmac_device *xdev;
 482         struct device *dev = &pdev->dev;
 483         struct dma_device *ddev;
 484         int irq;
 485         int nr_chans;
 486         int i, ret;
 487
 488         if (of_property_read_u32(dev->of_node, "dma-channels", &nr_chans))
 489                 return -EINVAL;
 490         if (nr_chans > XDMAC_MAX_CHANS)
 491                 nr_chans = XDMAC_MAX_CHANS;
 492
 493         xdev = devm_kzalloc(dev, struct_size(xdev, channels, nr_chans),
 494                             GFP_KERNEL);
 495         if (!xdev)
 496                 return -ENOMEM;
 497
 498         xdev->nr_chans = nr_chans;
 499         xdev->reg_base = devm_platform_ioremap_resource(pdev, 0);
 500         if (IS_ERR(xdev->reg_base))
 501                 return PTR_ERR(xdev->reg_base);
 502
 503         ddev = &xdev->ddev;
 504         ddev->dev = dev;
 505         dma_cap_zero(ddev->cap_mask);
 506         dma_cap_set(DMA_MEMCPY, ddev->cap_mask);
 507         dma_cap_set(DMA_SLAVE, ddev->cap_mask);
 508         ddev->src_addr_widths = UNIPHIER_XDMAC_BUSWIDTHS;
 509         ddev->dst_addr_widths = UNIPHIER_XDMAC_BUSWIDTHS;
 510         ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) |
 511                            BIT(DMA_MEM_TO_MEM);
 512         ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
 513         ddev->max_burst = XDMAC_MAX_WORDS;
 514         ddev->device_free_chan_resources = uniphier_xdmac_free_chan_resources;
 515         ddev->device_prep_dma_memcpy = uniphier_xdmac_prep_dma_memcpy;
 516         ddev->device_prep_slave_sg = uniphier_xdmac_prep_slave_sg;
 517         ddev->device_config = uniphier_xdmac_slave_config;
 518         ddev->device_terminate_all = uniphier_xdmac_terminate_all;
 519         ddev->device_synchronize = uniphier_xdmac_synchronize;
 520         ddev->device_tx_status = dma_cookie_status;
 521         ddev->device_issue_pending = uniphier_xdmac_issue_pending;
 522         INIT_LIST_HEAD(&ddev->channels);
 523
 524         for (i = 0; i < nr_chans; i++)
 525                 uniphier_xdmac_chan_init(xdev, i);
 526
 527         irq = platform_get_irq(pdev, 0);
 528         if (irq < 0)
 529                 return irq;
 530
 531         ret = devm_request_irq(dev, irq, uniphier_xdmac_irq_handler,
 532                                IRQF_SHARED, "xdmac", xdev);
 533         if (ret) {
 534                 dev_err(dev, "Failed to request IRQ\n");
 535                 return ret;
 536         }
 537
 538         ret = dma_async_device_register(ddev);
 539         if (ret) {
 540                 dev_err(dev, "Failed to register XDMA device\n");
 541                 return ret;
 542         }
 543
 544         ret = of_dma_controller_register(dev->of_node,
 545                                          of_dma_uniphier_xlate, xdev);
 546         if (ret) {
 547                 dev_err(dev, "Failed to register XDMA controller\n");
 548                 goto out_unregister_dmac;
 549         }
 550
 551         platform_set_drvdata(pdev, xdev);
 552
 553         dev_info(&pdev->dev, "UniPhier XDMAC driver (%d channels)\n",
 554                  nr_chans);
 555
 556         return 0;
 557
 558 out_unregister_dmac:
 559         dma_async_device_unregister(ddev);
 560
 561         return ret;
 562 }
 563
 564 static int uniphier_xdmac_remove(struct platform_device *pdev)
 565 {
 566         struct uniphier_xdmac_device *xdev = platform_get_drvdata(pdev);
 567         struct dma_device *ddev = &xdev->ddev;
 568         struct dma_chan *chan;
 569         int ret;
 570
 571         /*
 572          * Before reaching here, almost all descriptors have been freed by the
 573          * ->device_free_chan_resources() hook. However, each channel might
 574          * be still holding one descriptor that was on-flight at that moment.
 575          * Terminate it to make sure this hardware is no longer running. Then,
 576          * free the channel resources once again to avoid memory leak.
 577          */
 578         list_for_each_entry(chan, &ddev->channels, device_node) {
 579                 ret = dmaengine_terminate_sync(chan);
 580                 if (ret)
 581                         return ret;
 582                 uniphier_xdmac_free_chan_resources(chan);
 583         }
 584
 585         of_dma_controller_free(pdev->dev.of_node);
 586         dma_async_device_unregister(ddev);
 587
 588         return 0;
 589 }
 590
 591 static const struct of_device_id uniphier_xdmac_match[] = {
 592         { .compatible = "socionext,uniphier-xdmac" },
 593         { /* sentinel */ }
 594 };
 595 MODULE_DEVICE_TABLE(of, uniphier_xdmac_match);
 596
 597 static struct platform_driver uniphier_xdmac_driver = {
 598         .probe = uniphier_xdmac_probe,
 599         .remove = uniphier_xdmac_remove,
 600         .driver = {
 601                 .name = "uniphier-xdmac",
 602                 .of_match_table = uniphier_xdmac_match,
 603         },
 604 };
 605 module_platform_driver(uniphier_xdmac_driver);
 606
 607 MODULE_AUTHOR("Kunihiko Hayashi <hayashi.kunihiko@socionext.com>");
 608 MODULE_DESCRIPTION("UniPhier external DMA controller driver");
 609 MODULE_LICENSE("GPL v2");