drivers/dma/dw/core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Core driver for the Synopsys DesignWare DMA Controller
   4  *
   5  * Copyright (C) 2007-2008 Atmel Corporation
   6  * Copyright (C) 2010-2011 ST Microelectronics
   7  * Copyright (C) 2013 Intel Corporation
   8  */
   9
  10 #include <linux/bitops.h>
  11 #include <linux/delay.h>
  12 #include <linux/dmaengine.h>
  13 #include <linux/dma-mapping.h>
  14 #include <linux/dmapool.h>
  15 #include <linux/err.h>
  16 #include <linux/init.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/io.h>
  19 #include <linux/mm.h>
  20 #include <linux/module.h>
  21 #include <linux/slab.h>
  22 #include <linux/pm_runtime.h>
  23
  24 #include "../dmaengine.h"
  25 #include "internal.h"
  26
  27 /*
  28  * This supports the Synopsys "DesignWare AHB Central DMA Controller",
  29  * (DW_ahb_dmac) which is used with various AMBA 2.0 systems (not all
  30  * of which use ARM any more).  See the "Databook" from Synopsys for
  31  * information beyond what licensees probably provide.
  32  *
  33  * The driver has been tested with the Atmel AT32AP7000, which does not
  34  * support descriptor writeback.
  35  */
  36
  37 /* The set of bus widths supported by the DMA controller */
  38 #define DW_DMA_BUSWIDTHS                          \
  39         BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED)       | \
  40         BIT(DMA_SLAVE_BUSWIDTH_1_BYTE)          | \
  41         BIT(DMA_SLAVE_BUSWIDTH_2_BYTES)         | \
  42         BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
  43
  44 /*----------------------------------------------------------------------*/
  45
  46 static struct device *chan2dev(struct dma_chan *chan)
  47 {
  48         return &chan->dev->device;
  49 }
  50
  51 static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc)
  52 {
  53         return to_dw_desc(dwc->active_list.next);
  54 }
  55
  56 static dma_cookie_t dwc_tx_submit(struct dma_async_tx_descriptor *tx)
  57 {
  58         struct dw_desc          *desc = txd_to_dw_desc(tx);
  59         struct dw_dma_chan      *dwc = to_dw_dma_chan(tx->chan);
  60         dma_cookie_t            cookie;
  61         unsigned long           flags;
  62
  63         spin_lock_irqsave(&dwc->lock, flags);
  64         cookie = dma_cookie_assign(tx);
  65
  66         /*
  67          * REVISIT: We should attempt to chain as many descriptors as
  68          * possible, perhaps even appending to those already submitted
  69          * for DMA. But this is hard to do in a race-free manner.
  70          */
  71
  72         list_add_tail(&desc->desc_node, &dwc->queue);
  73         spin_unlock_irqrestore(&dwc->lock, flags);
  74         dev_vdbg(chan2dev(tx->chan), "%s: queued %u\n",
  75                  __func__, desc->txd.cookie);
  76
  77         return cookie;
  78 }
  79
  80 static struct dw_desc *dwc_desc_get(struct dw_dma_chan *dwc)
  81 {
  82         struct dw_dma *dw = to_dw_dma(dwc->chan.device);
  83         struct dw_desc *desc;
  84         dma_addr_t phys;
  85
  86         desc = dma_pool_zalloc(dw->desc_pool, GFP_ATOMIC, &phys);
  87         if (!desc)
  88                 return NULL;
  89
  90         dwc->descs_allocated++;
  91         INIT_LIST_HEAD(&desc->tx_list);
  92         dma_async_tx_descriptor_init(&desc->txd, &dwc->chan);
  93         desc->txd.tx_submit = dwc_tx_submit;
  94         desc->txd.flags = DMA_CTRL_ACK;
  95         desc->txd.phys = phys;
  96         return desc;
  97 }
  98
  99 static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)
 100 {
 101         struct dw_dma *dw = to_dw_dma(dwc->chan.device);
 102         struct dw_desc *child, *_next;
 103
 104         if (unlikely(!desc))
 105                 return;
 106
 107         list_for_each_entry_safe(child, _next, &desc->tx_list, desc_node) {
 108                 list_del(&child->desc_node);
 109                 dma_pool_free(dw->desc_pool, child, child->txd.phys);
 110                 dwc->descs_allocated--;
 111         }
 112
 113         dma_pool_free(dw->desc_pool, desc, desc->txd.phys);
 114         dwc->descs_allocated--;
 115 }
 116
 117 static void dwc_initialize(struct dw_dma_chan *dwc)
 118 {
 119         struct dw_dma *dw = to_dw_dma(dwc->chan.device);
 120
 121         dw->initialize_chan(dwc);
 122
 123         /* Enable interrupts */
 124         channel_set_bit(dw, MASK.XFER, dwc->mask);
 125         channel_set_bit(dw, MASK.ERROR, dwc->mask);
 126 }
 127
 128 /*----------------------------------------------------------------------*/
 129
 130 static inline void dwc_dump_chan_regs(struct dw_dma_chan *dwc)
 131 {
 132         dev_err(chan2dev(&dwc->chan),
 133                 "  SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
 134                 channel_readl(dwc, SAR),
 135                 channel_readl(dwc, DAR),
 136                 channel_readl(dwc, LLP),
 137                 channel_readl(dwc, CTL_HI),
 138                 channel_readl(dwc, CTL_LO));
 139 }
 140
 141 static inline void dwc_chan_disable(struct dw_dma *dw, struct dw_dma_chan *dwc)
 142 {
 143         channel_clear_bit(dw, CH_EN, dwc->mask);
 144         while (dma_readl(dw, CH_EN) & dwc->mask)
 145                 cpu_relax();
 146 }
 147
 148 /*----------------------------------------------------------------------*/
 149
 150 /* Perform single block transfer */
 151 static inline void dwc_do_single_block(struct dw_dma_chan *dwc,
 152                                        struct dw_desc *desc)
 153 {
 154         struct dw_dma   *dw = to_dw_dma(dwc->chan.device);
 155         u32             ctllo;
 156
 157         /*
 158          * Software emulation of LLP mode relies on interrupts to continue
 159          * multi block transfer.
 160          */
 161         ctllo = lli_read(desc, ctllo) | DWC_CTLL_INT_EN;
 162
 163         channel_writel(dwc, SAR, lli_read(desc, sar));
 164         channel_writel(dwc, DAR, lli_read(desc, dar));
 165         channel_writel(dwc, CTL_LO, ctllo);
 166         channel_writel(dwc, CTL_HI, lli_read(desc, ctlhi));
 167         channel_set_bit(dw, CH_EN, dwc->mask);
 168
 169         /* Move pointer to next descriptor */
 170         dwc->tx_node_active = dwc->tx_node_active->next;
 171 }
 172
 173 /* Called with dwc->lock held and bh disabled */
 174 static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first)
 175 {
 176         struct dw_dma   *dw = to_dw_dma(dwc->chan.device);
 177         u8              lms = DWC_LLP_LMS(dwc->dws.m_master);
 178         unsigned long   was_soft_llp;
 179
 180         /* ASSERT:  channel is idle */
 181         if (dma_readl(dw, CH_EN) & dwc->mask) {
 182                 dev_err(chan2dev(&dwc->chan),
 183                         "%s: BUG: Attempted to start non-idle channel\n",
 184                         __func__);
 185                 dwc_dump_chan_regs(dwc);
 186
 187                 /* The tasklet will hopefully advance the queue... */
 188                 return;
 189         }
 190
 191         if (dwc->nollp) {
 192                 was_soft_llp = test_and_set_bit(DW_DMA_IS_SOFT_LLP,
 193                                                 &dwc->flags);
 194                 if (was_soft_llp) {
 195                         dev_err(chan2dev(&dwc->chan),
 196                                 "BUG: Attempted to start new LLP transfer inside ongoing one\n");
 197                         return;
 198                 }
 199
 200                 dwc_initialize(dwc);
 201
 202                 first->residue = first->total_len;
 203                 dwc->tx_node_active = &first->tx_list;
 204
 205                 /* Submit first block */
 206                 dwc_do_single_block(dwc, first);
 207
 208                 return;
 209         }
 210
 211         dwc_initialize(dwc);
 212
 213         channel_writel(dwc, LLP, first->txd.phys | lms);
 214         channel_writel(dwc, CTL_LO, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
 215         channel_writel(dwc, CTL_HI, 0);
 216         channel_set_bit(dw, CH_EN, dwc->mask);
 217 }
 218
 219 static void dwc_dostart_first_queued(struct dw_dma_chan *dwc)
 220 {
 221         struct dw_desc *desc;
 222
 223         if (list_empty(&dwc->queue))
 224                 return;
 225
 226         list_move(dwc->queue.next, &dwc->active_list);
 227         desc = dwc_first_active(dwc);
 228         dev_vdbg(chan2dev(&dwc->chan), "%s: started %u\n", __func__, desc->txd.cookie);
 229         dwc_dostart(dwc, desc);
 230 }
 231
 232 /*----------------------------------------------------------------------*/
 233
 234 static void
 235 dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc,
 236                 bool callback_required)
 237 {
 238         struct dma_async_tx_descriptor  *txd = &desc->txd;
 239         struct dw_desc                  *child;
 240         unsigned long                   flags;
 241         struct dmaengine_desc_callback  cb;
 242
 243         dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie);
 244
 245         spin_lock_irqsave(&dwc->lock, flags);
 246         dma_cookie_complete(txd);
 247         if (callback_required)
 248                 dmaengine_desc_get_callback(txd, &cb);
 249         else
 250                 memset(&cb, 0, sizeof(cb));
 251
 252         /* async_tx_ack */
 253         list_for_each_entry(child, &desc->tx_list, desc_node)
 254                 async_tx_ack(&child->txd);
 255         async_tx_ack(&desc->txd);
 256         dwc_desc_put(dwc, desc);
 257         spin_unlock_irqrestore(&dwc->lock, flags);
 258
 259         dmaengine_desc_callback_invoke(&cb, NULL);
 260 }
 261
 262 static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc)
 263 {
 264         struct dw_desc *desc, *_desc;
 265         LIST_HEAD(list);
 266         unsigned long flags;
 267
 268         spin_lock_irqsave(&dwc->lock, flags);
 269         if (dma_readl(dw, CH_EN) & dwc->mask) {
 270                 dev_err(chan2dev(&dwc->chan),
 271                         "BUG: XFER bit set, but channel not idle!\n");
 272
 273                 /* Try to continue after resetting the channel... */
 274                 dwc_chan_disable(dw, dwc);
 275         }
 276
 277         /*
 278          * Submit queued descriptors ASAP, i.e. before we go through
 279          * the completed ones.
 280          */
 281         list_splice_init(&dwc->active_list, &list);
 282         dwc_dostart_first_queued(dwc);
 283
 284         spin_unlock_irqrestore(&dwc->lock, flags);
 285
 286         list_for_each_entry_safe(desc, _desc, &list, desc_node)
 287                 dwc_descriptor_complete(dwc, desc, true);
 288 }
 289
 290 /* Returns how many bytes were already received from source */
 291 static inline u32 dwc_get_sent(struct dw_dma_chan *dwc)
 292 {
 293         struct dw_dma *dw = to_dw_dma(dwc->chan.device);
 294         u32 ctlhi = channel_readl(dwc, CTL_HI);
 295         u32 ctllo = channel_readl(dwc, CTL_LO);
 296
 297         return dw->block2bytes(dwc, ctlhi, ctllo >> 4 & 7);
 298 }
 299
 300 static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
 301 {
 302         dma_addr_t llp;
 303         struct dw_desc *desc, *_desc;
 304         struct dw_desc *child;
 305         u32 status_xfer;
 306         unsigned long flags;
 307
 308         spin_lock_irqsave(&dwc->lock, flags);
 309         llp = channel_readl(dwc, LLP);
 310         status_xfer = dma_readl(dw, RAW.XFER);
 311
 312         if (status_xfer & dwc->mask) {
 313                 /* Everything we've submitted is done */
 314                 dma_writel(dw, CLEAR.XFER, dwc->mask);
 315
 316                 if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) {
 317                         struct list_head *head, *active = dwc->tx_node_active;
 318
 319                         /*
 320                          * We are inside first active descriptor.
 321                          * Otherwise something is really wrong.
 322                          */
 323                         desc = dwc_first_active(dwc);
 324
 325                         head = &desc->tx_list;
 326                         if (active != head) {
 327                                 /* Update residue to reflect last sent descriptor */
 328                                 if (active == head->next)
 329                                         desc->residue -= desc->len;
 330                                 else
 331                                         desc->residue -= to_dw_desc(active->prev)->len;
 332
 333                                 child = to_dw_desc(active);
 334
 335                                 /* Submit next block */
 336                                 dwc_do_single_block(dwc, child);
 337
 338                                 spin_unlock_irqrestore(&dwc->lock, flags);
 339                                 return;
 340                         }
 341
 342                         /* We are done here */
 343                         clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags);
 344                 }
 345
 346                 spin_unlock_irqrestore(&dwc->lock, flags);
 347
 348                 dwc_complete_all(dw, dwc);
 349                 return;
 350         }
 351
 352         if (list_empty(&dwc->active_list)) {
 353                 spin_unlock_irqrestore(&dwc->lock, flags);
 354                 return;
 355         }
 356
 357         if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) {
 358                 dev_vdbg(chan2dev(&dwc->chan), "%s: soft LLP mode\n", __func__);
 359                 spin_unlock_irqrestore(&dwc->lock, flags);
 360                 return;
 361         }
 362
 363         dev_vdbg(chan2dev(&dwc->chan), "%s: llp=%pad\n", __func__, &llp);
 364
 365         list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) {
 366                 /* Initial residue value */
 367                 desc->residue = desc->total_len;
 368
 369                 /* Check first descriptors addr */
 370                 if (desc->txd.phys == DWC_LLP_LOC(llp)) {
 371                         spin_unlock_irqrestore(&dwc->lock, flags);
 372                         return;
 373                 }
 374
 375                 /* Check first descriptors llp */
 376                 if (lli_read(desc, llp) == llp) {
 377                         /* This one is currently in progress */
 378                         desc->residue -= dwc_get_sent(dwc);
 379                         spin_unlock_irqrestore(&dwc->lock, flags);
 380                         return;
 381                 }
 382
 383                 desc->residue -= desc->len;
 384                 list_for_each_entry(child, &desc->tx_list, desc_node) {
 385                         if (lli_read(child, llp) == llp) {
 386                                 /* Currently in progress */
 387                                 desc->residue -= dwc_get_sent(dwc);
 388                                 spin_unlock_irqrestore(&dwc->lock, flags);
 389                                 return;
 390                         }
 391                         desc->residue -= child->len;
 392                 }
 393
 394                 /*
 395                  * No descriptors so far seem to be in progress, i.e.
 396                  * this one must be done.
 397                  */
 398                 spin_unlock_irqrestore(&dwc->lock, flags);
 399                 dwc_descriptor_complete(dwc, desc, true);
 400                 spin_lock_irqsave(&dwc->lock, flags);
 401         }
 402
 403         dev_err(chan2dev(&dwc->chan),
 404                 "BUG: All descriptors done, but channel not idle!\n");
 405
 406         /* Try to continue after resetting the channel... */
 407         dwc_chan_disable(dw, dwc);
 408
 409         dwc_dostart_first_queued(dwc);
 410         spin_unlock_irqrestore(&dwc->lock, flags);
 411 }
 412
 413 static inline void dwc_dump_lli(struct dw_dma_chan *dwc, struct dw_desc *desc)
 414 {
 415         dev_crit(chan2dev(&dwc->chan), "  desc: s0x%x d0x%x l0x%x c0x%x:%x\n",
 416                  lli_read(desc, sar),
 417                  lli_read(desc, dar),
 418                  lli_read(desc, llp),
 419                  lli_read(desc, ctlhi),
 420                  lli_read(desc, ctllo));
 421 }
 422
 423 static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
 424 {
 425         struct dw_desc *bad_desc;
 426         struct dw_desc *child;
 427         unsigned long flags;
 428
 429         dwc_scan_descriptors(dw, dwc);
 430
 431         spin_lock_irqsave(&dwc->lock, flags);
 432
 433         /*
 434          * The descriptor currently at the head of the active list is
 435          * borked. Since we don't have any way to report errors, we'll
 436          * just have to scream loudly and try to carry on.
 437          */
 438         bad_desc = dwc_first_active(dwc);
 439         list_del_init(&bad_desc->desc_node);
 440         list_move(dwc->queue.next, dwc->active_list.prev);
 441
 442         /* Clear the error flag and try to restart the controller */
 443         dma_writel(dw, CLEAR.ERROR, dwc->mask);
 444         if (!list_empty(&dwc->active_list))
 445                 dwc_dostart(dwc, dwc_first_active(dwc));
 446
 447         /*
 448          * WARN may seem harsh, but since this only happens
 449          * when someone submits a bad physical address in a
 450          * descriptor, we should consider ourselves lucky that the
 451          * controller flagged an error instead of scribbling over
 452          * random memory locations.
 453          */
 454         dev_WARN(chan2dev(&dwc->chan), "Bad descriptor submitted for DMA!\n"
 455                                        "  cookie: %d\n", bad_desc->txd.cookie);
 456         dwc_dump_lli(dwc, bad_desc);
 457         list_for_each_entry(child, &bad_desc->tx_list, desc_node)
 458                 dwc_dump_lli(dwc, child);
 459
 460         spin_unlock_irqrestore(&dwc->lock, flags);
 461
 462         /* Pretend the descriptor completed successfully */
 463         dwc_descriptor_complete(dwc, bad_desc, true);
 464 }
 465
 466 static void dw_dma_tasklet(unsigned long data)
 467 {
 468         struct dw_dma *dw = (struct dw_dma *)data;
 469         struct dw_dma_chan *dwc;
 470         u32 status_xfer;
 471         u32 status_err;
 472         unsigned int i;
 473
 474         status_xfer = dma_readl(dw, RAW.XFER);
 475         status_err = dma_readl(dw, RAW.ERROR);
 476
 477         dev_vdbg(dw->dma.dev, "%s: status_err=%x\n", __func__, status_err);
 478
 479         for (i = 0; i < dw->dma.chancnt; i++) {
 480                 dwc = &dw->chan[i];
 481                 if (test_bit(DW_DMA_IS_CYCLIC, &dwc->flags))
 482                         dev_vdbg(dw->dma.dev, "Cyclic xfer is not implemented\n");
 483                 else if (status_err & (1 << i))
 484                         dwc_handle_error(dw, dwc);
 485                 else if (status_xfer & (1 << i))
 486                         dwc_scan_descriptors(dw, dwc);
 487         }
 488
 489         /* Re-enable interrupts */
 490         channel_set_bit(dw, MASK.XFER, dw->all_chan_mask);
 491         channel_set_bit(dw, MASK.ERROR, dw->all_chan_mask);
 492 }
 493
 494 static irqreturn_t dw_dma_interrupt(int irq, void *dev_id)
 495 {
 496         struct dw_dma *dw = dev_id;
 497         u32 status;
 498
 499         /* Check if we have any interrupt from the DMAC which is not in use */
 500         if (!dw->in_use)
 501                 return IRQ_NONE;
 502
 503         status = dma_readl(dw, STATUS_INT);
 504         dev_vdbg(dw->dma.dev, "%s: status=0x%x\n", __func__, status);
 505
 506         /* Check if we have any interrupt from the DMAC */
 507         if (!status)
 508                 return IRQ_NONE;
 509
 510         /*
 511          * Just disable the interrupts. We'll turn them back on in the
 512          * softirq handler.
 513          */
 514         channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
 515         channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
 516         channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
 517
 518         status = dma_readl(dw, STATUS_INT);
 519         if (status) {
 520                 dev_err(dw->dma.dev,
 521                         "BUG: Unexpected interrupts pending: 0x%x\n",
 522                         status);
 523
 524                 /* Try to recover */
 525                 channel_clear_bit(dw, MASK.XFER, (1 << 8) - 1);
 526                 channel_clear_bit(dw, MASK.BLOCK, (1 << 8) - 1);
 527                 channel_clear_bit(dw, MASK.SRC_TRAN, (1 << 8) - 1);
 528                 channel_clear_bit(dw, MASK.DST_TRAN, (1 << 8) - 1);
 529                 channel_clear_bit(dw, MASK.ERROR, (1 << 8) - 1);
 530         }
 531
 532         tasklet_schedule(&dw->tasklet);
 533
 534         return IRQ_HANDLED;
 535 }
 536
 537 /*----------------------------------------------------------------------*/
 538
 539 static struct dma_async_tx_descriptor *
 540 dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 541                 size_t len, unsigned long flags)
 542 {
 543         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 544         struct dw_dma           *dw = to_dw_dma(chan->device);
 545         struct dw_desc          *desc;
 546         struct dw_desc          *first;
 547         struct dw_desc          *prev;
 548         size_t                  xfer_count;
 549         size_t                  offset;
 550         u8                      m_master = dwc->dws.m_master;
 551         unsigned int            src_width;
 552         unsigned int            dst_width;
 553         unsigned int            data_width = dw->pdata->data_width[m_master];
 554         u32                     ctllo, ctlhi;
 555         u8                      lms = DWC_LLP_LMS(m_master);
 556
 557         dev_vdbg(chan2dev(chan),
 558                         "%s: d%pad s%pad l0x%zx f0x%lx\n", __func__,
 559                         &dest, &src, len, flags);
 560
 561         if (unlikely(!len)) {
 562                 dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__);
 563                 return NULL;
 564         }
 565
 566         dwc->direction = DMA_MEM_TO_MEM;
 567
 568         src_width = dst_width = __ffs(data_width | src | dest | len);
 569
 570         ctllo = dw->prepare_ctllo(dwc)
 571                         | DWC_CTLL_DST_WIDTH(dst_width)
 572                         | DWC_CTLL_SRC_WIDTH(src_width)
 573                         | DWC_CTLL_DST_INC
 574                         | DWC_CTLL_SRC_INC
 575                         | DWC_CTLL_FC_M2M;
 576         prev = first = NULL;
 577
 578         for (offset = 0; offset < len; offset += xfer_count) {
 579                 desc = dwc_desc_get(dwc);
 580                 if (!desc)
 581                         goto err_desc_get;
 582
 583                 ctlhi = dw->bytes2block(dwc, len - offset, src_width, &xfer_count);
 584
 585                 lli_write(desc, sar, src + offset);
 586                 lli_write(desc, dar, dest + offset);
 587                 lli_write(desc, ctllo, ctllo);
 588                 lli_write(desc, ctlhi, ctlhi);
 589                 desc->len = xfer_count;
 590
 591                 if (!first) {
 592                         first = desc;
 593                 } else {
 594                         lli_write(prev, llp, desc->txd.phys | lms);
 595                         list_add_tail(&desc->desc_node, &first->tx_list);
 596                 }
 597                 prev = desc;
 598         }
 599
 600         if (flags & DMA_PREP_INTERRUPT)
 601                 /* Trigger interrupt after last block */
 602                 lli_set(prev, ctllo, DWC_CTLL_INT_EN);
 603
 604         prev->lli.llp = 0;
 605         lli_clear(prev, ctllo, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
 606         first->txd.flags = flags;
 607         first->total_len = len;
 608
 609         return &first->txd;
 610
 611 err_desc_get:
 612         dwc_desc_put(dwc, first);
 613         return NULL;
 614 }
 615
 616 static struct dma_async_tx_descriptor *
 617 dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 618                 unsigned int sg_len, enum dma_transfer_direction direction,
 619                 unsigned long flags, void *context)
 620 {
 621         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 622         struct dw_dma           *dw = to_dw_dma(chan->device);
 623         struct dma_slave_config *sconfig = &dwc->dma_sconfig;
 624         struct dw_desc          *prev;
 625         struct dw_desc          *first;
 626         u32                     ctllo, ctlhi;
 627         u8                      m_master = dwc->dws.m_master;
 628         u8                      lms = DWC_LLP_LMS(m_master);
 629         dma_addr_t              reg;
 630         unsigned int            reg_width;
 631         unsigned int            mem_width;
 632         unsigned int            data_width = dw->pdata->data_width[m_master];
 633         unsigned int            i;
 634         struct scatterlist      *sg;
 635         size_t                  total_len = 0;
 636
 637         dev_vdbg(chan2dev(chan), "%s\n", __func__);
 638
 639         if (unlikely(!is_slave_direction(direction) || !sg_len))
 640                 return NULL;
 641
 642         dwc->direction = direction;
 643
 644         prev = first = NULL;
 645
 646         switch (direction) {
 647         case DMA_MEM_TO_DEV:
 648                 reg_width = __ffs(sconfig->dst_addr_width);
 649                 reg = sconfig->dst_addr;
 650                 ctllo = dw->prepare_ctllo(dwc)
 651                                 | DWC_CTLL_DST_WIDTH(reg_width)
 652                                 | DWC_CTLL_DST_FIX
 653                                 | DWC_CTLL_SRC_INC;
 654
 655                 ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_M2P) :
 656                         DWC_CTLL_FC(DW_DMA_FC_D_M2P);
 657
 658                 for_each_sg(sgl, sg, sg_len, i) {
 659                         struct dw_desc  *desc;
 660                         u32             len, mem;
 661                         size_t          dlen;
 662
 663                         mem = sg_dma_address(sg);
 664                         len = sg_dma_len(sg);
 665
 666                         mem_width = __ffs(data_width | mem | len);
 667
 668 slave_sg_todev_fill_desc:
 669                         desc = dwc_desc_get(dwc);
 670                         if (!desc)
 671                                 goto err_desc_get;
 672
 673                         ctlhi = dw->bytes2block(dwc, len, mem_width, &dlen);
 674
 675                         lli_write(desc, sar, mem);
 676                         lli_write(desc, dar, reg);
 677                         lli_write(desc, ctlhi, ctlhi);
 678                         lli_write(desc, ctllo, ctllo | DWC_CTLL_SRC_WIDTH(mem_width));
 679                         desc->len = dlen;
 680
 681                         if (!first) {
 682                                 first = desc;
 683                         } else {
 684                                 lli_write(prev, llp, desc->txd.phys | lms);
 685                                 list_add_tail(&desc->desc_node, &first->tx_list);
 686                         }
 687                         prev = desc;
 688
 689                         mem += dlen;
 690                         len -= dlen;
 691                         total_len += dlen;
 692
 693                         if (len)
 694                                 goto slave_sg_todev_fill_desc;
 695                 }
 696                 break;
 697         case DMA_DEV_TO_MEM:
 698                 reg_width = __ffs(sconfig->src_addr_width);
 699                 reg = sconfig->src_addr;
 700                 ctllo = dw->prepare_ctllo(dwc)
 701                                 | DWC_CTLL_SRC_WIDTH(reg_width)
 702                                 | DWC_CTLL_DST_INC
 703                                 | DWC_CTLL_SRC_FIX;
 704
 705                 ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_P2M) :
 706                         DWC_CTLL_FC(DW_DMA_FC_D_P2M);
 707
 708                 for_each_sg(sgl, sg, sg_len, i) {
 709                         struct dw_desc  *desc;
 710                         u32             len, mem;
 711                         size_t          dlen;
 712
 713                         mem = sg_dma_address(sg);
 714                         len = sg_dma_len(sg);
 715
 716 slave_sg_fromdev_fill_desc:
 717                         desc = dwc_desc_get(dwc);
 718                         if (!desc)
 719                                 goto err_desc_get;
 720
 721                         ctlhi = dw->bytes2block(dwc, len, reg_width, &dlen);
 722
 723                         lli_write(desc, sar, reg);
 724                         lli_write(desc, dar, mem);
 725                         lli_write(desc, ctlhi, ctlhi);
 726                         mem_width = __ffs(data_width | mem | dlen);
 727                         lli_write(desc, ctllo, ctllo | DWC_CTLL_DST_WIDTH(mem_width));
 728                         desc->len = dlen;
 729
 730                         if (!first) {
 731                                 first = desc;
 732                         } else {
 733                                 lli_write(prev, llp, desc->txd.phys | lms);
 734                                 list_add_tail(&desc->desc_node, &first->tx_list);
 735                         }
 736                         prev = desc;
 737
 738                         mem += dlen;
 739                         len -= dlen;
 740                         total_len += dlen;
 741
 742                         if (len)
 743                                 goto slave_sg_fromdev_fill_desc;
 744                 }
 745                 break;
 746         default:
 747                 return NULL;
 748         }
 749
 750         if (flags & DMA_PREP_INTERRUPT)
 751                 /* Trigger interrupt after last block */
 752                 lli_set(prev, ctllo, DWC_CTLL_INT_EN);
 753
 754         prev->lli.llp = 0;
 755         lli_clear(prev, ctllo, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
 756         first->total_len = total_len;
 757
 758         return &first->txd;
 759
 760 err_desc_get:
 761         dev_err(chan2dev(chan),
 762                 "not enough descriptors available. Direction %d\n", direction);
 763         dwc_desc_put(dwc, first);
 764         return NULL;
 765 }
 766
 767 bool dw_dma_filter(struct dma_chan *chan, void *param)
 768 {
 769         struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
 770         struct dw_dma_slave *dws = param;
 771
 772         if (dws->dma_dev != chan->device->dev)
 773                 return false;
 774
 775         /* We have to copy data since dws can be temporary storage */
 776         memcpy(&dwc->dws, dws, sizeof(struct dw_dma_slave));
 777
 778         return true;
 779 }
 780 EXPORT_SYMBOL_GPL(dw_dma_filter);
 781
 782 static int dwc_config(struct dma_chan *chan, struct dma_slave_config *sconfig)
 783 {
 784         struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
 785         struct dw_dma *dw = to_dw_dma(chan->device);
 786
 787         memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig));
 788
 789         dw->encode_maxburst(dwc, &dwc->dma_sconfig.src_maxburst);
 790         dw->encode_maxburst(dwc, &dwc->dma_sconfig.dst_maxburst);
 791
 792         return 0;
 793 }
 794
 795 static void dwc_chan_pause(struct dw_dma_chan *dwc, bool drain)
 796 {
 797         struct dw_dma *dw = to_dw_dma(dwc->chan.device);
 798         unsigned int            count = 20;     /* timeout iterations */
 799
 800         dw->suspend_chan(dwc, drain);
 801
 802         while (!(channel_readl(dwc, CFG_LO) & DWC_CFGL_FIFO_EMPTY) && count--)
 803                 udelay(2);
 804
 805         set_bit(DW_DMA_IS_PAUSED, &dwc->flags);
 806 }
 807
 808 static int dwc_pause(struct dma_chan *chan)
 809 {
 810         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 811         unsigned long           flags;
 812
 813         spin_lock_irqsave(&dwc->lock, flags);
 814         dwc_chan_pause(dwc, false);
 815         spin_unlock_irqrestore(&dwc->lock, flags);
 816
 817         return 0;
 818 }
 819
 820 static inline void dwc_chan_resume(struct dw_dma_chan *dwc, bool drain)
 821 {
 822         struct dw_dma *dw = to_dw_dma(dwc->chan.device);
 823
 824         dw->resume_chan(dwc, drain);
 825
 826         clear_bit(DW_DMA_IS_PAUSED, &dwc->flags);
 827 }
 828
 829 static int dwc_resume(struct dma_chan *chan)
 830 {
 831         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 832         unsigned long           flags;
 833
 834         spin_lock_irqsave(&dwc->lock, flags);
 835
 836         if (test_bit(DW_DMA_IS_PAUSED, &dwc->flags))
 837                 dwc_chan_resume(dwc, false);
 838
 839         spin_unlock_irqrestore(&dwc->lock, flags);
 840
 841         return 0;
 842 }
 843
 844 static int dwc_terminate_all(struct dma_chan *chan)
 845 {
 846         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 847         struct dw_dma           *dw = to_dw_dma(chan->device);
 848         struct dw_desc          *desc, *_desc;
 849         unsigned long           flags;
 850         LIST_HEAD(list);
 851
 852         spin_lock_irqsave(&dwc->lock, flags);
 853
 854         clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags);
 855
 856         dwc_chan_pause(dwc, true);
 857
 858         dwc_chan_disable(dw, dwc);
 859
 860         dwc_chan_resume(dwc, true);
 861
 862         /* active_list entries will end up before queued entries */
 863         list_splice_init(&dwc->queue, &list);
 864         list_splice_init(&dwc->active_list, &list);
 865
 866         spin_unlock_irqrestore(&dwc->lock, flags);
 867
 868         /* Flush all pending and queued descriptors */
 869         list_for_each_entry_safe(desc, _desc, &list, desc_node)
 870                 dwc_descriptor_complete(dwc, desc, false);
 871
 872         return 0;
 873 }
 874
 875 static struct dw_desc *dwc_find_desc(struct dw_dma_chan *dwc, dma_cookie_t c)
 876 {
 877         struct dw_desc *desc;
 878
 879         list_for_each_entry(desc, &dwc->active_list, desc_node)
 880                 if (desc->txd.cookie == c)
 881                         return desc;
 882
 883         return NULL;
 884 }
 885
 886 static u32 dwc_get_residue(struct dw_dma_chan *dwc, dma_cookie_t cookie)
 887 {
 888         struct dw_desc *desc;
 889         unsigned long flags;
 890         u32 residue;
 891
 892         spin_lock_irqsave(&dwc->lock, flags);
 893
 894         desc = dwc_find_desc(dwc, cookie);
 895         if (desc) {
 896                 if (desc == dwc_first_active(dwc)) {
 897                         residue = desc->residue;
 898                         if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags) && residue)
 899                                 residue -= dwc_get_sent(dwc);
 900                 } else {
 901                         residue = desc->total_len;
 902                 }
 903         } else {
 904                 residue = 0;
 905         }
 906
 907         spin_unlock_irqrestore(&dwc->lock, flags);
 908         return residue;
 909 }
 910
 911 static enum dma_status
 912 dwc_tx_status(struct dma_chan *chan,
 913               dma_cookie_t cookie,
 914               struct dma_tx_state *txstate)
 915 {
 916         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 917         enum dma_status         ret;
 918
 919         ret = dma_cookie_status(chan, cookie, txstate);
 920         if (ret == DMA_COMPLETE)
 921                 return ret;
 922
 923         dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
 924
 925         ret = dma_cookie_status(chan, cookie, txstate);
 926         if (ret == DMA_COMPLETE)
 927                 return ret;
 928
 929         dma_set_residue(txstate, dwc_get_residue(dwc, cookie));
 930
 931         if (test_bit(DW_DMA_IS_PAUSED, &dwc->flags) && ret == DMA_IN_PROGRESS)
 932                 return DMA_PAUSED;
 933
 934         return ret;
 935 }
 936
 937 static void dwc_issue_pending(struct dma_chan *chan)
 938 {
 939         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 940         unsigned long           flags;
 941
 942         spin_lock_irqsave(&dwc->lock, flags);
 943         if (list_empty(&dwc->active_list))
 944                 dwc_dostart_first_queued(dwc);
 945         spin_unlock_irqrestore(&dwc->lock, flags);
 946 }
 947
 948 /*----------------------------------------------------------------------*/
 949
 950 void do_dw_dma_off(struct dw_dma *dw)
 951 {
 952         dma_writel(dw, CFG, 0);
 953
 954         channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
 955         channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
 956         channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask);
 957         channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask);
 958         channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
 959
 960         while (dma_readl(dw, CFG) & DW_CFG_DMA_EN)
 961                 cpu_relax();
 962 }
 963
 964 void do_dw_dma_on(struct dw_dma *dw)
 965 {
 966         dma_writel(dw, CFG, DW_CFG_DMA_EN);
 967 }
 968
 969 static int dwc_alloc_chan_resources(struct dma_chan *chan)
 970 {
 971         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 972         struct dw_dma           *dw = to_dw_dma(chan->device);
 973
 974         dev_vdbg(chan2dev(chan), "%s\n", __func__);
 975
 976         /* ASSERT:  channel is idle */
 977         if (dma_readl(dw, CH_EN) & dwc->mask) {
 978                 dev_dbg(chan2dev(chan), "DMA channel not idle?\n");
 979                 return -EIO;
 980         }
 981
 982         dma_cookie_init(chan);
 983
 984         /*
 985          * NOTE: some controllers may have additional features that we
 986          * need to initialize here, like "scatter-gather" (which
 987          * doesn't mean what you think it means), and status writeback.
 988          */
 989
 990         /*
 991          * We need controller-specific data to set up slave transfers.
 992          */
 993         if (chan->private && !dw_dma_filter(chan, chan->private)) {
 994                 dev_warn(chan2dev(chan), "Wrong controller-specific data\n");
 995                 return -EINVAL;
 996         }
 997
 998         /* Enable controller here if needed */
 999         if (!dw->in_use)
1000                 do_dw_dma_on(dw);
1001         dw->in_use |= dwc->mask;
1002
1003         return 0;
1004 }
1005
1006 static void dwc_free_chan_resources(struct dma_chan *chan)
1007 {
1008         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
1009         struct dw_dma           *dw = to_dw_dma(chan->device);
1010         unsigned long           flags;
1011
1012         dev_dbg(chan2dev(chan), "%s: descs allocated=%u\n", __func__,
1013                         dwc->descs_allocated);
1014
1015         /* ASSERT:  channel is idle */
1016         BUG_ON(!list_empty(&dwc->active_list));
1017         BUG_ON(!list_empty(&dwc->queue));
1018         BUG_ON(dma_readl(to_dw_dma(chan->device), CH_EN) & dwc->mask);
1019
1020         spin_lock_irqsave(&dwc->lock, flags);
1021
1022         /* Clear custom channel configuration */
1023         memset(&dwc->dws, 0, sizeof(struct dw_dma_slave));
1024
1025         /* Disable interrupts */
1026         channel_clear_bit(dw, MASK.XFER, dwc->mask);
1027         channel_clear_bit(dw, MASK.BLOCK, dwc->mask);
1028         channel_clear_bit(dw, MASK.ERROR, dwc->mask);
1029
1030         spin_unlock_irqrestore(&dwc->lock, flags);
1031
1032         /* Disable controller in case it was a last user */
1033         dw->in_use &= ~dwc->mask;
1034         if (!dw->in_use)
1035                 do_dw_dma_off(dw);
1036
1037         dev_vdbg(chan2dev(chan), "%s: done\n", __func__);
1038 }
1039
1040 int do_dma_probe(struct dw_dma_chip *chip)
1041 {
1042         struct dw_dma *dw = chip->dw;
1043         struct dw_dma_platform_data *pdata;
1044         bool                    autocfg = false;
1045         unsigned int            dw_params;
1046         unsigned int            i;
1047         int                     err;
1048
1049         dw->pdata = devm_kzalloc(chip->dev, sizeof(*dw->pdata), GFP_KERNEL);
1050         if (!dw->pdata)
1051                 return -ENOMEM;
1052
1053         dw->regs = chip->regs;
1054
1055         pm_runtime_get_sync(chip->dev);
1056
1057         if (!chip->pdata) {
1058                 dw_params = dma_readl(dw, DW_PARAMS);
1059                 dev_dbg(chip->dev, "DW_PARAMS: 0x%08x\n", dw_params);
1060
1061                 autocfg = dw_params >> DW_PARAMS_EN & 1;
1062                 if (!autocfg) {
1063                         err = -EINVAL;
1064                         goto err_pdata;
1065                 }
1066
1067                 /* Reassign the platform data pointer */
1068                 pdata = dw->pdata;
1069
1070                 /* Get hardware configuration parameters */
1071                 pdata->nr_channels = (dw_params >> DW_PARAMS_NR_CHAN & 7) + 1;
1072                 pdata->nr_masters = (dw_params >> DW_PARAMS_NR_MASTER & 3) + 1;
1073                 for (i = 0; i < pdata->nr_masters; i++) {
1074                         pdata->data_width[i] =
1075                                 4 << (dw_params >> DW_PARAMS_DATA_WIDTH(i) & 3);
1076                 }
1077                 pdata->block_size = dma_readl(dw, MAX_BLK_SIZE);
1078
1079                 /* Fill platform data with the default values */
1080                 pdata->chan_allocation_order = CHAN_ALLOCATION_ASCENDING;
1081                 pdata->chan_priority = CHAN_PRIORITY_ASCENDING;
1082         } else if (chip->pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS) {
1083                 err = -EINVAL;
1084                 goto err_pdata;
1085         } else {
1086                 memcpy(dw->pdata, chip->pdata, sizeof(*dw->pdata));
1087
1088                 /* Reassign the platform data pointer */
1089                 pdata = dw->pdata;
1090         }
1091
1092         dw->chan = devm_kcalloc(chip->dev, pdata->nr_channels, sizeof(*dw->chan),
1093                                 GFP_KERNEL);
1094         if (!dw->chan) {
1095                 err = -ENOMEM;
1096                 goto err_pdata;
1097         }
1098
1099         /* Calculate all channel mask before DMA setup */
1100         dw->all_chan_mask = (1 << pdata->nr_channels) - 1;
1101
1102         /* Force dma off, just in case */
1103         dw->disable(dw);
1104
1105         /* Device and instance ID for IRQ and DMA pool */
1106         dw->set_device_name(dw, chip->id);
1107
1108         /* Create a pool of consistent memory blocks for hardware descriptors */
1109         dw->desc_pool = dmam_pool_create(dw->name, chip->dev,
1110                                          sizeof(struct dw_desc), 4, 0);
1111         if (!dw->desc_pool) {
1112                 dev_err(chip->dev, "No memory for descriptors dma pool\n");
1113                 err = -ENOMEM;
1114                 goto err_pdata;
1115         }
1116
1117         tasklet_init(&dw->tasklet, dw_dma_tasklet, (unsigned long)dw);
1118
1119         err = request_irq(chip->irq, dw_dma_interrupt, IRQF_SHARED,
1120                           dw->name, dw);
1121         if (err)
1122                 goto err_pdata;
1123
1124         INIT_LIST_HEAD(&dw->dma.channels);
1125         for (i = 0; i < pdata->nr_channels; i++) {
1126                 struct dw_dma_chan      *dwc = &dw->chan[i];
1127
1128                 dwc->chan.device = &dw->dma;
1129                 dma_cookie_init(&dwc->chan);
1130                 if (pdata->chan_allocation_order == CHAN_ALLOCATION_ASCENDING)
1131                         list_add_tail(&dwc->chan.device_node,
1132                                         &dw->dma.channels);
1133                 else
1134                         list_add(&dwc->chan.device_node, &dw->dma.channels);
1135
1136                 /* 7 is highest priority & 0 is lowest. */
1137                 if (pdata->chan_priority == CHAN_PRIORITY_ASCENDING)
1138                         dwc->priority = pdata->nr_channels - i - 1;
1139                 else
1140                         dwc->priority = i;
1141
1142                 dwc->ch_regs = &__dw_regs(dw)->CHAN[i];
1143                 spin_lock_init(&dwc->lock);
1144                 dwc->mask = 1 << i;
1145
1146                 INIT_LIST_HEAD(&dwc->active_list);
1147                 INIT_LIST_HEAD(&dwc->queue);
1148
1149                 channel_clear_bit(dw, CH_EN, dwc->mask);
1150
1151                 dwc->direction = DMA_TRANS_NONE;
1152
1153                 /* Hardware configuration */
1154                 if (autocfg) {
1155                         unsigned int r = DW_DMA_MAX_NR_CHANNELS - i - 1;
1156                         void __iomem *addr = &__dw_regs(dw)->DWC_PARAMS[r];
1157                         unsigned int dwc_params = readl(addr);
1158
1159                         dev_dbg(chip->dev, "DWC_PARAMS[%d]: 0x%08x\n", i,
1160                                            dwc_params);
1161
1162                         /*
1163                          * Decode maximum block size for given channel. The
1164                          * stored 4 bit value represents blocks from 0x00 for 3
1165                          * up to 0x0a for 4095.
1166                          */
1167                         dwc->block_size =
1168                                 (4 << ((pdata->block_size >> 4 * i) & 0xf)) - 1;
1169                         dwc->nollp =
1170                                 (dwc_params >> DWC_PARAMS_MBLK_EN & 0x1) == 0;
1171                 } else {
1172                         dwc->block_size = pdata->block_size;
1173                         dwc->nollp = !pdata->multi_block[i];
1174                 }
1175         }
1176
1177         /* Clear all interrupts on all channels. */
1178         dma_writel(dw, CLEAR.XFER, dw->all_chan_mask);
1179         dma_writel(dw, CLEAR.BLOCK, dw->all_chan_mask);
1180         dma_writel(dw, CLEAR.SRC_TRAN, dw->all_chan_mask);
1181         dma_writel(dw, CLEAR.DST_TRAN, dw->all_chan_mask);
1182         dma_writel(dw, CLEAR.ERROR, dw->all_chan_mask);
1183
1184         /* Set capabilities */
1185         dma_cap_set(DMA_SLAVE, dw->dma.cap_mask);
1186         dma_cap_set(DMA_PRIVATE, dw->dma.cap_mask);
1187         dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask);
1188
1189         dw->dma.dev = chip->dev;
1190         dw->dma.device_alloc_chan_resources = dwc_alloc_chan_resources;
1191         dw->dma.device_free_chan_resources = dwc_free_chan_resources;
1192
1193         dw->dma.device_prep_dma_memcpy = dwc_prep_dma_memcpy;
1194         dw->dma.device_prep_slave_sg = dwc_prep_slave_sg;
1195
1196         dw->dma.device_config = dwc_config;
1197         dw->dma.device_pause = dwc_pause;
1198         dw->dma.device_resume = dwc_resume;
1199         dw->dma.device_terminate_all = dwc_terminate_all;
1200
1201         dw->dma.device_tx_status = dwc_tx_status;
1202         dw->dma.device_issue_pending = dwc_issue_pending;
1203
1204         /* DMA capabilities */
1205         dw->dma.src_addr_widths = DW_DMA_BUSWIDTHS;
1206         dw->dma.dst_addr_widths = DW_DMA_BUSWIDTHS;
1207         dw->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) |
1208                              BIT(DMA_MEM_TO_MEM);
1209         dw->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1210
1211         err = dma_async_device_register(&dw->dma);
1212         if (err)
1213                 goto err_dma_register;
1214
1215         dev_info(chip->dev, "DesignWare DMA Controller, %d channels\n",
1216                  pdata->nr_channels);
1217
1218         pm_runtime_put_sync_suspend(chip->dev);
1219
1220         return 0;
1221
1222 err_dma_register:
1223         free_irq(chip->irq, dw);
1224 err_pdata:
1225         pm_runtime_put_sync_suspend(chip->dev);
1226         return err;
1227 }
1228
1229 int do_dma_remove(struct dw_dma_chip *chip)
1230 {
1231         struct dw_dma           *dw = chip->dw;
1232         struct dw_dma_chan      *dwc, *_dwc;
1233
1234         pm_runtime_get_sync(chip->dev);
1235
1236         do_dw_dma_off(dw);
1237         dma_async_device_unregister(&dw->dma);
1238
1239         free_irq(chip->irq, dw);
1240         tasklet_kill(&dw->tasklet);
1241
1242         list_for_each_entry_safe(dwc, _dwc, &dw->dma.channels,
1243                         chan.device_node) {
1244                 list_del(&dwc->chan.device_node);
1245                 channel_clear_bit(dw, CH_EN, dwc->mask);
1246         }
1247
1248         pm_runtime_put_sync_suspend(chip->dev);
1249         return 0;
1250 }
1251
1252 int do_dw_dma_disable(struct dw_dma_chip *chip)
1253 {
1254         struct dw_dma *dw = chip->dw;
1255
1256         dw->disable(dw);
1257         return 0;
1258 }
1259 EXPORT_SYMBOL_GPL(do_dw_dma_disable);
1260
1261 int do_dw_dma_enable(struct dw_dma_chip *chip)
1262 {
1263         struct dw_dma *dw = chip->dw;
1264
1265         dw->enable(dw);
1266         return 0;
1267 }
1268 EXPORT_SYMBOL_GPL(do_dw_dma_enable);
1269
1270 MODULE_LICENSE("GPL v2");
1271 MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller core driver");
1272 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1273 MODULE_AUTHOR("Viresh Kumar <vireshk@kernel.org>");