drivers/dma/dw-edma/dw-edma-core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
   4  * Synopsys DesignWare eDMA core driver
   5  *
   6  * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
   7  */
   8
   9 #include <linux/module.h>
  10 #include <linux/device.h>
  11 #include <linux/kernel.h>
  12 #include <linux/pm_runtime.h>
  13 #include <linux/dmaengine.h>
  14 #include <linux/err.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/irq.h>
  17 #include <linux/dma/edma.h>
  18 #include <linux/dma-mapping.h>
  19
  20 #include "dw-edma-core.h"
  21 #include "dw-edma-v0-core.h"
  22 #include "../dmaengine.h"
  23 #include "../virt-dma.h"
  24
  25 static inline
  26 struct device *dchan2dev(struct dma_chan *dchan)
  27 {
  28         return &dchan->dev->device;
  29 }
  30
  31 static inline
  32 struct device *chan2dev(struct dw_edma_chan *chan)
  33 {
  34         return &chan->vc.chan.dev->device;
  35 }
  36
  37 static inline
  38 struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd)
  39 {
  40         return container_of(vd, struct dw_edma_desc, vd);
  41 }
  42
  43 static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk)
  44 {
  45         struct dw_edma_burst *burst;
  46
  47         burst = kzalloc(sizeof(*burst), GFP_NOWAIT);
  48         if (unlikely(!burst))
  49                 return NULL;
  50
  51         INIT_LIST_HEAD(&burst->list);
  52         if (chunk->burst) {
  53                 /* Create and add new element into the linked list */
  54                 chunk->bursts_alloc++;
  55                 list_add_tail(&burst->list, &chunk->burst->list);
  56         } else {
  57                 /* List head */
  58                 chunk->bursts_alloc = 0;
  59                 chunk->burst = burst;
  60         }
  61
  62         return burst;
  63 }
  64
  65 static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc)
  66 {
  67         struct dw_edma_chan *chan = desc->chan;
  68         struct dw_edma *dw = chan->chip->dw;
  69         struct dw_edma_chunk *chunk;
  70
  71         chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT);
  72         if (unlikely(!chunk))
  73                 return NULL;
  74
  75         INIT_LIST_HEAD(&chunk->list);
  76         chunk->chan = chan;
  77         /* Toggling change bit (CB) in each chunk, this is a mechanism to
  78          * inform the eDMA HW block that this is a new linked list ready
  79          * to be consumed.
  80          *  - Odd chunks originate CB equal to 0
  81          *  - Even chunks originate CB equal to 1
  82          */
  83         chunk->cb = !(desc->chunks_alloc % 2);
  84         if (chan->dir == EDMA_DIR_WRITE) {
  85                 chunk->ll_region.paddr = dw->ll_region_wr[chan->id].paddr;
  86                 chunk->ll_region.vaddr = dw->ll_region_wr[chan->id].vaddr;
  87         } else {
  88                 chunk->ll_region.paddr = dw->ll_region_rd[chan->id].paddr;
  89                 chunk->ll_region.vaddr = dw->ll_region_rd[chan->id].vaddr;
  90         }
  91
  92         if (desc->chunk) {
  93                 /* Create and add new element into the linked list */
  94                 if (!dw_edma_alloc_burst(chunk)) {
  95                         kfree(chunk);
  96                         return NULL;
  97                 }
  98                 desc->chunks_alloc++;
  99                 list_add_tail(&chunk->list, &desc->chunk->list);
 100         } else {
 101                 /* List head */
 102                 chunk->burst = NULL;
 103                 desc->chunks_alloc = 0;
 104                 desc->chunk = chunk;
 105         }
 106
 107         return chunk;
 108 }
 109
 110 static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan)
 111 {
 112         struct dw_edma_desc *desc;
 113
 114         desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
 115         if (unlikely(!desc))
 116                 return NULL;
 117
 118         desc->chan = chan;
 119         if (!dw_edma_alloc_chunk(desc)) {
 120                 kfree(desc);
 121                 return NULL;
 122         }
 123
 124         return desc;
 125 }
 126
 127 static void dw_edma_free_burst(struct dw_edma_chunk *chunk)
 128 {
 129         struct dw_edma_burst *child, *_next;
 130
 131         /* Remove all the list elements */
 132         list_for_each_entry_safe(child, _next, &chunk->burst->list, list) {
 133                 list_del(&child->list);
 134                 kfree(child);
 135                 chunk->bursts_alloc--;
 136         }
 137
 138         /* Remove the list head */
 139         kfree(child);
 140         chunk->burst = NULL;
 141 }
 142
 143 static void dw_edma_free_chunk(struct dw_edma_desc *desc)
 144 {
 145         struct dw_edma_chunk *child, *_next;
 146
 147         if (!desc->chunk)
 148                 return;
 149
 150         /* Remove all the list elements */
 151         list_for_each_entry_safe(child, _next, &desc->chunk->list, list) {
 152                 dw_edma_free_burst(child);
 153                 list_del(&child->list);
 154                 kfree(child);
 155                 desc->chunks_alloc--;
 156         }
 157
 158         /* Remove the list head */
 159         kfree(child);
 160         desc->chunk = NULL;
 161 }
 162
 163 static void dw_edma_free_desc(struct dw_edma_desc *desc)
 164 {
 165         dw_edma_free_chunk(desc);
 166         kfree(desc);
 167 }
 168
 169 static void vchan_free_desc(struct virt_dma_desc *vdesc)
 170 {
 171         dw_edma_free_desc(vd2dw_edma_desc(vdesc));
 172 }
 173
 174 static void dw_edma_start_transfer(struct dw_edma_chan *chan)
 175 {
 176         struct dw_edma_chunk *child;
 177         struct dw_edma_desc *desc;
 178         struct virt_dma_desc *vd;
 179
 180         vd = vchan_next_desc(&chan->vc);
 181         if (!vd)
 182                 return;
 183
 184         desc = vd2dw_edma_desc(vd);
 185         if (!desc)
 186                 return;
 187
 188         child = list_first_entry_or_null(&desc->chunk->list,
 189                                          struct dw_edma_chunk, list);
 190         if (!child)
 191                 return;
 192
 193         dw_edma_v0_core_start(child, !desc->xfer_sz);
 194         desc->xfer_sz += child->ll_region.sz;
 195         dw_edma_free_burst(child);
 196         list_del(&child->list);
 197         kfree(child);
 198         desc->chunks_alloc--;
 199 }
 200
 201 static int dw_edma_device_config(struct dma_chan *dchan,
 202                                  struct dma_slave_config *config)
 203 {
 204         struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
 205
 206         memcpy(&chan->config, config, sizeof(*config));
 207         chan->configured = true;
 208
 209         return 0;
 210 }
 211
 212 static int dw_edma_device_pause(struct dma_chan *dchan)
 213 {
 214         struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
 215         int err = 0;
 216
 217         if (!chan->configured)
 218                 err = -EPERM;
 219         else if (chan->status != EDMA_ST_BUSY)
 220                 err = -EPERM;
 221         else if (chan->request != EDMA_REQ_NONE)
 222                 err = -EPERM;
 223         else
 224                 chan->request = EDMA_REQ_PAUSE;
 225
 226         return err;
 227 }
 228
 229 static int dw_edma_device_resume(struct dma_chan *dchan)
 230 {
 231         struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
 232         int err = 0;
 233
 234         if (!chan->configured) {
 235                 err = -EPERM;
 236         } else if (chan->status != EDMA_ST_PAUSE) {
 237                 err = -EPERM;
 238         } else if (chan->request != EDMA_REQ_NONE) {
 239                 err = -EPERM;
 240         } else {
 241                 chan->status = EDMA_ST_BUSY;
 242                 dw_edma_start_transfer(chan);
 243         }
 244
 245         return err;
 246 }
 247
 248 static int dw_edma_device_terminate_all(struct dma_chan *dchan)
 249 {
 250         struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
 251         int err = 0;
 252         LIST_HEAD(head);
 253
 254         if (!chan->configured) {
 255                 /* Do nothing */
 256         } else if (chan->status == EDMA_ST_PAUSE) {
 257                 chan->status = EDMA_ST_IDLE;
 258                 chan->configured = false;
 259         } else if (chan->status == EDMA_ST_IDLE) {
 260                 chan->configured = false;
 261         } else if (dw_edma_v0_core_ch_status(chan) == DMA_COMPLETE) {
 262                 /*
 263                  * The channel is in a false BUSY state, probably didn't
 264                  * receive or lost an interrupt
 265                  */
 266                 chan->status = EDMA_ST_IDLE;
 267                 chan->configured = false;
 268         } else if (chan->request > EDMA_REQ_PAUSE) {
 269                 err = -EPERM;
 270         } else {
 271                 chan->request = EDMA_REQ_STOP;
 272         }
 273
 274         return err;
 275 }
 276
 277 static void dw_edma_device_issue_pending(struct dma_chan *dchan)
 278 {
 279         struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
 280         unsigned long flags;
 281
 282         spin_lock_irqsave(&chan->vc.lock, flags);
 283         if (chan->configured && chan->request == EDMA_REQ_NONE &&
 284             chan->status == EDMA_ST_IDLE && vchan_issue_pending(&chan->vc)) {
 285                 chan->status = EDMA_ST_BUSY;
 286                 dw_edma_start_transfer(chan);
 287         }
 288         spin_unlock_irqrestore(&chan->vc.lock, flags);
 289 }
 290
 291 static enum dma_status
 292 dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
 293                          struct dma_tx_state *txstate)
 294 {
 295         struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
 296         struct dw_edma_desc *desc;
 297         struct virt_dma_desc *vd;
 298         unsigned long flags;
 299         enum dma_status ret;
 300         u32 residue = 0;
 301
 302         ret = dma_cookie_status(dchan, cookie, txstate);
 303         if (ret == DMA_COMPLETE)
 304                 return ret;
 305
 306         if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE)
 307                 ret = DMA_PAUSED;
 308
 309         if (!txstate)
 310                 goto ret_residue;
 311
 312         spin_lock_irqsave(&chan->vc.lock, flags);
 313         vd = vchan_find_desc(&chan->vc, cookie);
 314         if (vd) {
 315                 desc = vd2dw_edma_desc(vd);
 316                 if (desc)
 317                         residue = desc->alloc_sz - desc->xfer_sz;
 318         }
 319         spin_unlock_irqrestore(&chan->vc.lock, flags);
 320
 321 ret_residue:
 322         dma_set_residue(txstate, residue);
 323
 324         return ret;
 325 }
 326
 327 static struct dma_async_tx_descriptor *
 328 dw_edma_device_transfer(struct dw_edma_transfer *xfer)
 329 {
 330         struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan);
 331         enum dma_transfer_direction dir = xfer->direction;
 332         phys_addr_t src_addr, dst_addr;
 333         struct scatterlist *sg = NULL;
 334         struct dw_edma_chunk *chunk;
 335         struct dw_edma_burst *burst;
 336         struct dw_edma_desc *desc;
 337         u32 cnt = 0;
 338         int i;
 339
 340         if (!chan->configured)
 341                 return NULL;
 342
 343         switch (chan->config.direction) {
 344         case DMA_DEV_TO_MEM: /* local DMA */
 345                 if (dir == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_READ)
 346                         break;
 347                 return NULL;
 348         case DMA_MEM_TO_DEV: /* local DMA */
 349                 if (dir == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_WRITE)
 350                         break;
 351                 return NULL;
 352         default: /* remote DMA */
 353                 if (dir == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_READ)
 354                         break;
 355                 if (dir == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_WRITE)
 356                         break;
 357                 return NULL;
 358         }
 359
 360         if (xfer->type == EDMA_XFER_CYCLIC) {
 361                 if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt)
 362                         return NULL;
 363         } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
 364                 if (xfer->xfer.sg.len < 1)
 365                         return NULL;
 366         } else if (xfer->type == EDMA_XFER_INTERLEAVED) {
 367                 if (!xfer->xfer.il->numf)
 368                         return NULL;
 369                 if (xfer->xfer.il->numf > 0 && xfer->xfer.il->frame_size > 0)
 370                         return NULL;
 371         } else {
 372                 return NULL;
 373         }
 374
 375         desc = dw_edma_alloc_desc(chan);
 376         if (unlikely(!desc))
 377                 goto err_alloc;
 378
 379         chunk = dw_edma_alloc_chunk(desc);
 380         if (unlikely(!chunk))
 381                 goto err_alloc;
 382
 383         if (xfer->type == EDMA_XFER_INTERLEAVED) {
 384                 src_addr = xfer->xfer.il->src_start;
 385                 dst_addr = xfer->xfer.il->dst_start;
 386         } else {
 387                 src_addr = chan->config.src_addr;
 388                 dst_addr = chan->config.dst_addr;
 389         }
 390
 391         if (xfer->type == EDMA_XFER_CYCLIC) {
 392                 cnt = xfer->xfer.cyclic.cnt;
 393         } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
 394                 cnt = xfer->xfer.sg.len;
 395                 sg = xfer->xfer.sg.sgl;
 396         } else if (xfer->type == EDMA_XFER_INTERLEAVED) {
 397                 if (xfer->xfer.il->numf > 0)
 398                         cnt = xfer->xfer.il->numf;
 399                 else
 400                         cnt = xfer->xfer.il->frame_size;
 401         }
 402
 403         for (i = 0; i < cnt; i++) {
 404                 if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg)
 405                         break;
 406
 407                 if (chunk->bursts_alloc == chan->ll_max) {
 408                         chunk = dw_edma_alloc_chunk(desc);
 409                         if (unlikely(!chunk))
 410                                 goto err_alloc;
 411                 }
 412
 413                 burst = dw_edma_alloc_burst(chunk);
 414                 if (unlikely(!burst))
 415                         goto err_alloc;
 416
 417                 if (xfer->type == EDMA_XFER_CYCLIC)
 418                         burst->sz = xfer->xfer.cyclic.len;
 419                 else if (xfer->type == EDMA_XFER_SCATTER_GATHER)
 420                         burst->sz = sg_dma_len(sg);
 421                 else if (xfer->type == EDMA_XFER_INTERLEAVED)
 422                         burst->sz = xfer->xfer.il->sgl[i].size;
 423
 424                 chunk->ll_region.sz += burst->sz;
 425                 desc->alloc_sz += burst->sz;
 426
 427                 if (chan->dir == EDMA_DIR_WRITE) {
 428                         burst->sar = src_addr;
 429                         if (xfer->type == EDMA_XFER_CYCLIC) {
 430                                 burst->dar = xfer->xfer.cyclic.paddr;
 431                         } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
 432                                 src_addr += sg_dma_len(sg);
 433                                 burst->dar = sg_dma_address(sg);
 434                                 /* Unlike the typical assumption by other
 435                                  * drivers/IPs the peripheral memory isn't
 436                                  * a FIFO memory, in this case, it's a
 437                                  * linear memory and that why the source
 438                                  * and destination addresses are increased
 439                                  * by the same portion (data length)
 440                                  */
 441                         }
 442                 } else {
 443                         burst->dar = dst_addr;
 444                         if (xfer->type == EDMA_XFER_CYCLIC) {
 445                                 burst->sar = xfer->xfer.cyclic.paddr;
 446                         } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
 447                                 dst_addr += sg_dma_len(sg);
 448                                 burst->sar = sg_dma_address(sg);
 449                                 /* Unlike the typical assumption by other
 450                                  * drivers/IPs the peripheral memory isn't
 451                                  * a FIFO memory, in this case, it's a
 452                                  * linear memory and that why the source
 453                                  * and destination addresses are increased
 454                                  * by the same portion (data length)
 455                                  */
 456                         }
 457                 }
 458
 459                 if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
 460                         sg = sg_next(sg);
 461                 } else if (xfer->type == EDMA_XFER_INTERLEAVED &&
 462                            xfer->xfer.il->frame_size > 0) {
 463                         struct dma_interleaved_template *il = xfer->xfer.il;
 464                         struct data_chunk *dc = &il->sgl[i];
 465
 466                         if (il->src_sgl) {
 467                                 src_addr += burst->sz;
 468                                 src_addr += dmaengine_get_src_icg(il, dc);
 469                         }
 470
 471                         if (il->dst_sgl) {
 472                                 dst_addr += burst->sz;
 473                                 dst_addr += dmaengine_get_dst_icg(il, dc);
 474                         }
 475                 }
 476         }
 477
 478         return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags);
 479
 480 err_alloc:
 481         if (desc)
 482                 dw_edma_free_desc(desc);
 483
 484         return NULL;
 485 }
 486
 487 static struct dma_async_tx_descriptor *
 488 dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
 489                              unsigned int len,
 490                              enum dma_transfer_direction direction,
 491                              unsigned long flags, void *context)
 492 {
 493         struct dw_edma_transfer xfer;
 494
 495         xfer.dchan = dchan;
 496         xfer.direction = direction;
 497         xfer.xfer.sg.sgl = sgl;
 498         xfer.xfer.sg.len = len;
 499         xfer.flags = flags;
 500         xfer.type = EDMA_XFER_SCATTER_GATHER;
 501
 502         return dw_edma_device_transfer(&xfer);
 503 }
 504
 505 static struct dma_async_tx_descriptor *
 506 dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr,
 507                                size_t len, size_t count,
 508                                enum dma_transfer_direction direction,
 509                                unsigned long flags)
 510 {
 511         struct dw_edma_transfer xfer;
 512
 513         xfer.dchan = dchan;
 514         xfer.direction = direction;
 515         xfer.xfer.cyclic.paddr = paddr;
 516         xfer.xfer.cyclic.len = len;
 517         xfer.xfer.cyclic.cnt = count;
 518         xfer.flags = flags;
 519         xfer.type = EDMA_XFER_CYCLIC;
 520
 521         return dw_edma_device_transfer(&xfer);
 522 }
 523
 524 static struct dma_async_tx_descriptor *
 525 dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan,
 526                                     struct dma_interleaved_template *ilt,
 527                                     unsigned long flags)
 528 {
 529         struct dw_edma_transfer xfer;
 530
 531         xfer.dchan = dchan;
 532         xfer.direction = ilt->dir;
 533         xfer.xfer.il = ilt;
 534         xfer.flags = flags;
 535         xfer.type = EDMA_XFER_INTERLEAVED;
 536
 537         return dw_edma_device_transfer(&xfer);
 538 }
 539
 540 static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
 541 {
 542         struct dw_edma_desc *desc;
 543         struct virt_dma_desc *vd;
 544         unsigned long flags;
 545
 546         dw_edma_v0_core_clear_done_int(chan);
 547
 548         spin_lock_irqsave(&chan->vc.lock, flags);
 549         vd = vchan_next_desc(&chan->vc);
 550         if (vd) {
 551                 switch (chan->request) {
 552                 case EDMA_REQ_NONE:
 553                         desc = vd2dw_edma_desc(vd);
 554                         if (desc->chunks_alloc) {
 555                                 chan->status = EDMA_ST_BUSY;
 556                                 dw_edma_start_transfer(chan);
 557                         } else {
 558                                 list_del(&vd->node);
 559                                 vchan_cookie_complete(vd);
 560                                 chan->status = EDMA_ST_IDLE;
 561                         }
 562                         break;
 563
 564                 case EDMA_REQ_STOP:
 565                         list_del(&vd->node);
 566                         vchan_cookie_complete(vd);
 567                         chan->request = EDMA_REQ_NONE;
 568                         chan->status = EDMA_ST_IDLE;
 569                         break;
 570
 571                 case EDMA_REQ_PAUSE:
 572                         chan->request = EDMA_REQ_NONE;
 573                         chan->status = EDMA_ST_PAUSE;
 574                         break;
 575
 576                 default:
 577                         break;
 578                 }
 579         }
 580         spin_unlock_irqrestore(&chan->vc.lock, flags);
 581 }
 582
 583 static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
 584 {
 585         struct virt_dma_desc *vd;
 586         unsigned long flags;
 587
 588         dw_edma_v0_core_clear_abort_int(chan);
 589
 590         spin_lock_irqsave(&chan->vc.lock, flags);
 591         vd = vchan_next_desc(&chan->vc);
 592         if (vd) {
 593                 list_del(&vd->node);
 594                 vchan_cookie_complete(vd);
 595         }
 596         spin_unlock_irqrestore(&chan->vc.lock, flags);
 597         chan->request = EDMA_REQ_NONE;
 598         chan->status = EDMA_ST_IDLE;
 599 }
 600
 601 static irqreturn_t dw_edma_interrupt(int irq, void *data, bool write)
 602 {
 603         struct dw_edma_irq *dw_irq = data;
 604         struct dw_edma *dw = dw_irq->dw;
 605         unsigned long total, pos, val;
 606         unsigned long off;
 607         u32 mask;
 608
 609         if (write) {
 610                 total = dw->wr_ch_cnt;
 611                 off = 0;
 612                 mask = dw_irq->wr_mask;
 613         } else {
 614                 total = dw->rd_ch_cnt;
 615                 off = dw->wr_ch_cnt;
 616                 mask = dw_irq->rd_mask;
 617         }
 618
 619         val = dw_edma_v0_core_status_done_int(dw, write ?
 620                                                           EDMA_DIR_WRITE :
 621                                                           EDMA_DIR_READ);
 622         val &= mask;
 623         for_each_set_bit(pos, &val, total) {
 624                 struct dw_edma_chan *chan = &dw->chan[pos + off];
 625
 626                 dw_edma_done_interrupt(chan);
 627         }
 628
 629         val = dw_edma_v0_core_status_abort_int(dw, write ?
 630                                                            EDMA_DIR_WRITE :
 631                                                            EDMA_DIR_READ);
 632         val &= mask;
 633         for_each_set_bit(pos, &val, total) {
 634                 struct dw_edma_chan *chan = &dw->chan[pos + off];
 635
 636                 dw_edma_abort_interrupt(chan);
 637         }
 638
 639         return IRQ_HANDLED;
 640 }
 641
 642 static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data)
 643 {
 644         return dw_edma_interrupt(irq, data, true);
 645 }
 646
 647 static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data)
 648 {
 649         return dw_edma_interrupt(irq, data, false);
 650 }
 651
 652 static irqreturn_t dw_edma_interrupt_common(int irq, void *data)
 653 {
 654         dw_edma_interrupt(irq, data, true);
 655         dw_edma_interrupt(irq, data, false);
 656
 657         return IRQ_HANDLED;
 658 }
 659
 660 static int dw_edma_alloc_chan_resources(struct dma_chan *dchan)
 661 {
 662         struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
 663
 664         if (chan->status != EDMA_ST_IDLE)
 665                 return -EBUSY;
 666
 667         pm_runtime_get(chan->chip->dev);
 668
 669         return 0;
 670 }
 671
 672 static void dw_edma_free_chan_resources(struct dma_chan *dchan)
 673 {
 674         unsigned long timeout = jiffies + msecs_to_jiffies(5000);
 675         struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
 676         int ret;
 677
 678         while (time_before(jiffies, timeout)) {
 679                 ret = dw_edma_device_terminate_all(dchan);
 680                 if (!ret)
 681                         break;
 682
 683                 if (time_after_eq(jiffies, timeout))
 684                         return;
 685
 686                 cpu_relax();
 687         }
 688
 689         pm_runtime_put(chan->chip->dev);
 690 }
 691
 692 static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write,
 693                                  u32 wr_alloc, u32 rd_alloc)
 694 {
 695         struct dw_edma_region *dt_region;
 696         struct device *dev = chip->dev;
 697         struct dw_edma *dw = chip->dw;
 698         struct dw_edma_chan *chan;
 699         struct dw_edma_irq *irq;
 700         struct dma_device *dma;
 701         u32 alloc, off_alloc;
 702         u32 i, j, cnt;
 703         int err = 0;
 704         u32 pos;
 705
 706         if (write) {
 707                 i = 0;
 708                 cnt = dw->wr_ch_cnt;
 709                 dma = &dw->wr_edma;
 710                 alloc = wr_alloc;
 711                 off_alloc = 0;
 712         } else {
 713                 i = dw->wr_ch_cnt;
 714                 cnt = dw->rd_ch_cnt;
 715                 dma = &dw->rd_edma;
 716                 alloc = rd_alloc;
 717                 off_alloc = wr_alloc;
 718         }
 719
 720         INIT_LIST_HEAD(&dma->channels);
 721         for (j = 0; (alloc || dw->nr_irqs == 1) && j < cnt; j++, i++) {
 722                 chan = &dw->chan[i];
 723
 724                 dt_region = devm_kzalloc(dev, sizeof(*dt_region), GFP_KERNEL);
 725                 if (!dt_region)
 726                         return -ENOMEM;
 727
 728                 chan->vc.chan.private = dt_region;
 729
 730                 chan->chip = chip;
 731                 chan->id = j;
 732                 chan->dir = write ? EDMA_DIR_WRITE : EDMA_DIR_READ;
 733                 chan->configured = false;
 734                 chan->request = EDMA_REQ_NONE;
 735                 chan->status = EDMA_ST_IDLE;
 736
 737                 if (write)
 738                         chan->ll_max = (dw->ll_region_wr[j].sz / EDMA_LL_SZ);
 739                 else
 740                         chan->ll_max = (dw->ll_region_rd[j].sz / EDMA_LL_SZ);
 741                 chan->ll_max -= 1;
 742
 743                 dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n",
 744                          write ? "write" : "read", j, chan->ll_max);
 745
 746                 if (dw->nr_irqs == 1)
 747                         pos = 0;
 748                 else
 749                         pos = off_alloc + (j % alloc);
 750
 751                 irq = &dw->irq[pos];
 752
 753                 if (write)
 754                         irq->wr_mask |= BIT(j);
 755                 else
 756                         irq->rd_mask |= BIT(j);
 757
 758                 irq->dw = dw;
 759                 memcpy(&chan->msi, &irq->msi, sizeof(chan->msi));
 760
 761                 dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n",
 762                          write ? "write" : "read", j,
 763                          chan->msi.address_hi, chan->msi.address_lo,
 764                          chan->msi.data);
 765
 766                 chan->vc.desc_free = vchan_free_desc;
 767                 vchan_init(&chan->vc, dma);
 768
 769                 if (write) {
 770                         dt_region->paddr = dw->dt_region_wr[j].paddr;
 771                         dt_region->vaddr = dw->dt_region_wr[j].vaddr;
 772                         dt_region->sz = dw->dt_region_wr[j].sz;
 773                 } else {
 774                         dt_region->paddr = dw->dt_region_rd[j].paddr;
 775                         dt_region->vaddr = dw->dt_region_rd[j].vaddr;
 776                         dt_region->sz = dw->dt_region_rd[j].sz;
 777                 }
 778
 779                 dw_edma_v0_core_device_config(chan);
 780         }
 781
 782         /* Set DMA channel capabilities */
 783         dma_cap_zero(dma->cap_mask);
 784         dma_cap_set(DMA_SLAVE, dma->cap_mask);
 785         dma_cap_set(DMA_CYCLIC, dma->cap_mask);
 786         dma_cap_set(DMA_PRIVATE, dma->cap_mask);
 787         dma_cap_set(DMA_INTERLEAVE, dma->cap_mask);
 788         dma->directions = BIT(write ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV);
 789         dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
 790         dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
 791         dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
 792         dma->chancnt = cnt;
 793
 794         /* Set DMA channel callbacks */
 795         dma->dev = chip->dev;
 796         dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources;
 797         dma->device_free_chan_resources = dw_edma_free_chan_resources;
 798         dma->device_config = dw_edma_device_config;
 799         dma->device_pause = dw_edma_device_pause;
 800         dma->device_resume = dw_edma_device_resume;
 801         dma->device_terminate_all = dw_edma_device_terminate_all;
 802         dma->device_issue_pending = dw_edma_device_issue_pending;
 803         dma->device_tx_status = dw_edma_device_tx_status;
 804         dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg;
 805         dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic;
 806         dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma;
 807
 808         dma_set_max_seg_size(dma->dev, U32_MAX);
 809
 810         /* Register DMA device */
 811         err = dma_async_device_register(dma);
 812
 813         return err;
 814 }
 815
 816 static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt)
 817 {
 818         if (*nr_irqs && *alloc < cnt) {
 819                 (*alloc)++;
 820                 (*nr_irqs)--;
 821         }
 822 }
 823
 824 static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt)
 825 {
 826         while (*mask * alloc < cnt)
 827                 (*mask)++;
 828 }
 829
 830 static int dw_edma_irq_request(struct dw_edma_chip *chip,
 831                                u32 *wr_alloc, u32 *rd_alloc)
 832 {
 833         struct device *dev = chip->dev;
 834         struct dw_edma *dw = chip->dw;
 835         u32 wr_mask = 1;
 836         u32 rd_mask = 1;
 837         int i, err = 0;
 838         u32 ch_cnt;
 839         int irq;
 840
 841         ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;
 842
 843         if (dw->nr_irqs < 1)
 844                 return -EINVAL;
 845
 846         if (dw->nr_irqs == 1) {
 847                 /* Common IRQ shared among all channels */
 848                 irq = dw->ops->irq_vector(dev, 0);
 849                 err = request_irq(irq, dw_edma_interrupt_common,
 850                                   IRQF_SHARED, dw->name, &dw->irq[0]);
 851                 if (err) {
 852                         dw->nr_irqs = 0;
 853                         return err;
 854                 }
 855
 856                 if (irq_get_msi_desc(irq))
 857                         get_cached_msi_msg(irq, &dw->irq[0].msi);
 858         } else {
 859                 /* Distribute IRQs equally among all channels */
 860                 int tmp = dw->nr_irqs;
 861
 862                 while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) {
 863                         dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt);
 864                         dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt);
 865                 }
 866
 867                 dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt);
 868                 dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt);
 869
 870                 for (i = 0; i < (*wr_alloc + *rd_alloc); i++) {
 871                         irq = dw->ops->irq_vector(dev, i);
 872                         err = request_irq(irq,
 873                                           i < *wr_alloc ?
 874                                                 dw_edma_interrupt_write :
 875                                                 dw_edma_interrupt_read,
 876                                           IRQF_SHARED, dw->name,
 877                                           &dw->irq[i]);
 878                         if (err) {
 879                                 dw->nr_irqs = i;
 880                                 return err;
 881                         }
 882
 883                         if (irq_get_msi_desc(irq))
 884                                 get_cached_msi_msg(irq, &dw->irq[i].msi);
 885                 }
 886
 887                 dw->nr_irqs = i;
 888         }
 889
 890         return err;
 891 }
 892
 893 int dw_edma_probe(struct dw_edma_chip *chip)
 894 {
 895         struct device *dev;
 896         struct dw_edma *dw;
 897         u32 wr_alloc = 0;
 898         u32 rd_alloc = 0;
 899         int i, err;
 900
 901         if (!chip)
 902                 return -EINVAL;
 903
 904         dev = chip->dev;
 905         if (!dev)
 906                 return -EINVAL;
 907
 908         dw = chip->dw;
 909         if (!dw || !dw->irq || !dw->ops || !dw->ops->irq_vector)
 910                 return -EINVAL;
 911
 912         raw_spin_lock_init(&dw->lock);
 913
 914         dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt,
 915                               dw_edma_v0_core_ch_count(dw, EDMA_DIR_WRITE));
 916         dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH);
 917
 918         dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt,
 919                               dw_edma_v0_core_ch_count(dw, EDMA_DIR_READ));
 920         dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH);
 921
 922         if (!dw->wr_ch_cnt && !dw->rd_ch_cnt)
 923                 return -EINVAL;
 924
 925         dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n",
 926                  dw->wr_ch_cnt, dw->rd_ch_cnt);
 927
 928         /* Allocate channels */
 929         dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt,
 930                                 sizeof(*dw->chan), GFP_KERNEL);
 931         if (!dw->chan)
 932                 return -ENOMEM;
 933
 934         snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%d", chip->id);
 935
 936         /* Disable eDMA, only to establish the ideal initial conditions */
 937         dw_edma_v0_core_off(dw);
 938
 939         /* Request IRQs */
 940         err = dw_edma_irq_request(chip, &wr_alloc, &rd_alloc);
 941         if (err)
 942                 return err;
 943
 944         /* Setup write channels */
 945         err = dw_edma_channel_setup(chip, true, wr_alloc, rd_alloc);
 946         if (err)
 947                 goto err_irq_free;
 948
 949         /* Setup read channels */
 950         err = dw_edma_channel_setup(chip, false, wr_alloc, rd_alloc);
 951         if (err)
 952                 goto err_irq_free;
 953
 954         /* Power management */
 955         pm_runtime_enable(dev);
 956
 957         /* Turn debugfs on */
 958         dw_edma_v0_core_debugfs_on(chip);
 959
 960         return 0;
 961
 962 err_irq_free:
 963         for (i = (dw->nr_irqs - 1); i >= 0; i--)
 964                 free_irq(dw->ops->irq_vector(dev, i), &dw->irq[i]);
 965
 966         dw->nr_irqs = 0;
 967
 968         return err;
 969 }
 970 EXPORT_SYMBOL_GPL(dw_edma_probe);
 971
 972 int dw_edma_remove(struct dw_edma_chip *chip)
 973 {
 974         struct dw_edma_chan *chan, *_chan;
 975         struct device *dev = chip->dev;
 976         struct dw_edma *dw = chip->dw;
 977         int i;
 978
 979         /* Disable eDMA */
 980         dw_edma_v0_core_off(dw);
 981
 982         /* Free irqs */
 983         for (i = (dw->nr_irqs - 1); i >= 0; i--)
 984                 free_irq(dw->ops->irq_vector(dev, i), &dw->irq[i]);
 985
 986         /* Power management */
 987         pm_runtime_disable(dev);
 988
 989         /* Deregister eDMA device */
 990         dma_async_device_unregister(&dw->wr_edma);
 991         list_for_each_entry_safe(chan, _chan, &dw->wr_edma.channels,
 992                                  vc.chan.device_node) {
 993                 tasklet_kill(&chan->vc.task);
 994                 list_del(&chan->vc.chan.device_node);
 995         }
 996
 997         dma_async_device_unregister(&dw->rd_edma);
 998         list_for_each_entry_safe(chan, _chan, &dw->rd_edma.channels,
 999                                  vc.chan.device_node) {
1000                 tasklet_kill(&chan->vc.task);
1001                 list_del(&chan->vc.chan.device_node);
1002         }
1003
1004         /* Turn debugfs off */
1005         dw_edma_v0_core_debugfs_off(chip);
1006
1007         return 0;
1008 }
1009 EXPORT_SYMBOL_GPL(dw_edma_remove);
1010
1011 MODULE_LICENSE("GPL v2");
1012 MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver");
1013 MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>");