drivers/dma/ti/omap-dma.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * OMAP DMAengine support
   4  */
   5 #include <linux/cpu_pm.h>
   6 #include <linux/delay.h>
   7 #include <linux/dmaengine.h>
   8 #include <linux/dma-mapping.h>
   9 #include <linux/dmapool.h>
  10 #include <linux/err.h>
  11 #include <linux/init.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/list.h>
  14 #include <linux/module.h>
  15 #include <linux/omap-dma.h>
  16 #include <linux/platform_device.h>
  17 #include <linux/slab.h>
  18 #include <linux/spinlock.h>
  19 #include <linux/of_dma.h>
  20 #include <linux/of_device.h>
  21
  22 #include "../virt-dma.h"
  23
  24 #define OMAP_SDMA_REQUESTS      127
  25 #define OMAP_SDMA_CHANNELS      32
  26
  27 struct omap_dma_config {
  28         int lch_end;
  29         unsigned int rw_priority:1;
  30         unsigned int needs_busy_check:1;
  31         unsigned int may_lose_context:1;
  32         unsigned int needs_lch_clear:1;
  33 };
  34
  35 struct omap_dma_context {
  36         u32 irqenable_l0;
  37         u32 irqenable_l1;
  38         u32 ocp_sysconfig;
  39         u32 gcr;
  40 };
  41
  42 struct omap_dmadev {
  43         struct dma_device ddev;
  44         spinlock_t lock;
  45         void __iomem *base;
  46         const struct omap_dma_reg *reg_map;
  47         struct omap_system_dma_plat_info *plat;
  48         const struct omap_dma_config *cfg;
  49         struct notifier_block nb;
  50         struct omap_dma_context context;
  51         int lch_count;
  52         DECLARE_BITMAP(lch_bitmap, OMAP_SDMA_CHANNELS);
  53         struct mutex lch_lock;          /* for assigning logical channels */
  54         bool legacy;
  55         bool ll123_supported;
  56         struct dma_pool *desc_pool;
  57         unsigned dma_requests;
  58         spinlock_t irq_lock;
  59         uint32_t irq_enable_mask;
  60         struct omap_chan **lch_map;
  61 };
  62
  63 struct omap_chan {
  64         struct virt_dma_chan vc;
  65         void __iomem *channel_base;
  66         const struct omap_dma_reg *reg_map;
  67         uint32_t ccr;
  68
  69         struct dma_slave_config cfg;
  70         unsigned dma_sig;
  71         bool cyclic;
  72         bool paused;
  73         bool running;
  74
  75         int dma_ch;
  76         struct omap_desc *desc;
  77         unsigned sgidx;
  78 };
  79
  80 #define DESC_NXT_SV_REFRESH     (0x1 << 24)
  81 #define DESC_NXT_SV_REUSE       (0x2 << 24)
  82 #define DESC_NXT_DV_REFRESH     (0x1 << 26)
  83 #define DESC_NXT_DV_REUSE       (0x2 << 26)
  84 #define DESC_NTYPE_TYPE2        (0x2 << 29)
  85
  86 /* Type 2 descriptor with Source or Destination address update */
  87 struct omap_type2_desc {
  88         uint32_t next_desc;
  89         uint32_t en;
  90         uint32_t addr; /* src or dst */
  91         uint16_t fn;
  92         uint16_t cicr;
  93         int16_t cdei;
  94         int16_t csei;
  95         int32_t cdfi;
  96         int32_t csfi;
  97 } __packed;
  98
  99 struct omap_sg {
 100         dma_addr_t addr;
 101         uint32_t en;            /* number of elements (24-bit) */
 102         uint32_t fn;            /* number of frames (16-bit) */
 103         int32_t fi;             /* for double indexing */
 104         int16_t ei;             /* for double indexing */
 105
 106         /* Linked list */
 107         struct omap_type2_desc *t2_desc;
 108         dma_addr_t t2_desc_paddr;
 109 };
 110
 111 struct omap_desc {
 112         struct virt_dma_desc vd;
 113         bool using_ll;
 114         enum dma_transfer_direction dir;
 115         dma_addr_t dev_addr;
 116         bool polled;
 117
 118         int32_t fi;             /* for OMAP_DMA_SYNC_PACKET / double indexing */
 119         int16_t ei;             /* for double indexing */
 120         uint8_t es;             /* CSDP_DATA_TYPE_xxx */
 121         uint32_t ccr;           /* CCR value */
 122         uint16_t clnk_ctrl;     /* CLNK_CTRL value */
 123         uint16_t cicr;          /* CICR value */
 124         uint32_t csdp;          /* CSDP value */
 125
 126         unsigned sglen;
 127         struct omap_sg sg[];
 128 };
 129
 130 enum {
 131         CAPS_0_SUPPORT_LL123    = BIT(20),      /* Linked List type1/2/3 */
 132         CAPS_0_SUPPORT_LL4      = BIT(21),      /* Linked List type4 */
 133
 134         CCR_FS                  = BIT(5),
 135         CCR_READ_PRIORITY       = BIT(6),
 136         CCR_ENABLE              = BIT(7),
 137         CCR_AUTO_INIT           = BIT(8),       /* OMAP1 only */
 138         CCR_REPEAT              = BIT(9),       /* OMAP1 only */
 139         CCR_OMAP31_DISABLE      = BIT(10),      /* OMAP1 only */
 140         CCR_SUSPEND_SENSITIVE   = BIT(8),       /* OMAP2+ only */
 141         CCR_RD_ACTIVE           = BIT(9),       /* OMAP2+ only */
 142         CCR_WR_ACTIVE           = BIT(10),      /* OMAP2+ only */
 143         CCR_SRC_AMODE_CONSTANT  = 0 << 12,
 144         CCR_SRC_AMODE_POSTINC   = 1 << 12,
 145         CCR_SRC_AMODE_SGLIDX    = 2 << 12,
 146         CCR_SRC_AMODE_DBLIDX    = 3 << 12,
 147         CCR_DST_AMODE_CONSTANT  = 0 << 14,
 148         CCR_DST_AMODE_POSTINC   = 1 << 14,
 149         CCR_DST_AMODE_SGLIDX    = 2 << 14,
 150         CCR_DST_AMODE_DBLIDX    = 3 << 14,
 151         CCR_CONSTANT_FILL       = BIT(16),
 152         CCR_TRANSPARENT_COPY    = BIT(17),
 153         CCR_BS                  = BIT(18),
 154         CCR_SUPERVISOR          = BIT(22),
 155         CCR_PREFETCH            = BIT(23),
 156         CCR_TRIGGER_SRC         = BIT(24),
 157         CCR_BUFFERING_DISABLE   = BIT(25),
 158         CCR_WRITE_PRIORITY      = BIT(26),
 159         CCR_SYNC_ELEMENT        = 0,
 160         CCR_SYNC_FRAME          = CCR_FS,
 161         CCR_SYNC_BLOCK          = CCR_BS,
 162         CCR_SYNC_PACKET         = CCR_BS | CCR_FS,
 163
 164         CSDP_DATA_TYPE_8        = 0,
 165         CSDP_DATA_TYPE_16       = 1,
 166         CSDP_DATA_TYPE_32       = 2,
 167         CSDP_SRC_PORT_EMIFF     = 0 << 2, /* OMAP1 only */
 168         CSDP_SRC_PORT_EMIFS     = 1 << 2, /* OMAP1 only */
 169         CSDP_SRC_PORT_OCP_T1    = 2 << 2, /* OMAP1 only */
 170         CSDP_SRC_PORT_TIPB      = 3 << 2, /* OMAP1 only */
 171         CSDP_SRC_PORT_OCP_T2    = 4 << 2, /* OMAP1 only */
 172         CSDP_SRC_PORT_MPUI      = 5 << 2, /* OMAP1 only */
 173         CSDP_SRC_PACKED         = BIT(6),
 174         CSDP_SRC_BURST_1        = 0 << 7,
 175         CSDP_SRC_BURST_16       = 1 << 7,
 176         CSDP_SRC_BURST_32       = 2 << 7,
 177         CSDP_SRC_BURST_64       = 3 << 7,
 178         CSDP_DST_PORT_EMIFF     = 0 << 9, /* OMAP1 only */
 179         CSDP_DST_PORT_EMIFS     = 1 << 9, /* OMAP1 only */
 180         CSDP_DST_PORT_OCP_T1    = 2 << 9, /* OMAP1 only */
 181         CSDP_DST_PORT_TIPB      = 3 << 9, /* OMAP1 only */
 182         CSDP_DST_PORT_OCP_T2    = 4 << 9, /* OMAP1 only */
 183         CSDP_DST_PORT_MPUI      = 5 << 9, /* OMAP1 only */
 184         CSDP_DST_PACKED         = BIT(13),
 185         CSDP_DST_BURST_1        = 0 << 14,
 186         CSDP_DST_BURST_16       = 1 << 14,
 187         CSDP_DST_BURST_32       = 2 << 14,
 188         CSDP_DST_BURST_64       = 3 << 14,
 189         CSDP_WRITE_NON_POSTED   = 0 << 16,
 190         CSDP_WRITE_POSTED       = 1 << 16,
 191         CSDP_WRITE_LAST_NON_POSTED = 2 << 16,
 192
 193         CICR_TOUT_IE            = BIT(0),       /* OMAP1 only */
 194         CICR_DROP_IE            = BIT(1),
 195         CICR_HALF_IE            = BIT(2),
 196         CICR_FRAME_IE           = BIT(3),
 197         CICR_LAST_IE            = BIT(4),
 198         CICR_BLOCK_IE           = BIT(5),
 199         CICR_PKT_IE             = BIT(7),       /* OMAP2+ only */
 200         CICR_TRANS_ERR_IE       = BIT(8),       /* OMAP2+ only */
 201         CICR_SUPERVISOR_ERR_IE  = BIT(10),      /* OMAP2+ only */
 202         CICR_MISALIGNED_ERR_IE  = BIT(11),      /* OMAP2+ only */
 203         CICR_DRAIN_IE           = BIT(12),      /* OMAP2+ only */
 204         CICR_SUPER_BLOCK_IE     = BIT(14),      /* OMAP2+ only */
 205
 206         CLNK_CTRL_ENABLE_LNK    = BIT(15),
 207
 208         CDP_DST_VALID_INC       = 0 << 0,
 209         CDP_DST_VALID_RELOAD    = 1 << 0,
 210         CDP_DST_VALID_REUSE     = 2 << 0,
 211         CDP_SRC_VALID_INC       = 0 << 2,
 212         CDP_SRC_VALID_RELOAD    = 1 << 2,
 213         CDP_SRC_VALID_REUSE     = 2 << 2,
 214         CDP_NTYPE_TYPE1         = 1 << 4,
 215         CDP_NTYPE_TYPE2         = 2 << 4,
 216         CDP_NTYPE_TYPE3         = 3 << 4,
 217         CDP_TMODE_NORMAL        = 0 << 8,
 218         CDP_TMODE_LLIST         = 1 << 8,
 219         CDP_FAST                = BIT(10),
 220 };
 221
 222 static const unsigned es_bytes[] = {
 223         [CSDP_DATA_TYPE_8] = 1,
 224         [CSDP_DATA_TYPE_16] = 2,
 225         [CSDP_DATA_TYPE_32] = 4,
 226 };
 227
 228 static bool omap_dma_filter_fn(struct dma_chan *chan, void *param);
 229 static struct of_dma_filter_info omap_dma_info = {
 230         .filter_fn = omap_dma_filter_fn,
 231 };
 232
 233 static inline struct omap_dmadev *to_omap_dma_dev(struct dma_device *d)
 234 {
 235         return container_of(d, struct omap_dmadev, ddev);
 236 }
 237
 238 static inline struct omap_chan *to_omap_dma_chan(struct dma_chan *c)
 239 {
 240         return container_of(c, struct omap_chan, vc.chan);
 241 }
 242
 243 static inline struct omap_desc *to_omap_dma_desc(struct dma_async_tx_descriptor *t)
 244 {
 245         return container_of(t, struct omap_desc, vd.tx);
 246 }
 247
 248 static void omap_dma_desc_free(struct virt_dma_desc *vd)
 249 {
 250         struct omap_desc *d = to_omap_dma_desc(&vd->tx);
 251
 252         if (d->using_ll) {
 253                 struct omap_dmadev *od = to_omap_dma_dev(vd->tx.chan->device);
 254                 int i;
 255
 256                 for (i = 0; i < d->sglen; i++) {
 257                         if (d->sg[i].t2_desc)
 258                                 dma_pool_free(od->desc_pool, d->sg[i].t2_desc,
 259                                               d->sg[i].t2_desc_paddr);
 260                 }
 261         }
 262
 263         kfree(d);
 264 }
 265
 266 static void omap_dma_fill_type2_desc(struct omap_desc *d, int idx,
 267                                      enum dma_transfer_direction dir, bool last)
 268 {
 269         struct omap_sg *sg = &d->sg[idx];
 270         struct omap_type2_desc *t2_desc = sg->t2_desc;
 271
 272         if (idx)
 273                 d->sg[idx - 1].t2_desc->next_desc = sg->t2_desc_paddr;
 274         if (last)
 275                 t2_desc->next_desc = 0xfffffffc;
 276
 277         t2_desc->en = sg->en;
 278         t2_desc->addr = sg->addr;
 279         t2_desc->fn = sg->fn & 0xffff;
 280         t2_desc->cicr = d->cicr;
 281         if (!last)
 282                 t2_desc->cicr &= ~CICR_BLOCK_IE;
 283
 284         switch (dir) {
 285         case DMA_DEV_TO_MEM:
 286                 t2_desc->cdei = sg->ei;
 287                 t2_desc->csei = d->ei;
 288                 t2_desc->cdfi = sg->fi;
 289                 t2_desc->csfi = d->fi;
 290
 291                 t2_desc->en |= DESC_NXT_DV_REFRESH;
 292                 t2_desc->en |= DESC_NXT_SV_REUSE;
 293                 break;
 294         case DMA_MEM_TO_DEV:
 295                 t2_desc->cdei = d->ei;
 296                 t2_desc->csei = sg->ei;
 297                 t2_desc->cdfi = d->fi;
 298                 t2_desc->csfi = sg->fi;
 299
 300                 t2_desc->en |= DESC_NXT_SV_REFRESH;
 301                 t2_desc->en |= DESC_NXT_DV_REUSE;
 302                 break;
 303         default:
 304                 return;
 305         }
 306
 307         t2_desc->en |= DESC_NTYPE_TYPE2;
 308 }
 309
 310 static void omap_dma_write(uint32_t val, unsigned type, void __iomem *addr)
 311 {
 312         switch (type) {
 313         case OMAP_DMA_REG_16BIT:
 314                 writew_relaxed(val, addr);
 315                 break;
 316         case OMAP_DMA_REG_2X16BIT:
 317                 writew_relaxed(val, addr);
 318                 writew_relaxed(val >> 16, addr + 2);
 319                 break;
 320         case OMAP_DMA_REG_32BIT:
 321                 writel_relaxed(val, addr);
 322                 break;
 323         default:
 324                 WARN_ON(1);
 325         }
 326 }
 327
 328 static unsigned omap_dma_read(unsigned type, void __iomem *addr)
 329 {
 330         unsigned val;
 331
 332         switch (type) {
 333         case OMAP_DMA_REG_16BIT:
 334                 val = readw_relaxed(addr);
 335                 break;
 336         case OMAP_DMA_REG_2X16BIT:
 337                 val = readw_relaxed(addr);
 338                 val |= readw_relaxed(addr + 2) << 16;
 339                 break;
 340         case OMAP_DMA_REG_32BIT:
 341                 val = readl_relaxed(addr);
 342                 break;
 343         default:
 344                 WARN_ON(1);
 345                 val = 0;
 346         }
 347
 348         return val;
 349 }
 350
 351 static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val)
 352 {
 353         const struct omap_dma_reg *r = od->reg_map + reg;
 354
 355         WARN_ON(r->stride);
 356
 357         omap_dma_write(val, r->type, od->base + r->offset);
 358 }
 359
 360 static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg)
 361 {
 362         const struct omap_dma_reg *r = od->reg_map + reg;
 363
 364         WARN_ON(r->stride);
 365
 366         return omap_dma_read(r->type, od->base + r->offset);
 367 }
 368
 369 static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val)
 370 {
 371         const struct omap_dma_reg *r = c->reg_map + reg;
 372
 373         omap_dma_write(val, r->type, c->channel_base + r->offset);
 374 }
 375
 376 static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg)
 377 {
 378         const struct omap_dma_reg *r = c->reg_map + reg;
 379
 380         return omap_dma_read(r->type, c->channel_base + r->offset);
 381 }
 382
 383 static void omap_dma_clear_csr(struct omap_chan *c)
 384 {
 385         if (dma_omap1())
 386                 omap_dma_chan_read(c, CSR);
 387         else
 388                 omap_dma_chan_write(c, CSR, ~0);
 389 }
 390
 391 static unsigned omap_dma_get_csr(struct omap_chan *c)
 392 {
 393         unsigned val = omap_dma_chan_read(c, CSR);
 394
 395         if (!dma_omap1())
 396                 omap_dma_chan_write(c, CSR, val);
 397
 398         return val;
 399 }
 400
 401 static void omap_dma_clear_lch(struct omap_dmadev *od, int lch)
 402 {
 403         struct omap_chan *c;
 404         int i;
 405
 406         c = od->lch_map[lch];
 407         if (!c)
 408                 return;
 409
 410         for (i = CSDP; i <= od->cfg->lch_end; i++)
 411                 omap_dma_chan_write(c, i, 0);
 412 }
 413
 414 static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c,
 415         unsigned lch)
 416 {
 417         c->channel_base = od->base + od->plat->channel_stride * lch;
 418
 419         od->lch_map[lch] = c;
 420 }
 421
 422 static void omap_dma_start(struct omap_chan *c, struct omap_desc *d)
 423 {
 424         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 425         uint16_t cicr = d->cicr;
 426
 427         if (__dma_omap15xx(od->plat->dma_attr))
 428                 omap_dma_chan_write(c, CPC, 0);
 429         else
 430                 omap_dma_chan_write(c, CDAC, 0);
 431
 432         omap_dma_clear_csr(c);
 433
 434         if (d->using_ll) {
 435                 uint32_t cdp = CDP_TMODE_LLIST | CDP_NTYPE_TYPE2 | CDP_FAST;
 436
 437                 if (d->dir == DMA_DEV_TO_MEM)
 438                         cdp |= (CDP_DST_VALID_RELOAD | CDP_SRC_VALID_REUSE);
 439                 else
 440                         cdp |= (CDP_DST_VALID_REUSE | CDP_SRC_VALID_RELOAD);
 441                 omap_dma_chan_write(c, CDP, cdp);
 442
 443                 omap_dma_chan_write(c, CNDP, d->sg[0].t2_desc_paddr);
 444                 omap_dma_chan_write(c, CCDN, 0);
 445                 omap_dma_chan_write(c, CCFN, 0xffff);
 446                 omap_dma_chan_write(c, CCEN, 0xffffff);
 447
 448                 cicr &= ~CICR_BLOCK_IE;
 449         } else if (od->ll123_supported) {
 450                 omap_dma_chan_write(c, CDP, 0);
 451         }
 452
 453         /* Enable interrupts */
 454         omap_dma_chan_write(c, CICR, cicr);
 455
 456         /* Enable channel */
 457         omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE);
 458
 459         c->running = true;
 460 }
 461
 462 static void omap_dma_drain_chan(struct omap_chan *c)
 463 {
 464         int i;
 465         u32 val;
 466
 467         /* Wait for sDMA FIFO to drain */
 468         for (i = 0; ; i++) {
 469                 val = omap_dma_chan_read(c, CCR);
 470                 if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)))
 471                         break;
 472
 473                 if (i > 100)
 474                         break;
 475
 476                 udelay(5);
 477         }
 478
 479         if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))
 480                 dev_err(c->vc.chan.device->dev,
 481                         "DMA drain did not complete on lch %d\n",
 482                         c->dma_ch);
 483 }
 484
 485 static int omap_dma_stop(struct omap_chan *c)
 486 {
 487         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 488         uint32_t val;
 489
 490         /* disable irq */
 491         omap_dma_chan_write(c, CICR, 0);
 492
 493         omap_dma_clear_csr(c);
 494
 495         val = omap_dma_chan_read(c, CCR);
 496         if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) {
 497                 uint32_t sysconfig;
 498
 499                 sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG);
 500                 val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK;
 501                 val |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE);
 502                 omap_dma_glbl_write(od, OCP_SYSCONFIG, val);
 503
 504                 val = omap_dma_chan_read(c, CCR);
 505                 val &= ~CCR_ENABLE;
 506                 omap_dma_chan_write(c, CCR, val);
 507
 508                 if (!(c->ccr & CCR_BUFFERING_DISABLE))
 509                         omap_dma_drain_chan(c);
 510
 511                 omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig);
 512         } else {
 513                 if (!(val & CCR_ENABLE))
 514                         return -EINVAL;
 515
 516                 val &= ~CCR_ENABLE;
 517                 omap_dma_chan_write(c, CCR, val);
 518
 519                 if (!(c->ccr & CCR_BUFFERING_DISABLE))
 520                         omap_dma_drain_chan(c);
 521         }
 522
 523         mb();
 524
 525         if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) {
 526                 val = omap_dma_chan_read(c, CLNK_CTRL);
 527
 528                 if (dma_omap1())
 529                         val |= 1 << 14; /* set the STOP_LNK bit */
 530                 else
 531                         val &= ~CLNK_CTRL_ENABLE_LNK;
 532
 533                 omap_dma_chan_write(c, CLNK_CTRL, val);
 534         }
 535         c->running = false;
 536         return 0;
 537 }
 538
 539 static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d)
 540 {
 541         struct omap_sg *sg = d->sg + c->sgidx;
 542         unsigned cxsa, cxei, cxfi;
 543
 544         if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
 545                 cxsa = CDSA;
 546                 cxei = CDEI;
 547                 cxfi = CDFI;
 548         } else {
 549                 cxsa = CSSA;
 550                 cxei = CSEI;
 551                 cxfi = CSFI;
 552         }
 553
 554         omap_dma_chan_write(c, cxsa, sg->addr);
 555         omap_dma_chan_write(c, cxei, sg->ei);
 556         omap_dma_chan_write(c, cxfi, sg->fi);
 557         omap_dma_chan_write(c, CEN, sg->en);
 558         omap_dma_chan_write(c, CFN, sg->fn);
 559
 560         omap_dma_start(c, d);
 561         c->sgidx++;
 562 }
 563
 564 static void omap_dma_start_desc(struct omap_chan *c)
 565 {
 566         struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
 567         struct omap_desc *d;
 568         unsigned cxsa, cxei, cxfi;
 569
 570         if (!vd) {
 571                 c->desc = NULL;
 572                 return;
 573         }
 574
 575         list_del(&vd->node);
 576
 577         c->desc = d = to_omap_dma_desc(&vd->tx);
 578         c->sgidx = 0;
 579
 580         /*
 581          * This provides the necessary barrier to ensure data held in
 582          * DMA coherent memory is visible to the DMA engine prior to
 583          * the transfer starting.
 584          */
 585         mb();
 586
 587         omap_dma_chan_write(c, CCR, d->ccr);
 588         if (dma_omap1())
 589                 omap_dma_chan_write(c, CCR2, d->ccr >> 16);
 590
 591         if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
 592                 cxsa = CSSA;
 593                 cxei = CSEI;
 594                 cxfi = CSFI;
 595         } else {
 596                 cxsa = CDSA;
 597                 cxei = CDEI;
 598                 cxfi = CDFI;
 599         }
 600
 601         omap_dma_chan_write(c, cxsa, d->dev_addr);
 602         omap_dma_chan_write(c, cxei, d->ei);
 603         omap_dma_chan_write(c, cxfi, d->fi);
 604         omap_dma_chan_write(c, CSDP, d->csdp);
 605         omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl);
 606
 607         omap_dma_start_sg(c, d);
 608 }
 609
 610 static void omap_dma_callback(int ch, u16 status, void *data)
 611 {
 612         struct omap_chan *c = data;
 613         struct omap_desc *d;
 614         unsigned long flags;
 615
 616         spin_lock_irqsave(&c->vc.lock, flags);
 617         d = c->desc;
 618         if (d) {
 619                 if (c->cyclic) {
 620                         vchan_cyclic_callback(&d->vd);
 621                 } else if (d->using_ll || c->sgidx == d->sglen) {
 622                         omap_dma_start_desc(c);
 623                         vchan_cookie_complete(&d->vd);
 624                 } else {
 625                         omap_dma_start_sg(c, d);
 626                 }
 627         }
 628         spin_unlock_irqrestore(&c->vc.lock, flags);
 629 }
 630
 631 static irqreturn_t omap_dma_irq(int irq, void *devid)
 632 {
 633         struct omap_dmadev *od = devid;
 634         unsigned status, channel;
 635
 636         spin_lock(&od->irq_lock);
 637
 638         status = omap_dma_glbl_read(od, IRQSTATUS_L1);
 639         status &= od->irq_enable_mask;
 640         if (status == 0) {
 641                 spin_unlock(&od->irq_lock);
 642                 return IRQ_NONE;
 643         }
 644
 645         while ((channel = ffs(status)) != 0) {
 646                 unsigned mask, csr;
 647                 struct omap_chan *c;
 648
 649                 channel -= 1;
 650                 mask = BIT(channel);
 651                 status &= ~mask;
 652
 653                 c = od->lch_map[channel];
 654                 if (c == NULL) {
 655                         /* This should never happen */
 656                         dev_err(od->ddev.dev, "invalid channel %u\n", channel);
 657                         continue;
 658                 }
 659
 660                 csr = omap_dma_get_csr(c);
 661                 omap_dma_glbl_write(od, IRQSTATUS_L1, mask);
 662
 663                 omap_dma_callback(channel, csr, c);
 664         }
 665
 666         spin_unlock(&od->irq_lock);
 667
 668         return IRQ_HANDLED;
 669 }
 670
 671 static int omap_dma_get_lch(struct omap_dmadev *od, int *lch)
 672 {
 673         int channel;
 674
 675         mutex_lock(&od->lch_lock);
 676         channel = find_first_zero_bit(od->lch_bitmap, od->lch_count);
 677         if (channel >= od->lch_count)
 678                 goto out_busy;
 679         set_bit(channel, od->lch_bitmap);
 680         mutex_unlock(&od->lch_lock);
 681
 682         omap_dma_clear_lch(od, channel);
 683         *lch = channel;
 684
 685         return 0;
 686
 687 out_busy:
 688         mutex_unlock(&od->lch_lock);
 689         *lch = -EINVAL;
 690
 691         return -EBUSY;
 692 }
 693
 694 static void omap_dma_put_lch(struct omap_dmadev *od, int lch)
 695 {
 696         omap_dma_clear_lch(od, lch);
 697         mutex_lock(&od->lch_lock);
 698         clear_bit(lch, od->lch_bitmap);
 699         mutex_unlock(&od->lch_lock);
 700 }
 701
 702 static int omap_dma_alloc_chan_resources(struct dma_chan *chan)
 703 {
 704         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 705         struct omap_chan *c = to_omap_dma_chan(chan);
 706         struct device *dev = od->ddev.dev;
 707         int ret;
 708
 709         if (od->legacy) {
 710                 ret = omap_request_dma(c->dma_sig, "DMA engine",
 711                                        omap_dma_callback, c, &c->dma_ch);
 712         } else {
 713                 ret = omap_dma_get_lch(od, &c->dma_ch);
 714         }
 715
 716         dev_dbg(dev, "allocating channel %u for %u\n", c->dma_ch, c->dma_sig);
 717
 718         if (ret >= 0) {
 719                 omap_dma_assign(od, c, c->dma_ch);
 720
 721                 if (!od->legacy) {
 722                         unsigned val;
 723
 724                         spin_lock_irq(&od->irq_lock);
 725                         val = BIT(c->dma_ch);
 726                         omap_dma_glbl_write(od, IRQSTATUS_L1, val);
 727                         od->irq_enable_mask |= val;
 728                         omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
 729
 730                         val = omap_dma_glbl_read(od, IRQENABLE_L0);
 731                         val &= ~BIT(c->dma_ch);
 732                         omap_dma_glbl_write(od, IRQENABLE_L0, val);
 733                         spin_unlock_irq(&od->irq_lock);
 734                 }
 735         }
 736
 737         if (dma_omap1()) {
 738                 if (__dma_omap16xx(od->plat->dma_attr)) {
 739                         c->ccr = CCR_OMAP31_DISABLE;
 740                         /* Duplicate what plat-omap/dma.c does */
 741                         c->ccr |= c->dma_ch + 1;
 742                 } else {
 743                         c->ccr = c->dma_sig & 0x1f;
 744                 }
 745         } else {
 746                 c->ccr = c->dma_sig & 0x1f;
 747                 c->ccr |= (c->dma_sig & ~0x1f) << 14;
 748         }
 749         if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING)
 750                 c->ccr |= CCR_BUFFERING_DISABLE;
 751
 752         return ret;
 753 }
 754
 755 static void omap_dma_free_chan_resources(struct dma_chan *chan)
 756 {
 757         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 758         struct omap_chan *c = to_omap_dma_chan(chan);
 759
 760         if (!od->legacy) {
 761                 spin_lock_irq(&od->irq_lock);
 762                 od->irq_enable_mask &= ~BIT(c->dma_ch);
 763                 omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
 764                 spin_unlock_irq(&od->irq_lock);
 765         }
 766
 767         c->channel_base = NULL;
 768         od->lch_map[c->dma_ch] = NULL;
 769         vchan_free_chan_resources(&c->vc);
 770
 771         if (od->legacy)
 772                 omap_free_dma(c->dma_ch);
 773         else
 774                 omap_dma_put_lch(od, c->dma_ch);
 775
 776         dev_dbg(od->ddev.dev, "freeing channel %u used for %u\n", c->dma_ch,
 777                 c->dma_sig);
 778         c->dma_sig = 0;
 779 }
 780
 781 static size_t omap_dma_sg_size(struct omap_sg *sg)
 782 {
 783         return sg->en * sg->fn;
 784 }
 785
 786 static size_t omap_dma_desc_size(struct omap_desc *d)
 787 {
 788         unsigned i;
 789         size_t size;
 790
 791         for (size = i = 0; i < d->sglen; i++)
 792                 size += omap_dma_sg_size(&d->sg[i]);
 793
 794         return size * es_bytes[d->es];
 795 }
 796
 797 static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr)
 798 {
 799         unsigned i;
 800         size_t size, es_size = es_bytes[d->es];
 801
 802         for (size = i = 0; i < d->sglen; i++) {
 803                 size_t this_size = omap_dma_sg_size(&d->sg[i]) * es_size;
 804
 805                 if (size)
 806                         size += this_size;
 807                 else if (addr >= d->sg[i].addr &&
 808                          addr < d->sg[i].addr + this_size)
 809                         size += d->sg[i].addr + this_size - addr;
 810         }
 811         return size;
 812 }
 813
 814 /*
 815  * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
 816  * read before the DMA controller finished disabling the channel.
 817  */
 818 static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg)
 819 {
 820         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 821         uint32_t val;
 822
 823         val = omap_dma_chan_read(c, reg);
 824         if (val == 0 && od->plat->errata & DMA_ERRATA_3_3)
 825                 val = omap_dma_chan_read(c, reg);
 826
 827         return val;
 828 }
 829
 830 static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c)
 831 {
 832         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 833         dma_addr_t addr, cdac;
 834
 835         if (__dma_omap15xx(od->plat->dma_attr)) {
 836                 addr = omap_dma_chan_read(c, CPC);
 837         } else {
 838                 addr = omap_dma_chan_read_3_3(c, CSAC);
 839                 cdac = omap_dma_chan_read_3_3(c, CDAC);
 840
 841                 /*
 842                  * CDAC == 0 indicates that the DMA transfer on the channel has
 843                  * not been started (no data has been transferred so far).
 844                  * Return the programmed source start address in this case.
 845                  */
 846                 if (cdac == 0)
 847                         addr = omap_dma_chan_read(c, CSSA);
 848         }
 849
 850         if (dma_omap1())
 851                 addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000;
 852
 853         return addr;
 854 }
 855
 856 static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c)
 857 {
 858         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 859         dma_addr_t addr;
 860
 861         if (__dma_omap15xx(od->plat->dma_attr)) {
 862                 addr = omap_dma_chan_read(c, CPC);
 863         } else {
 864                 addr = omap_dma_chan_read_3_3(c, CDAC);
 865
 866                 /*
 867                  * CDAC == 0 indicates that the DMA transfer on the channel
 868                  * has not been started (no data has been transferred so
 869                  * far).  Return the programmed destination start address in
 870                  * this case.
 871                  */
 872                 if (addr == 0)
 873                         addr = omap_dma_chan_read(c, CDSA);
 874         }
 875
 876         if (dma_omap1())
 877                 addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000;
 878
 879         return addr;
 880 }
 881
 882 static enum dma_status omap_dma_tx_status(struct dma_chan *chan,
 883         dma_cookie_t cookie, struct dma_tx_state *txstate)
 884 {
 885         struct omap_chan *c = to_omap_dma_chan(chan);
 886         enum dma_status ret;
 887         unsigned long flags;
 888         struct omap_desc *d = NULL;
 889
 890         ret = dma_cookie_status(chan, cookie, txstate);
 891         if (ret == DMA_COMPLETE)
 892                 return ret;
 893
 894         spin_lock_irqsave(&c->vc.lock, flags);
 895         if (c->desc && c->desc->vd.tx.cookie == cookie)
 896                 d = c->desc;
 897
 898         if (!txstate)
 899                 goto out;
 900
 901         if (d) {
 902                 dma_addr_t pos;
 903
 904                 if (d->dir == DMA_MEM_TO_DEV)
 905                         pos = omap_dma_get_src_pos(c);
 906                 else if (d->dir == DMA_DEV_TO_MEM  || d->dir == DMA_MEM_TO_MEM)
 907                         pos = omap_dma_get_dst_pos(c);
 908                 else
 909                         pos = 0;
 910
 911                 txstate->residue = omap_dma_desc_size_pos(d, pos);
 912         } else {
 913                 struct virt_dma_desc *vd = vchan_find_desc(&c->vc, cookie);
 914
 915                 if (vd)
 916                         txstate->residue = omap_dma_desc_size(
 917                                                 to_omap_dma_desc(&vd->tx));
 918                 else
 919                         txstate->residue = 0;
 920         }
 921
 922 out:
 923         if (ret == DMA_IN_PROGRESS && c->paused) {
 924                 ret = DMA_PAUSED;
 925         } else if (d && d->polled && c->running) {
 926                 uint32_t ccr = omap_dma_chan_read(c, CCR);
 927                 /*
 928                  * The channel is no longer active, set the return value
 929                  * accordingly and mark it as completed
 930                  */
 931                 if (!(ccr & CCR_ENABLE)) {
 932                         ret = DMA_COMPLETE;
 933                         omap_dma_start_desc(c);
 934                         vchan_cookie_complete(&d->vd);
 935                 }
 936         }
 937
 938         spin_unlock_irqrestore(&c->vc.lock, flags);
 939
 940         return ret;
 941 }
 942
 943 static void omap_dma_issue_pending(struct dma_chan *chan)
 944 {
 945         struct omap_chan *c = to_omap_dma_chan(chan);
 946         unsigned long flags;
 947
 948         spin_lock_irqsave(&c->vc.lock, flags);
 949         if (vchan_issue_pending(&c->vc) && !c->desc)
 950                 omap_dma_start_desc(c);
 951         spin_unlock_irqrestore(&c->vc.lock, flags);
 952 }
 953
 954 static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
 955         struct dma_chan *chan, struct scatterlist *sgl, unsigned sglen,
 956         enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
 957 {
 958         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 959         struct omap_chan *c = to_omap_dma_chan(chan);
 960         enum dma_slave_buswidth dev_width;
 961         struct scatterlist *sgent;
 962         struct omap_desc *d;
 963         dma_addr_t dev_addr;
 964         unsigned i, es, en, frame_bytes;
 965         bool ll_failed = false;
 966         u32 burst;
 967         u32 port_window, port_window_bytes;
 968
 969         if (dir == DMA_DEV_TO_MEM) {
 970                 dev_addr = c->cfg.src_addr;
 971                 dev_width = c->cfg.src_addr_width;
 972                 burst = c->cfg.src_maxburst;
 973                 port_window = c->cfg.src_port_window_size;
 974         } else if (dir == DMA_MEM_TO_DEV) {
 975                 dev_addr = c->cfg.dst_addr;
 976                 dev_width = c->cfg.dst_addr_width;
 977                 burst = c->cfg.dst_maxburst;
 978                 port_window = c->cfg.dst_port_window_size;
 979         } else {
 980                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
 981                 return NULL;
 982         }
 983
 984         /* Bus width translates to the element size (ES) */
 985         switch (dev_width) {
 986         case DMA_SLAVE_BUSWIDTH_1_BYTE:
 987                 es = CSDP_DATA_TYPE_8;
 988                 break;
 989         case DMA_SLAVE_BUSWIDTH_2_BYTES:
 990                 es = CSDP_DATA_TYPE_16;
 991                 break;
 992         case DMA_SLAVE_BUSWIDTH_4_BYTES:
 993                 es = CSDP_DATA_TYPE_32;
 994                 break;
 995         default: /* not reached */
 996                 return NULL;
 997         }
 998
 999         /* Now allocate and setup the descriptor. */
1000         d = kzalloc(struct_size(d, sg, sglen), GFP_ATOMIC);
1001         if (!d)
1002                 return NULL;
1003
1004         d->dir = dir;
1005         d->dev_addr = dev_addr;
1006         d->es = es;
1007
1008         /* When the port_window is used, one frame must cover the window */
1009         if (port_window) {
1010                 burst = port_window;
1011                 port_window_bytes = port_window * es_bytes[es];
1012
1013                 d->ei = 1;
1014                 /*
1015                  * One frame covers the port_window and by  configure
1016                  * the source frame index to be -1 * (port_window - 1)
1017                  * we instruct the sDMA that after a frame is processed
1018                  * it should move back to the start of the window.
1019                  */
1020                 d->fi = -(port_window_bytes - 1);
1021         }
1022
1023         d->ccr = c->ccr | CCR_SYNC_FRAME;
1024         if (dir == DMA_DEV_TO_MEM) {
1025                 d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED;
1026
1027                 d->ccr |= CCR_DST_AMODE_POSTINC;
1028                 if (port_window) {
1029                         d->ccr |= CCR_SRC_AMODE_DBLIDX;
1030
1031                         if (port_window_bytes >= 64)
1032                                 d->csdp |= CSDP_SRC_BURST_64;
1033                         else if (port_window_bytes >= 32)
1034                                 d->csdp |= CSDP_SRC_BURST_32;
1035                         else if (port_window_bytes >= 16)
1036                                 d->csdp |= CSDP_SRC_BURST_16;
1037
1038                 } else {
1039                         d->ccr |= CCR_SRC_AMODE_CONSTANT;
1040                 }
1041         } else {
1042                 d->csdp = CSDP_SRC_BURST_64 | CSDP_SRC_PACKED;
1043
1044                 d->ccr |= CCR_SRC_AMODE_POSTINC;
1045                 if (port_window) {
1046                         d->ccr |= CCR_DST_AMODE_DBLIDX;
1047
1048                         if (port_window_bytes >= 64)
1049                                 d->csdp |= CSDP_DST_BURST_64;
1050                         else if (port_window_bytes >= 32)
1051                                 d->csdp |= CSDP_DST_BURST_32;
1052                         else if (port_window_bytes >= 16)
1053                                 d->csdp |= CSDP_DST_BURST_16;
1054                 } else {
1055                         d->ccr |= CCR_DST_AMODE_CONSTANT;
1056                 }
1057         }
1058
1059         d->cicr = CICR_DROP_IE | CICR_BLOCK_IE;
1060         d->csdp |= es;
1061
1062         if (dma_omap1()) {
1063                 d->cicr |= CICR_TOUT_IE;
1064
1065                 if (dir == DMA_DEV_TO_MEM)
1066                         d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_TIPB;
1067                 else
1068                         d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF;
1069         } else {
1070                 if (dir == DMA_DEV_TO_MEM)
1071                         d->ccr |= CCR_TRIGGER_SRC;
1072
1073                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1074
1075                 if (port_window)
1076                         d->csdp |= CSDP_WRITE_LAST_NON_POSTED;
1077         }
1078         if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS)
1079                 d->clnk_ctrl = c->dma_ch;
1080
1081         /*
1082          * Build our scatterlist entries: each contains the address,
1083          * the number of elements (EN) in each frame, and the number of
1084          * frames (FN).  Number of bytes for this entry = ES * EN * FN.
1085          *
1086          * Burst size translates to number of elements with frame sync.
1087          * Note: DMA engine defines burst to be the number of dev-width
1088          * transfers.
1089          */
1090         en = burst;
1091         frame_bytes = es_bytes[es] * en;
1092
1093         if (sglen >= 2)
1094                 d->using_ll = od->ll123_supported;
1095
1096         for_each_sg(sgl, sgent, sglen, i) {
1097                 struct omap_sg *osg = &d->sg[i];
1098
1099                 osg->addr = sg_dma_address(sgent);
1100                 osg->en = en;
1101                 osg->fn = sg_dma_len(sgent) / frame_bytes;
1102
1103                 if (d->using_ll) {
1104                         osg->t2_desc = dma_pool_alloc(od->desc_pool, GFP_ATOMIC,
1105                                                       &osg->t2_desc_paddr);
1106                         if (!osg->t2_desc) {
1107                                 dev_err(chan->device->dev,
1108                                         "t2_desc[%d] allocation failed\n", i);
1109                                 ll_failed = true;
1110                                 d->using_ll = false;
1111                                 continue;
1112                         }
1113
1114                         omap_dma_fill_type2_desc(d, i, dir, (i == sglen - 1));
1115                 }
1116         }
1117
1118         d->sglen = sglen;
1119
1120         /* Release the dma_pool entries if one allocation failed */
1121         if (ll_failed) {
1122                 for (i = 0; i < d->sglen; i++) {
1123                         struct omap_sg *osg = &d->sg[i];
1124
1125                         if (osg->t2_desc) {
1126                                 dma_pool_free(od->desc_pool, osg->t2_desc,
1127                                               osg->t2_desc_paddr);
1128                                 osg->t2_desc = NULL;
1129                         }
1130                 }
1131         }
1132
1133         return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
1134 }
1135
1136 static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic(
1137         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
1138         size_t period_len, enum dma_transfer_direction dir, unsigned long flags)
1139 {
1140         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1141         struct omap_chan *c = to_omap_dma_chan(chan);
1142         enum dma_slave_buswidth dev_width;
1143         struct omap_desc *d;
1144         dma_addr_t dev_addr;
1145         unsigned es;
1146         u32 burst;
1147
1148         if (dir == DMA_DEV_TO_MEM) {
1149                 dev_addr = c->cfg.src_addr;
1150                 dev_width = c->cfg.src_addr_width;
1151                 burst = c->cfg.src_maxburst;
1152         } else if (dir == DMA_MEM_TO_DEV) {
1153                 dev_addr = c->cfg.dst_addr;
1154                 dev_width = c->cfg.dst_addr_width;
1155                 burst = c->cfg.dst_maxburst;
1156         } else {
1157                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
1158                 return NULL;
1159         }
1160
1161         /* Bus width translates to the element size (ES) */
1162         switch (dev_width) {
1163         case DMA_SLAVE_BUSWIDTH_1_BYTE:
1164                 es = CSDP_DATA_TYPE_8;
1165                 break;
1166         case DMA_SLAVE_BUSWIDTH_2_BYTES:
1167                 es = CSDP_DATA_TYPE_16;
1168                 break;
1169         case DMA_SLAVE_BUSWIDTH_4_BYTES:
1170                 es = CSDP_DATA_TYPE_32;
1171                 break;
1172         default: /* not reached */
1173                 return NULL;
1174         }
1175
1176         /* Now allocate and setup the descriptor. */
1177         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1178         if (!d)
1179                 return NULL;
1180
1181         d->dir = dir;
1182         d->dev_addr = dev_addr;
1183         d->fi = burst;
1184         d->es = es;
1185         d->sg[0].addr = buf_addr;
1186         d->sg[0].en = period_len / es_bytes[es];
1187         d->sg[0].fn = buf_len / period_len;
1188         d->sglen = 1;
1189
1190         d->ccr = c->ccr;
1191         if (dir == DMA_DEV_TO_MEM)
1192                 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT;
1193         else
1194                 d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC;
1195
1196         d->cicr = CICR_DROP_IE;
1197         if (flags & DMA_PREP_INTERRUPT)
1198                 d->cicr |= CICR_FRAME_IE;
1199
1200         d->csdp = es;
1201
1202         if (dma_omap1()) {
1203                 d->cicr |= CICR_TOUT_IE;
1204
1205                 if (dir == DMA_DEV_TO_MEM)
1206                         d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_MPUI;
1207                 else
1208                         d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF;
1209         } else {
1210                 if (burst)
1211                         d->ccr |= CCR_SYNC_PACKET;
1212                 else
1213                         d->ccr |= CCR_SYNC_ELEMENT;
1214
1215                 if (dir == DMA_DEV_TO_MEM) {
1216                         d->ccr |= CCR_TRIGGER_SRC;
1217                         d->csdp |= CSDP_DST_PACKED;
1218                 } else {
1219                         d->csdp |= CSDP_SRC_PACKED;
1220                 }
1221
1222                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1223
1224                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1225         }
1226
1227         if (__dma_omap15xx(od->plat->dma_attr))
1228                 d->ccr |= CCR_AUTO_INIT | CCR_REPEAT;
1229         else
1230                 d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK;
1231
1232         c->cyclic = true;
1233
1234         return vchan_tx_prep(&c->vc, &d->vd, flags);
1235 }
1236
1237 static struct dma_async_tx_descriptor *omap_dma_prep_dma_memcpy(
1238         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1239         size_t len, unsigned long tx_flags)
1240 {
1241         struct omap_chan *c = to_omap_dma_chan(chan);
1242         struct omap_desc *d;
1243         uint8_t data_type;
1244
1245         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1246         if (!d)
1247                 return NULL;
1248
1249         data_type = __ffs((src | dest | len));
1250         if (data_type > CSDP_DATA_TYPE_32)
1251                 data_type = CSDP_DATA_TYPE_32;
1252
1253         d->dir = DMA_MEM_TO_MEM;
1254         d->dev_addr = src;
1255         d->fi = 0;
1256         d->es = data_type;
1257         d->sg[0].en = len / BIT(data_type);
1258         d->sg[0].fn = 1;
1259         d->sg[0].addr = dest;
1260         d->sglen = 1;
1261         d->ccr = c->ccr;
1262         d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC;
1263
1264         if (tx_flags & DMA_PREP_INTERRUPT)
1265                 d->cicr |= CICR_FRAME_IE;
1266         else
1267                 d->polled = true;
1268
1269         d->csdp = data_type;
1270
1271         if (dma_omap1()) {
1272                 d->cicr |= CICR_TOUT_IE;
1273                 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1274         } else {
1275                 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1276                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1277                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1278         }
1279
1280         return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
1281 }
1282
1283 static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved(
1284         struct dma_chan *chan, struct dma_interleaved_template *xt,
1285         unsigned long flags)
1286 {
1287         struct omap_chan *c = to_omap_dma_chan(chan);
1288         struct omap_desc *d;
1289         struct omap_sg *sg;
1290         uint8_t data_type;
1291         size_t src_icg, dst_icg;
1292
1293         /* Slave mode is not supported */
1294         if (is_slave_direction(xt->dir))
1295                 return NULL;
1296
1297         if (xt->frame_size != 1 || xt->numf == 0)
1298                 return NULL;
1299
1300         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1301         if (!d)
1302                 return NULL;
1303
1304         data_type = __ffs((xt->src_start | xt->dst_start | xt->sgl[0].size));
1305         if (data_type > CSDP_DATA_TYPE_32)
1306                 data_type = CSDP_DATA_TYPE_32;
1307
1308         sg = &d->sg[0];
1309         d->dir = DMA_MEM_TO_MEM;
1310         d->dev_addr = xt->src_start;
1311         d->es = data_type;
1312         sg->en = xt->sgl[0].size / BIT(data_type);
1313         sg->fn = xt->numf;
1314         sg->addr = xt->dst_start;
1315         d->sglen = 1;
1316         d->ccr = c->ccr;
1317
1318         src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
1319         dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
1320         if (src_icg) {
1321                 d->ccr |= CCR_SRC_AMODE_DBLIDX;
1322                 d->ei = 1;
1323                 d->fi = src_icg + 1;
1324         } else if (xt->src_inc) {
1325                 d->ccr |= CCR_SRC_AMODE_POSTINC;
1326                 d->fi = 0;
1327         } else {
1328                 dev_err(chan->device->dev,
1329                         "%s: SRC constant addressing is not supported\n",
1330                         __func__);
1331                 kfree(d);
1332                 return NULL;
1333         }
1334
1335         if (dst_icg) {
1336                 d->ccr |= CCR_DST_AMODE_DBLIDX;
1337                 sg->ei = 1;
1338                 sg->fi = dst_icg + 1;
1339         } else if (xt->dst_inc) {
1340                 d->ccr |= CCR_DST_AMODE_POSTINC;
1341                 sg->fi = 0;
1342         } else {
1343                 dev_err(chan->device->dev,
1344                         "%s: DST constant addressing is not supported\n",
1345                         __func__);
1346                 kfree(d);
1347                 return NULL;
1348         }
1349
1350         d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
1351
1352         d->csdp = data_type;
1353
1354         if (dma_omap1()) {
1355                 d->cicr |= CICR_TOUT_IE;
1356                 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1357         } else {
1358                 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1359                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1360                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1361         }
1362
1363         return vchan_tx_prep(&c->vc, &d->vd, flags);
1364 }
1365
1366 static int omap_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg)
1367 {
1368         struct omap_chan *c = to_omap_dma_chan(chan);
1369
1370         if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
1371             cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
1372                 return -EINVAL;
1373
1374         if (cfg->src_maxburst > chan->device->max_burst ||
1375             cfg->dst_maxburst > chan->device->max_burst)
1376                 return -EINVAL;
1377
1378         memcpy(&c->cfg, cfg, sizeof(c->cfg));
1379
1380         return 0;
1381 }
1382
1383 static int omap_dma_terminate_all(struct dma_chan *chan)
1384 {
1385         struct omap_chan *c = to_omap_dma_chan(chan);
1386         unsigned long flags;
1387         LIST_HEAD(head);
1388
1389         spin_lock_irqsave(&c->vc.lock, flags);
1390
1391         /*
1392          * Stop DMA activity: we assume the callback will not be called
1393          * after omap_dma_stop() returns (even if it does, it will see
1394          * c->desc is NULL and exit.)
1395          */
1396         if (c->desc) {
1397                 vchan_terminate_vdesc(&c->desc->vd);
1398                 c->desc = NULL;
1399                 /* Avoid stopping the dma twice */
1400                 if (!c->paused)
1401                         omap_dma_stop(c);
1402         }
1403
1404         c->cyclic = false;
1405         c->paused = false;
1406
1407         vchan_get_all_descriptors(&c->vc, &head);
1408         spin_unlock_irqrestore(&c->vc.lock, flags);
1409         vchan_dma_desc_free_list(&c->vc, &head);
1410
1411         return 0;
1412 }
1413
1414 static void omap_dma_synchronize(struct dma_chan *chan)
1415 {
1416         struct omap_chan *c = to_omap_dma_chan(chan);
1417
1418         vchan_synchronize(&c->vc);
1419 }
1420
1421 static int omap_dma_pause(struct dma_chan *chan)
1422 {
1423         struct omap_chan *c = to_omap_dma_chan(chan);
1424         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1425         unsigned long flags;
1426         int ret = -EINVAL;
1427         bool can_pause = false;
1428
1429         spin_lock_irqsave(&od->irq_lock, flags);
1430
1431         if (!c->desc)
1432                 goto out;
1433
1434         if (c->cyclic)
1435                 can_pause = true;
1436
1437         /*
1438          * We do not allow DMA_MEM_TO_DEV transfers to be paused.
1439          * From the AM572x TRM, 16.1.4.18 Disabling a Channel During Transfer:
1440          * "When a channel is disabled during a transfer, the channel undergoes
1441          * an abort, unless it is hardware-source-synchronized …".
1442          * A source-synchronised channel is one where the fetching of data is
1443          * under control of the device. In other words, a device-to-memory
1444          * transfer. So, a destination-synchronised channel (which would be a
1445          * memory-to-device transfer) undergoes an abort if the the CCR_ENABLE
1446          * bit is cleared.
1447          * From 16.1.4.20.4.6.2 Abort: "If an abort trigger occurs, the channel
1448          * aborts immediately after completion of current read/write
1449          * transactions and then the FIFO is cleaned up." The term "cleaned up"
1450          * is not defined. TI recommends to check that RD_ACTIVE and WR_ACTIVE
1451          * are both clear _before_ disabling the channel, otherwise data loss
1452          * will occur.
1453          * The problem is that if the channel is active, then device activity
1454          * can result in DMA activity starting between reading those as both
1455          * clear and the write to DMA_CCR to clear the enable bit hitting the
1456          * hardware. If the DMA hardware can't drain the data in its FIFO to the
1457          * destination, then data loss "might" occur (say if we write to an UART
1458          * and the UART is not accepting any further data).
1459          */
1460         else if (c->desc->dir == DMA_DEV_TO_MEM)
1461                 can_pause = true;
1462
1463         if (can_pause && !c->paused) {
1464                 ret = omap_dma_stop(c);
1465                 if (!ret)
1466                         c->paused = true;
1467         }
1468 out:
1469         spin_unlock_irqrestore(&od->irq_lock, flags);
1470
1471         return ret;
1472 }
1473
1474 static int omap_dma_resume(struct dma_chan *chan)
1475 {
1476         struct omap_chan *c = to_omap_dma_chan(chan);
1477         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1478         unsigned long flags;
1479         int ret = -EINVAL;
1480
1481         spin_lock_irqsave(&od->irq_lock, flags);
1482
1483         if (c->paused && c->desc) {
1484                 mb();
1485
1486                 /* Restore channel link register */
1487                 omap_dma_chan_write(c, CLNK_CTRL, c->desc->clnk_ctrl);
1488
1489                 omap_dma_start(c, c->desc);
1490                 c->paused = false;
1491                 ret = 0;
1492         }
1493         spin_unlock_irqrestore(&od->irq_lock, flags);
1494
1495         return ret;
1496 }
1497
1498 static int omap_dma_chan_init(struct omap_dmadev *od)
1499 {
1500         struct omap_chan *c;
1501
1502         c = kzalloc(sizeof(*c), GFP_KERNEL);
1503         if (!c)
1504                 return -ENOMEM;
1505
1506         c->reg_map = od->reg_map;
1507         c->vc.desc_free = omap_dma_desc_free;
1508         vchan_init(&c->vc, &od->ddev);
1509
1510         return 0;
1511 }
1512
1513 static void omap_dma_free(struct omap_dmadev *od)
1514 {
1515         while (!list_empty(&od->ddev.channels)) {
1516                 struct omap_chan *c = list_first_entry(&od->ddev.channels,
1517                         struct omap_chan, vc.chan.device_node);
1518
1519                 list_del(&c->vc.chan.device_node);
1520                 tasklet_kill(&c->vc.task);
1521                 kfree(c);
1522         }
1523 }
1524
1525 /* Currently used by omap2 & 3 to block deeper SoC idle states */
1526 static bool omap_dma_busy(struct omap_dmadev *od)
1527 {
1528         struct omap_chan *c;
1529         int lch = -1;
1530
1531         while (1) {
1532                 lch = find_next_bit(od->lch_bitmap, od->lch_count, lch + 1);
1533                 if (lch >= od->lch_count)
1534                         break;
1535                 c = od->lch_map[lch];
1536                 if (!c)
1537                         continue;
1538                 if (omap_dma_chan_read(c, CCR) & CCR_ENABLE)
1539                         return true;
1540         }
1541
1542         return false;
1543 }
1544
1545 /* Currently only used for omap2. For omap1, also a check for lcd_dma is needed */
1546 static int omap_dma_busy_notifier(struct notifier_block *nb,
1547                                   unsigned long cmd, void *v)
1548 {
1549         struct omap_dmadev *od;
1550
1551         od = container_of(nb, struct omap_dmadev, nb);
1552
1553         switch (cmd) {
1554         case CPU_CLUSTER_PM_ENTER:
1555                 if (omap_dma_busy(od))
1556                         return NOTIFY_BAD;
1557                 break;
1558         case CPU_CLUSTER_PM_ENTER_FAILED:
1559         case CPU_CLUSTER_PM_EXIT:
1560                 break;
1561         }
1562
1563         return NOTIFY_OK;
1564 }
1565
1566 /*
1567  * We are using IRQENABLE_L1, and legacy DMA code was using IRQENABLE_L0.
1568  * As the DSP may be using IRQENABLE_L2 and L3, let's not touch those for
1569  * now. Context save seems to be only currently needed on omap3.
1570  */
1571 static void omap_dma_context_save(struct omap_dmadev *od)
1572 {
1573         od->context.irqenable_l0 = omap_dma_glbl_read(od, IRQENABLE_L0);
1574         od->context.irqenable_l1 = omap_dma_glbl_read(od, IRQENABLE_L1);
1575         od->context.ocp_sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG);
1576         od->context.gcr = omap_dma_glbl_read(od, GCR);
1577 }
1578
1579 static void omap_dma_context_restore(struct omap_dmadev *od)
1580 {
1581         int i;
1582
1583         omap_dma_glbl_write(od, GCR, od->context.gcr);
1584         omap_dma_glbl_write(od, OCP_SYSCONFIG, od->context.ocp_sysconfig);
1585         omap_dma_glbl_write(od, IRQENABLE_L0, od->context.irqenable_l0);
1586         omap_dma_glbl_write(od, IRQENABLE_L1, od->context.irqenable_l1);
1587
1588         /* Clear IRQSTATUS_L0 as legacy DMA code is no longer doing it */
1589         if (od->plat->errata & DMA_ROMCODE_BUG)
1590                 omap_dma_glbl_write(od, IRQSTATUS_L0, 0);
1591
1592         /* Clear dma channels */
1593         for (i = 0; i < od->lch_count; i++)
1594                 omap_dma_clear_lch(od, i);
1595 }
1596
1597 /* Currently only used for omap3 */
1598 static int omap_dma_context_notifier(struct notifier_block *nb,
1599                                      unsigned long cmd, void *v)
1600 {
1601         struct omap_dmadev *od;
1602
1603         od = container_of(nb, struct omap_dmadev, nb);
1604
1605         switch (cmd) {
1606         case CPU_CLUSTER_PM_ENTER:
1607                 if (omap_dma_busy(od))
1608                         return NOTIFY_BAD;
1609                 omap_dma_context_save(od);
1610                 break;
1611         case CPU_CLUSTER_PM_ENTER_FAILED:
1612         case CPU_CLUSTER_PM_EXIT:
1613                 omap_dma_context_restore(od);
1614                 break;
1615         }
1616
1617         return NOTIFY_OK;
1618 }
1619
1620 static void omap_dma_init_gcr(struct omap_dmadev *od, int arb_rate,
1621                               int max_fifo_depth, int tparams)
1622 {
1623         u32 val;
1624
1625         /* Set only for omap2430 and later */
1626         if (!od->cfg->rw_priority)
1627                 return;
1628
1629         if (max_fifo_depth == 0)
1630                 max_fifo_depth = 1;
1631         if (arb_rate == 0)
1632                 arb_rate = 1;
1633
1634         val = 0xff & max_fifo_depth;
1635         val |= (0x3 & tparams) << 12;
1636         val |= (arb_rate & 0xff) << 16;
1637
1638         omap_dma_glbl_write(od, GCR, val);
1639 }
1640
1641 #define OMAP_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
1642                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
1643                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
1644
1645 /*
1646  * No flags currently set for default configuration as omap1 is still
1647  * using platform data.
1648  */
1649 static const struct omap_dma_config default_cfg;
1650
1651 static int omap_dma_probe(struct platform_device *pdev)
1652 {
1653         const struct omap_dma_config *conf;
1654         struct omap_dmadev *od;
1655         struct resource *res;
1656         int rc, i, irq;
1657         u32 val;
1658
1659         od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
1660         if (!od)
1661                 return -ENOMEM;
1662
1663         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1664         od->base = devm_ioremap_resource(&pdev->dev, res);
1665         if (IS_ERR(od->base))
1666                 return PTR_ERR(od->base);
1667
1668         conf = of_device_get_match_data(&pdev->dev);
1669         if (conf) {
1670                 od->cfg = conf;
1671                 od->plat = dev_get_platdata(&pdev->dev);
1672                 if (!od->plat) {
1673                         dev_err(&pdev->dev, "omap_system_dma_plat_info is missing");
1674                         return -ENODEV;
1675                 }
1676         } else {
1677                 od->cfg = &default_cfg;
1678
1679                 od->plat = omap_get_plat_info();
1680                 if (!od->plat)
1681                         return -EPROBE_DEFER;
1682         }
1683
1684         od->reg_map = od->plat->reg_map;
1685
1686         dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
1687         dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);
1688         dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);
1689         dma_cap_set(DMA_INTERLEAVE, od->ddev.cap_mask);
1690         od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources;
1691         od->ddev.device_free_chan_resources = omap_dma_free_chan_resources;
1692         od->ddev.device_tx_status = omap_dma_tx_status;
1693         od->ddev.device_issue_pending = omap_dma_issue_pending;
1694         od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg;
1695         od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic;
1696         od->ddev.device_prep_dma_memcpy = omap_dma_prep_dma_memcpy;
1697         od->ddev.device_prep_interleaved_dma = omap_dma_prep_dma_interleaved;
1698         od->ddev.device_config = omap_dma_slave_config;
1699         od->ddev.device_pause = omap_dma_pause;
1700         od->ddev.device_resume = omap_dma_resume;
1701         od->ddev.device_terminate_all = omap_dma_terminate_all;
1702         od->ddev.device_synchronize = omap_dma_synchronize;
1703         od->ddev.src_addr_widths = OMAP_DMA_BUSWIDTHS;
1704         od->ddev.dst_addr_widths = OMAP_DMA_BUSWIDTHS;
1705         od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1706         if (__dma_omap15xx(od->plat->dma_attr))
1707                 od->ddev.residue_granularity =
1708                                 DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
1709         else
1710                 od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1711         od->ddev.max_burst = SZ_16M - 1; /* CCEN: 24bit unsigned */
1712         od->ddev.dev = &pdev->dev;
1713         INIT_LIST_HEAD(&od->ddev.channels);
1714         mutex_init(&od->lch_lock);
1715         spin_lock_init(&od->lock);
1716         spin_lock_init(&od->irq_lock);
1717
1718         /* Number of DMA requests */
1719         od->dma_requests = OMAP_SDMA_REQUESTS;
1720         if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
1721                                                       "dma-requests",
1722                                                       &od->dma_requests)) {
1723                 dev_info(&pdev->dev,
1724                          "Missing dma-requests property, using %u.\n",
1725                          OMAP_SDMA_REQUESTS);
1726         }
1727
1728         /* Number of available logical channels */
1729         if (!pdev->dev.of_node) {
1730                 od->lch_count = od->plat->dma_attr->lch_count;
1731                 if (unlikely(!od->lch_count))
1732                         od->lch_count = OMAP_SDMA_CHANNELS;
1733         } else if (of_property_read_u32(pdev->dev.of_node, "dma-channels",
1734                                         &od->lch_count)) {
1735                 dev_info(&pdev->dev,
1736                          "Missing dma-channels property, using %u.\n",
1737                          OMAP_SDMA_CHANNELS);
1738                 od->lch_count = OMAP_SDMA_CHANNELS;
1739         }
1740
1741         /* Mask of allowed logical channels */
1742         if (pdev->dev.of_node && !of_property_read_u32(pdev->dev.of_node,
1743                                                        "dma-channel-mask",
1744                                                        &val)) {
1745                 /* Tag channels not in mask as reserved */
1746                 val = ~val;
1747                 bitmap_from_arr32(od->lch_bitmap, &val, od->lch_count);
1748         }
1749         if (od->plat->dma_attr->dev_caps & HS_CHANNELS_RESERVED)
1750                 bitmap_set(od->lch_bitmap, 0, 2);
1751
1752         od->lch_map = devm_kcalloc(&pdev->dev, od->lch_count,
1753                                    sizeof(*od->lch_map),
1754                                    GFP_KERNEL);
1755         if (!od->lch_map)
1756                 return -ENOMEM;
1757
1758         for (i = 0; i < od->dma_requests; i++) {
1759                 rc = omap_dma_chan_init(od);
1760                 if (rc) {
1761                         omap_dma_free(od);
1762                         return rc;
1763                 }
1764         }
1765
1766         irq = platform_get_irq(pdev, 1);
1767         if (irq <= 0) {
1768                 dev_info(&pdev->dev, "failed to get L1 IRQ: %d\n", irq);
1769                 od->legacy = true;
1770         } else {
1771                 /* Disable all interrupts */
1772                 od->irq_enable_mask = 0;
1773                 omap_dma_glbl_write(od, IRQENABLE_L1, 0);
1774
1775                 rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq,
1776                                       IRQF_SHARED, "omap-dma-engine", od);
1777                 if (rc) {
1778                         omap_dma_free(od);
1779                         return rc;
1780                 }
1781         }
1782
1783         if (omap_dma_glbl_read(od, CAPS_0) & CAPS_0_SUPPORT_LL123)
1784                 od->ll123_supported = true;
1785
1786         od->ddev.filter.map = od->plat->slave_map;
1787         od->ddev.filter.mapcnt = od->plat->slavecnt;
1788         od->ddev.filter.fn = omap_dma_filter_fn;
1789
1790         if (od->ll123_supported) {
1791                 od->desc_pool = dma_pool_create(dev_name(&pdev->dev),
1792                                                 &pdev->dev,
1793                                                 sizeof(struct omap_type2_desc),
1794                                                 4, 0);
1795                 if (!od->desc_pool) {
1796                         dev_err(&pdev->dev,
1797                                 "unable to allocate descriptor pool\n");
1798                         od->ll123_supported = false;
1799                 }
1800         }
1801
1802         rc = dma_async_device_register(&od->ddev);
1803         if (rc) {
1804                 pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n",
1805                         rc);
1806                 omap_dma_free(od);
1807                 return rc;
1808         }
1809
1810         platform_set_drvdata(pdev, od);
1811
1812         if (pdev->dev.of_node) {
1813                 omap_dma_info.dma_cap = od->ddev.cap_mask;
1814
1815                 /* Device-tree DMA controller registration */
1816                 rc = of_dma_controller_register(pdev->dev.of_node,
1817                                 of_dma_simple_xlate, &omap_dma_info);
1818                 if (rc) {
1819                         pr_warn("OMAP-DMA: failed to register DMA controller\n");
1820                         dma_async_device_unregister(&od->ddev);
1821                         omap_dma_free(od);
1822                 }
1823         }
1824
1825         omap_dma_init_gcr(od, DMA_DEFAULT_ARB_RATE, DMA_DEFAULT_FIFO_DEPTH, 0);
1826
1827         if (od->cfg->needs_busy_check) {
1828                 od->nb.notifier_call = omap_dma_busy_notifier;
1829                 cpu_pm_register_notifier(&od->nb);
1830         } else if (od->cfg->may_lose_context) {
1831                 od->nb.notifier_call = omap_dma_context_notifier;
1832                 cpu_pm_register_notifier(&od->nb);
1833         }
1834
1835         dev_info(&pdev->dev, "OMAP DMA engine driver%s\n",
1836                  od->ll123_supported ? " (LinkedList1/2/3 supported)" : "");
1837
1838         return rc;
1839 }
1840
1841 static int omap_dma_remove(struct platform_device *pdev)
1842 {
1843         struct omap_dmadev *od = platform_get_drvdata(pdev);
1844         int irq;
1845
1846         if (od->cfg->may_lose_context)
1847                 cpu_pm_unregister_notifier(&od->nb);
1848
1849         if (pdev->dev.of_node)
1850                 of_dma_controller_free(pdev->dev.of_node);
1851
1852         irq = platform_get_irq(pdev, 1);
1853         devm_free_irq(&pdev->dev, irq, od);
1854
1855         dma_async_device_unregister(&od->ddev);
1856
1857         if (!od->legacy) {
1858                 /* Disable all interrupts */
1859                 omap_dma_glbl_write(od, IRQENABLE_L0, 0);
1860         }
1861
1862         if (od->ll123_supported)
1863                 dma_pool_destroy(od->desc_pool);
1864
1865         omap_dma_free(od);
1866
1867         return 0;
1868 }
1869
1870 static const struct omap_dma_config omap2420_data = {
1871         .lch_end = CCFN,
1872         .rw_priority = true,
1873         .needs_lch_clear = true,
1874         .needs_busy_check = true,
1875 };
1876
1877 static const struct omap_dma_config omap2430_data = {
1878         .lch_end = CCFN,
1879         .rw_priority = true,
1880         .needs_lch_clear = true,
1881 };
1882
1883 static const struct omap_dma_config omap3430_data = {
1884         .lch_end = CCFN,
1885         .rw_priority = true,
1886         .needs_lch_clear = true,
1887         .may_lose_context = true,
1888 };
1889
1890 static const struct omap_dma_config omap3630_data = {
1891         .lch_end = CCDN,
1892         .rw_priority = true,
1893         .needs_lch_clear = true,
1894         .may_lose_context = true,
1895 };
1896
1897 static const struct omap_dma_config omap4_data = {
1898         .lch_end = CCDN,
1899         .rw_priority = true,
1900         .needs_lch_clear = true,
1901 };
1902
1903 static const struct of_device_id omap_dma_match[] = {
1904         { .compatible = "ti,omap2420-sdma", .data = &omap2420_data, },
1905         { .compatible = "ti,omap2430-sdma", .data = &omap2430_data, },
1906         { .compatible = "ti,omap3430-sdma", .data = &omap3430_data, },
1907         { .compatible = "ti,omap3630-sdma", .data = &omap3630_data, },
1908         { .compatible = "ti,omap4430-sdma", .data = &omap4_data, },
1909         {},
1910 };
1911 MODULE_DEVICE_TABLE(of, omap_dma_match);
1912
1913 static struct platform_driver omap_dma_driver = {
1914         .probe  = omap_dma_probe,
1915         .remove = omap_dma_remove,
1916         .driver = {
1917                 .name = "omap-dma-engine",
1918                 .of_match_table = omap_dma_match,
1919         },
1920 };
1921
1922 static bool omap_dma_filter_fn(struct dma_chan *chan, void *param)
1923 {
1924         if (chan->device->dev->driver == &omap_dma_driver.driver) {
1925                 struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1926                 struct omap_chan *c = to_omap_dma_chan(chan);
1927                 unsigned req = *(unsigned *)param;
1928
1929                 if (req <= od->dma_requests) {
1930                         c->dma_sig = req;
1931                         return true;
1932                 }
1933         }
1934         return false;
1935 }
1936
1937 static int omap_dma_init(void)
1938 {
1939         return platform_driver_register(&omap_dma_driver);
1940 }
1941 subsys_initcall(omap_dma_init);
1942
1943 static void __exit omap_dma_exit(void)
1944 {
1945         platform_driver_unregister(&omap_dma_driver);
1946 }
1947 module_exit(omap_dma_exit);
1948
1949 MODULE_AUTHOR("Russell King");
1950 MODULE_LICENSE("GPL");