drivers/dma/omap-dma.c

   1 /*
   2  * OMAP DMAengine support
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8 #include <linux/delay.h>
   9 #include <linux/dmaengine.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/dmapool.h>
  12 #include <linux/err.h>
  13 #include <linux/init.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/list.h>
  16 #include <linux/module.h>
  17 #include <linux/omap-dma.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/slab.h>
  20 #include <linux/spinlock.h>
  21 #include <linux/of_dma.h>
  22 #include <linux/of_device.h>
  23
  24 #include "virt-dma.h"
  25
  26 #define OMAP_SDMA_REQUESTS      127
  27 #define OMAP_SDMA_CHANNELS      32
  28
  29 struct omap_dmadev {
  30         struct dma_device ddev;
  31         spinlock_t lock;
  32         void __iomem *base;
  33         const struct omap_dma_reg *reg_map;
  34         struct omap_system_dma_plat_info *plat;
  35         bool legacy;
  36         bool ll123_supported;
  37         struct dma_pool *desc_pool;
  38         unsigned dma_requests;
  39         spinlock_t irq_lock;
  40         uint32_t irq_enable_mask;
  41         struct omap_chan **lch_map;
  42 };
  43
  44 struct omap_chan {
  45         struct virt_dma_chan vc;
  46         void __iomem *channel_base;
  47         const struct omap_dma_reg *reg_map;
  48         uint32_t ccr;
  49
  50         struct dma_slave_config cfg;
  51         unsigned dma_sig;
  52         bool cyclic;
  53         bool paused;
  54         bool running;
  55
  56         int dma_ch;
  57         struct omap_desc *desc;
  58         unsigned sgidx;
  59 };
  60
  61 #define DESC_NXT_SV_REFRESH     (0x1 << 24)
  62 #define DESC_NXT_SV_REUSE       (0x2 << 24)
  63 #define DESC_NXT_DV_REFRESH     (0x1 << 26)
  64 #define DESC_NXT_DV_REUSE       (0x2 << 26)
  65 #define DESC_NTYPE_TYPE2        (0x2 << 29)
  66
  67 /* Type 2 descriptor with Source or Destination address update */
  68 struct omap_type2_desc {
  69         uint32_t next_desc;
  70         uint32_t en;
  71         uint32_t addr; /* src or dst */
  72         uint16_t fn;
  73         uint16_t cicr;
  74         int16_t cdei;
  75         int16_t csei;
  76         int32_t cdfi;
  77         int32_t csfi;
  78 } __packed;
  79
  80 struct omap_sg {
  81         dma_addr_t addr;
  82         uint32_t en;            /* number of elements (24-bit) */
  83         uint32_t fn;            /* number of frames (16-bit) */
  84         int32_t fi;             /* for double indexing */
  85         int16_t ei;             /* for double indexing */
  86
  87         /* Linked list */
  88         struct omap_type2_desc *t2_desc;
  89         dma_addr_t t2_desc_paddr;
  90 };
  91
  92 struct omap_desc {
  93         struct virt_dma_desc vd;
  94         bool using_ll;
  95         enum dma_transfer_direction dir;
  96         dma_addr_t dev_addr;
  97
  98         int32_t fi;             /* for OMAP_DMA_SYNC_PACKET / double indexing */
  99         int16_t ei;             /* for double indexing */
 100         uint8_t es;             /* CSDP_DATA_TYPE_xxx */
 101         uint32_t ccr;           /* CCR value */
 102         uint16_t clnk_ctrl;     /* CLNK_CTRL value */
 103         uint16_t cicr;          /* CICR value */
 104         uint32_t csdp;          /* CSDP value */
 105
 106         unsigned sglen;
 107         struct omap_sg sg[0];
 108 };
 109
 110 enum {
 111         CAPS_0_SUPPORT_LL123    = BIT(20),      /* Linked List type1/2/3 */
 112         CAPS_0_SUPPORT_LL4      = BIT(21),      /* Linked List type4 */
 113
 114         CCR_FS                  = BIT(5),
 115         CCR_READ_PRIORITY       = BIT(6),
 116         CCR_ENABLE              = BIT(7),
 117         CCR_AUTO_INIT           = BIT(8),       /* OMAP1 only */
 118         CCR_REPEAT              = BIT(9),       /* OMAP1 only */
 119         CCR_OMAP31_DISABLE      = BIT(10),      /* OMAP1 only */
 120         CCR_SUSPEND_SENSITIVE   = BIT(8),       /* OMAP2+ only */
 121         CCR_RD_ACTIVE           = BIT(9),       /* OMAP2+ only */
 122         CCR_WR_ACTIVE           = BIT(10),      /* OMAP2+ only */
 123         CCR_SRC_AMODE_CONSTANT  = 0 << 12,
 124         CCR_SRC_AMODE_POSTINC   = 1 << 12,
 125         CCR_SRC_AMODE_SGLIDX    = 2 << 12,
 126         CCR_SRC_AMODE_DBLIDX    = 3 << 12,
 127         CCR_DST_AMODE_CONSTANT  = 0 << 14,
 128         CCR_DST_AMODE_POSTINC   = 1 << 14,
 129         CCR_DST_AMODE_SGLIDX    = 2 << 14,
 130         CCR_DST_AMODE_DBLIDX    = 3 << 14,
 131         CCR_CONSTANT_FILL       = BIT(16),
 132         CCR_TRANSPARENT_COPY    = BIT(17),
 133         CCR_BS                  = BIT(18),
 134         CCR_SUPERVISOR          = BIT(22),
 135         CCR_PREFETCH            = BIT(23),
 136         CCR_TRIGGER_SRC         = BIT(24),
 137         CCR_BUFFERING_DISABLE   = BIT(25),
 138         CCR_WRITE_PRIORITY      = BIT(26),
 139         CCR_SYNC_ELEMENT        = 0,
 140         CCR_SYNC_FRAME          = CCR_FS,
 141         CCR_SYNC_BLOCK          = CCR_BS,
 142         CCR_SYNC_PACKET         = CCR_BS | CCR_FS,
 143
 144         CSDP_DATA_TYPE_8        = 0,
 145         CSDP_DATA_TYPE_16       = 1,
 146         CSDP_DATA_TYPE_32       = 2,
 147         CSDP_SRC_PORT_EMIFF     = 0 << 2, /* OMAP1 only */
 148         CSDP_SRC_PORT_EMIFS     = 1 << 2, /* OMAP1 only */
 149         CSDP_SRC_PORT_OCP_T1    = 2 << 2, /* OMAP1 only */
 150         CSDP_SRC_PORT_TIPB      = 3 << 2, /* OMAP1 only */
 151         CSDP_SRC_PORT_OCP_T2    = 4 << 2, /* OMAP1 only */
 152         CSDP_SRC_PORT_MPUI      = 5 << 2, /* OMAP1 only */
 153         CSDP_SRC_PACKED         = BIT(6),
 154         CSDP_SRC_BURST_1        = 0 << 7,
 155         CSDP_SRC_BURST_16       = 1 << 7,
 156         CSDP_SRC_BURST_32       = 2 << 7,
 157         CSDP_SRC_BURST_64       = 3 << 7,
 158         CSDP_DST_PORT_EMIFF     = 0 << 9, /* OMAP1 only */
 159         CSDP_DST_PORT_EMIFS     = 1 << 9, /* OMAP1 only */
 160         CSDP_DST_PORT_OCP_T1    = 2 << 9, /* OMAP1 only */
 161         CSDP_DST_PORT_TIPB      = 3 << 9, /* OMAP1 only */
 162         CSDP_DST_PORT_OCP_T2    = 4 << 9, /* OMAP1 only */
 163         CSDP_DST_PORT_MPUI      = 5 << 9, /* OMAP1 only */
 164         CSDP_DST_PACKED         = BIT(13),
 165         CSDP_DST_BURST_1        = 0 << 14,
 166         CSDP_DST_BURST_16       = 1 << 14,
 167         CSDP_DST_BURST_32       = 2 << 14,
 168         CSDP_DST_BURST_64       = 3 << 14,
 169         CSDP_WRITE_NON_POSTED   = 0 << 16,
 170         CSDP_WRITE_POSTED       = 1 << 16,
 171         CSDP_WRITE_LAST_NON_POSTED = 2 << 16,
 172
 173         CICR_TOUT_IE            = BIT(0),       /* OMAP1 only */
 174         CICR_DROP_IE            = BIT(1),
 175         CICR_HALF_IE            = BIT(2),
 176         CICR_FRAME_IE           = BIT(3),
 177         CICR_LAST_IE            = BIT(4),
 178         CICR_BLOCK_IE           = BIT(5),
 179         CICR_PKT_IE             = BIT(7),       /* OMAP2+ only */
 180         CICR_TRANS_ERR_IE       = BIT(8),       /* OMAP2+ only */
 181         CICR_SUPERVISOR_ERR_IE  = BIT(10),      /* OMAP2+ only */
 182         CICR_MISALIGNED_ERR_IE  = BIT(11),      /* OMAP2+ only */
 183         CICR_DRAIN_IE           = BIT(12),      /* OMAP2+ only */
 184         CICR_SUPER_BLOCK_IE     = BIT(14),      /* OMAP2+ only */
 185
 186         CLNK_CTRL_ENABLE_LNK    = BIT(15),
 187
 188         CDP_DST_VALID_INC       = 0 << 0,
 189         CDP_DST_VALID_RELOAD    = 1 << 0,
 190         CDP_DST_VALID_REUSE     = 2 << 0,
 191         CDP_SRC_VALID_INC       = 0 << 2,
 192         CDP_SRC_VALID_RELOAD    = 1 << 2,
 193         CDP_SRC_VALID_REUSE     = 2 << 2,
 194         CDP_NTYPE_TYPE1         = 1 << 4,
 195         CDP_NTYPE_TYPE2         = 2 << 4,
 196         CDP_NTYPE_TYPE3         = 3 << 4,
 197         CDP_TMODE_NORMAL        = 0 << 8,
 198         CDP_TMODE_LLIST         = 1 << 8,
 199         CDP_FAST                = BIT(10),
 200 };
 201
 202 static const unsigned es_bytes[] = {
 203         [CSDP_DATA_TYPE_8] = 1,
 204         [CSDP_DATA_TYPE_16] = 2,
 205         [CSDP_DATA_TYPE_32] = 4,
 206 };
 207
 208 static struct of_dma_filter_info omap_dma_info = {
 209         .filter_fn = omap_dma_filter_fn,
 210 };
 211
 212 static inline struct omap_dmadev *to_omap_dma_dev(struct dma_device *d)
 213 {
 214         return container_of(d, struct omap_dmadev, ddev);
 215 }
 216
 217 static inline struct omap_chan *to_omap_dma_chan(struct dma_chan *c)
 218 {
 219         return container_of(c, struct omap_chan, vc.chan);
 220 }
 221
 222 static inline struct omap_desc *to_omap_dma_desc(struct dma_async_tx_descriptor *t)
 223 {
 224         return container_of(t, struct omap_desc, vd.tx);
 225 }
 226
 227 static void omap_dma_desc_free(struct virt_dma_desc *vd)
 228 {
 229         struct omap_desc *d = to_omap_dma_desc(&vd->tx);
 230
 231         if (d->using_ll) {
 232                 struct omap_dmadev *od = to_omap_dma_dev(vd->tx.chan->device);
 233                 int i;
 234
 235                 for (i = 0; i < d->sglen; i++) {
 236                         if (d->sg[i].t2_desc)
 237                                 dma_pool_free(od->desc_pool, d->sg[i].t2_desc,
 238                                               d->sg[i].t2_desc_paddr);
 239                 }
 240         }
 241
 242         kfree(d);
 243 }
 244
 245 static void omap_dma_fill_type2_desc(struct omap_desc *d, int idx,
 246                                      enum dma_transfer_direction dir, bool last)
 247 {
 248         struct omap_sg *sg = &d->sg[idx];
 249         struct omap_type2_desc *t2_desc = sg->t2_desc;
 250
 251         if (idx)
 252                 d->sg[idx - 1].t2_desc->next_desc = sg->t2_desc_paddr;
 253         if (last)
 254                 t2_desc->next_desc = 0xfffffffc;
 255
 256         t2_desc->en = sg->en;
 257         t2_desc->addr = sg->addr;
 258         t2_desc->fn = sg->fn & 0xffff;
 259         t2_desc->cicr = d->cicr;
 260         if (!last)
 261                 t2_desc->cicr &= ~CICR_BLOCK_IE;
 262
 263         switch (dir) {
 264         case DMA_DEV_TO_MEM:
 265                 t2_desc->cdei = sg->ei;
 266                 t2_desc->csei = d->ei;
 267                 t2_desc->cdfi = sg->fi;
 268                 t2_desc->csfi = d->fi;
 269
 270                 t2_desc->en |= DESC_NXT_DV_REFRESH;
 271                 t2_desc->en |= DESC_NXT_SV_REUSE;
 272                 break;
 273         case DMA_MEM_TO_DEV:
 274                 t2_desc->cdei = d->ei;
 275                 t2_desc->csei = sg->ei;
 276                 t2_desc->cdfi = d->fi;
 277                 t2_desc->csfi = sg->fi;
 278
 279                 t2_desc->en |= DESC_NXT_SV_REFRESH;
 280                 t2_desc->en |= DESC_NXT_DV_REUSE;
 281                 break;
 282         default:
 283                 return;
 284         }
 285
 286         t2_desc->en |= DESC_NTYPE_TYPE2;
 287 }
 288
 289 static void omap_dma_write(uint32_t val, unsigned type, void __iomem *addr)
 290 {
 291         switch (type) {
 292         case OMAP_DMA_REG_16BIT:
 293                 writew_relaxed(val, addr);
 294                 break;
 295         case OMAP_DMA_REG_2X16BIT:
 296                 writew_relaxed(val, addr);
 297                 writew_relaxed(val >> 16, addr + 2);
 298                 break;
 299         case OMAP_DMA_REG_32BIT:
 300                 writel_relaxed(val, addr);
 301                 break;
 302         default:
 303                 WARN_ON(1);
 304         }
 305 }
 306
 307 static unsigned omap_dma_read(unsigned type, void __iomem *addr)
 308 {
 309         unsigned val;
 310
 311         switch (type) {
 312         case OMAP_DMA_REG_16BIT:
 313                 val = readw_relaxed(addr);
 314                 break;
 315         case OMAP_DMA_REG_2X16BIT:
 316                 val = readw_relaxed(addr);
 317                 val |= readw_relaxed(addr + 2) << 16;
 318                 break;
 319         case OMAP_DMA_REG_32BIT:
 320                 val = readl_relaxed(addr);
 321                 break;
 322         default:
 323                 WARN_ON(1);
 324                 val = 0;
 325         }
 326
 327         return val;
 328 }
 329
 330 static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val)
 331 {
 332         const struct omap_dma_reg *r = od->reg_map + reg;
 333
 334         WARN_ON(r->stride);
 335
 336         omap_dma_write(val, r->type, od->base + r->offset);
 337 }
 338
 339 static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg)
 340 {
 341         const struct omap_dma_reg *r = od->reg_map + reg;
 342
 343         WARN_ON(r->stride);
 344
 345         return omap_dma_read(r->type, od->base + r->offset);
 346 }
 347
 348 static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val)
 349 {
 350         const struct omap_dma_reg *r = c->reg_map + reg;
 351
 352         omap_dma_write(val, r->type, c->channel_base + r->offset);
 353 }
 354
 355 static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg)
 356 {
 357         const struct omap_dma_reg *r = c->reg_map + reg;
 358
 359         return omap_dma_read(r->type, c->channel_base + r->offset);
 360 }
 361
 362 static void omap_dma_clear_csr(struct omap_chan *c)
 363 {
 364         if (dma_omap1())
 365                 omap_dma_chan_read(c, CSR);
 366         else
 367                 omap_dma_chan_write(c, CSR, ~0);
 368 }
 369
 370 static unsigned omap_dma_get_csr(struct omap_chan *c)
 371 {
 372         unsigned val = omap_dma_chan_read(c, CSR);
 373
 374         if (!dma_omap1())
 375                 omap_dma_chan_write(c, CSR, val);
 376
 377         return val;
 378 }
 379
 380 static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c,
 381         unsigned lch)
 382 {
 383         c->channel_base = od->base + od->plat->channel_stride * lch;
 384
 385         od->lch_map[lch] = c;
 386 }
 387
 388 static void omap_dma_start(struct omap_chan *c, struct omap_desc *d)
 389 {
 390         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 391         uint16_t cicr = d->cicr;
 392
 393         if (__dma_omap15xx(od->plat->dma_attr))
 394                 omap_dma_chan_write(c, CPC, 0);
 395         else
 396                 omap_dma_chan_write(c, CDAC, 0);
 397
 398         omap_dma_clear_csr(c);
 399
 400         if (d->using_ll) {
 401                 uint32_t cdp = CDP_TMODE_LLIST | CDP_NTYPE_TYPE2 | CDP_FAST;
 402
 403                 if (d->dir == DMA_DEV_TO_MEM)
 404                         cdp |= (CDP_DST_VALID_RELOAD | CDP_SRC_VALID_REUSE);
 405                 else
 406                         cdp |= (CDP_DST_VALID_REUSE | CDP_SRC_VALID_RELOAD);
 407                 omap_dma_chan_write(c, CDP, cdp);
 408
 409                 omap_dma_chan_write(c, CNDP, d->sg[0].t2_desc_paddr);
 410                 omap_dma_chan_write(c, CCDN, 0);
 411                 omap_dma_chan_write(c, CCFN, 0xffff);
 412                 omap_dma_chan_write(c, CCEN, 0xffffff);
 413
 414                 cicr &= ~CICR_BLOCK_IE;
 415         } else if (od->ll123_supported) {
 416                 omap_dma_chan_write(c, CDP, 0);
 417         }
 418
 419         /* Enable interrupts */
 420         omap_dma_chan_write(c, CICR, cicr);
 421
 422         /* Enable channel */
 423         omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE);
 424
 425         c->running = true;
 426 }
 427
 428 static void omap_dma_drain_chan(struct omap_chan *c)
 429 {
 430         int i;
 431         u32 val;
 432
 433         /* Wait for sDMA FIFO to drain */
 434         for (i = 0; ; i++) {
 435                 val = omap_dma_chan_read(c, CCR);
 436                 if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)))
 437                         break;
 438
 439                 if (i > 100)
 440                         break;
 441
 442                 udelay(5);
 443         }
 444
 445         if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))
 446                 dev_err(c->vc.chan.device->dev,
 447                         "DMA drain did not complete on lch %d\n",
 448                         c->dma_ch);
 449 }
 450
 451 static int omap_dma_stop(struct omap_chan *c)
 452 {
 453         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 454         uint32_t val;
 455
 456         /* disable irq */
 457         omap_dma_chan_write(c, CICR, 0);
 458
 459         omap_dma_clear_csr(c);
 460
 461         val = omap_dma_chan_read(c, CCR);
 462         if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) {
 463                 uint32_t sysconfig;
 464
 465                 sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG);
 466                 val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK;
 467                 val |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE);
 468                 omap_dma_glbl_write(od, OCP_SYSCONFIG, val);
 469
 470                 val = omap_dma_chan_read(c, CCR);
 471                 val &= ~CCR_ENABLE;
 472                 omap_dma_chan_write(c, CCR, val);
 473
 474                 if (!(c->ccr & CCR_BUFFERING_DISABLE))
 475                         omap_dma_drain_chan(c);
 476
 477                 omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig);
 478         } else {
 479                 if (!(val & CCR_ENABLE))
 480                         return -EINVAL;
 481
 482                 val &= ~CCR_ENABLE;
 483                 omap_dma_chan_write(c, CCR, val);
 484
 485                 if (!(c->ccr & CCR_BUFFERING_DISABLE))
 486                         omap_dma_drain_chan(c);
 487         }
 488
 489         mb();
 490
 491         if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) {
 492                 val = omap_dma_chan_read(c, CLNK_CTRL);
 493
 494                 if (dma_omap1())
 495                         val |= 1 << 14; /* set the STOP_LNK bit */
 496                 else
 497                         val &= ~CLNK_CTRL_ENABLE_LNK;
 498
 499                 omap_dma_chan_write(c, CLNK_CTRL, val);
 500         }
 501         c->running = false;
 502         return 0;
 503 }
 504
 505 static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d)
 506 {
 507         struct omap_sg *sg = d->sg + c->sgidx;
 508         unsigned cxsa, cxei, cxfi;
 509
 510         if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
 511                 cxsa = CDSA;
 512                 cxei = CDEI;
 513                 cxfi = CDFI;
 514         } else {
 515                 cxsa = CSSA;
 516                 cxei = CSEI;
 517                 cxfi = CSFI;
 518         }
 519
 520         omap_dma_chan_write(c, cxsa, sg->addr);
 521         omap_dma_chan_write(c, cxei, sg->ei);
 522         omap_dma_chan_write(c, cxfi, sg->fi);
 523         omap_dma_chan_write(c, CEN, sg->en);
 524         omap_dma_chan_write(c, CFN, sg->fn);
 525
 526         omap_dma_start(c, d);
 527         c->sgidx++;
 528 }
 529
 530 static void omap_dma_start_desc(struct omap_chan *c)
 531 {
 532         struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
 533         struct omap_desc *d;
 534         unsigned cxsa, cxei, cxfi;
 535
 536         if (!vd) {
 537                 c->desc = NULL;
 538                 return;
 539         }
 540
 541         list_del(&vd->node);
 542
 543         c->desc = d = to_omap_dma_desc(&vd->tx);
 544         c->sgidx = 0;
 545
 546         /*
 547          * This provides the necessary barrier to ensure data held in
 548          * DMA coherent memory is visible to the DMA engine prior to
 549          * the transfer starting.
 550          */
 551         mb();
 552
 553         omap_dma_chan_write(c, CCR, d->ccr);
 554         if (dma_omap1())
 555                 omap_dma_chan_write(c, CCR2, d->ccr >> 16);
 556
 557         if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
 558                 cxsa = CSSA;
 559                 cxei = CSEI;
 560                 cxfi = CSFI;
 561         } else {
 562                 cxsa = CDSA;
 563                 cxei = CDEI;
 564                 cxfi = CDFI;
 565         }
 566
 567         omap_dma_chan_write(c, cxsa, d->dev_addr);
 568         omap_dma_chan_write(c, cxei, d->ei);
 569         omap_dma_chan_write(c, cxfi, d->fi);
 570         omap_dma_chan_write(c, CSDP, d->csdp);
 571         omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl);
 572
 573         omap_dma_start_sg(c, d);
 574 }
 575
 576 static void omap_dma_callback(int ch, u16 status, void *data)
 577 {
 578         struct omap_chan *c = data;
 579         struct omap_desc *d;
 580         unsigned long flags;
 581
 582         spin_lock_irqsave(&c->vc.lock, flags);
 583         d = c->desc;
 584         if (d) {
 585                 if (c->cyclic) {
 586                         vchan_cyclic_callback(&d->vd);
 587                 } else if (d->using_ll || c->sgidx == d->sglen) {
 588                         omap_dma_start_desc(c);
 589                         vchan_cookie_complete(&d->vd);
 590                 } else {
 591                         omap_dma_start_sg(c, d);
 592                 }
 593         }
 594         spin_unlock_irqrestore(&c->vc.lock, flags);
 595 }
 596
 597 static irqreturn_t omap_dma_irq(int irq, void *devid)
 598 {
 599         struct omap_dmadev *od = devid;
 600         unsigned status, channel;
 601
 602         spin_lock(&od->irq_lock);
 603
 604         status = omap_dma_glbl_read(od, IRQSTATUS_L1);
 605         status &= od->irq_enable_mask;
 606         if (status == 0) {
 607                 spin_unlock(&od->irq_lock);
 608                 return IRQ_NONE;
 609         }
 610
 611         while ((channel = ffs(status)) != 0) {
 612                 unsigned mask, csr;
 613                 struct omap_chan *c;
 614
 615                 channel -= 1;
 616                 mask = BIT(channel);
 617                 status &= ~mask;
 618
 619                 c = od->lch_map[channel];
 620                 if (c == NULL) {
 621                         /* This should never happen */
 622                         dev_err(od->ddev.dev, "invalid channel %u\n", channel);
 623                         continue;
 624                 }
 625
 626                 csr = omap_dma_get_csr(c);
 627                 omap_dma_glbl_write(od, IRQSTATUS_L1, mask);
 628
 629                 omap_dma_callback(channel, csr, c);
 630         }
 631
 632         spin_unlock(&od->irq_lock);
 633
 634         return IRQ_HANDLED;
 635 }
 636
 637 static int omap_dma_alloc_chan_resources(struct dma_chan *chan)
 638 {
 639         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 640         struct omap_chan *c = to_omap_dma_chan(chan);
 641         struct device *dev = od->ddev.dev;
 642         int ret;
 643
 644         if (od->legacy) {
 645                 ret = omap_request_dma(c->dma_sig, "DMA engine",
 646                                        omap_dma_callback, c, &c->dma_ch);
 647         } else {
 648                 ret = omap_request_dma(c->dma_sig, "DMA engine", NULL, NULL,
 649                                        &c->dma_ch);
 650         }
 651
 652         dev_dbg(dev, "allocating channel %u for %u\n", c->dma_ch, c->dma_sig);
 653
 654         if (ret >= 0) {
 655                 omap_dma_assign(od, c, c->dma_ch);
 656
 657                 if (!od->legacy) {
 658                         unsigned val;
 659
 660                         spin_lock_irq(&od->irq_lock);
 661                         val = BIT(c->dma_ch);
 662                         omap_dma_glbl_write(od, IRQSTATUS_L1, val);
 663                         od->irq_enable_mask |= val;
 664                         omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
 665
 666                         val = omap_dma_glbl_read(od, IRQENABLE_L0);
 667                         val &= ~BIT(c->dma_ch);
 668                         omap_dma_glbl_write(od, IRQENABLE_L0, val);
 669                         spin_unlock_irq(&od->irq_lock);
 670                 }
 671         }
 672
 673         if (dma_omap1()) {
 674                 if (__dma_omap16xx(od->plat->dma_attr)) {
 675                         c->ccr = CCR_OMAP31_DISABLE;
 676                         /* Duplicate what plat-omap/dma.c does */
 677                         c->ccr |= c->dma_ch + 1;
 678                 } else {
 679                         c->ccr = c->dma_sig & 0x1f;
 680                 }
 681         } else {
 682                 c->ccr = c->dma_sig & 0x1f;
 683                 c->ccr |= (c->dma_sig & ~0x1f) << 14;
 684         }
 685         if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING)
 686                 c->ccr |= CCR_BUFFERING_DISABLE;
 687
 688         return ret;
 689 }
 690
 691 static void omap_dma_free_chan_resources(struct dma_chan *chan)
 692 {
 693         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 694         struct omap_chan *c = to_omap_dma_chan(chan);
 695
 696         if (!od->legacy) {
 697                 spin_lock_irq(&od->irq_lock);
 698                 od->irq_enable_mask &= ~BIT(c->dma_ch);
 699                 omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
 700                 spin_unlock_irq(&od->irq_lock);
 701         }
 702
 703         c->channel_base = NULL;
 704         od->lch_map[c->dma_ch] = NULL;
 705         vchan_free_chan_resources(&c->vc);
 706         omap_free_dma(c->dma_ch);
 707
 708         dev_dbg(od->ddev.dev, "freeing channel %u used for %u\n", c->dma_ch,
 709                 c->dma_sig);
 710         c->dma_sig = 0;
 711 }
 712
 713 static size_t omap_dma_sg_size(struct omap_sg *sg)
 714 {
 715         return sg->en * sg->fn;
 716 }
 717
 718 static size_t omap_dma_desc_size(struct omap_desc *d)
 719 {
 720         unsigned i;
 721         size_t size;
 722
 723         for (size = i = 0; i < d->sglen; i++)
 724                 size += omap_dma_sg_size(&d->sg[i]);
 725
 726         return size * es_bytes[d->es];
 727 }
 728
 729 static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr)
 730 {
 731         unsigned i;
 732         size_t size, es_size = es_bytes[d->es];
 733
 734         for (size = i = 0; i < d->sglen; i++) {
 735                 size_t this_size = omap_dma_sg_size(&d->sg[i]) * es_size;
 736
 737                 if (size)
 738                         size += this_size;
 739                 else if (addr >= d->sg[i].addr &&
 740                          addr < d->sg[i].addr + this_size)
 741                         size += d->sg[i].addr + this_size - addr;
 742         }
 743         return size;
 744 }
 745
 746 /*
 747  * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
 748  * read before the DMA controller finished disabling the channel.
 749  */
 750 static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg)
 751 {
 752         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 753         uint32_t val;
 754
 755         val = omap_dma_chan_read(c, reg);
 756         if (val == 0 && od->plat->errata & DMA_ERRATA_3_3)
 757                 val = omap_dma_chan_read(c, reg);
 758
 759         return val;
 760 }
 761
 762 static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c)
 763 {
 764         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 765         dma_addr_t addr, cdac;
 766
 767         if (__dma_omap15xx(od->plat->dma_attr)) {
 768                 addr = omap_dma_chan_read(c, CPC);
 769         } else {
 770                 addr = omap_dma_chan_read_3_3(c, CSAC);
 771                 cdac = omap_dma_chan_read_3_3(c, CDAC);
 772
 773                 /*
 774                  * CDAC == 0 indicates that the DMA transfer on the channel has
 775                  * not been started (no data has been transferred so far).
 776                  * Return the programmed source start address in this case.
 777                  */
 778                 if (cdac == 0)
 779                         addr = omap_dma_chan_read(c, CSSA);
 780         }
 781
 782         if (dma_omap1())
 783                 addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000;
 784
 785         return addr;
 786 }
 787
 788 static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c)
 789 {
 790         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 791         dma_addr_t addr;
 792
 793         if (__dma_omap15xx(od->plat->dma_attr)) {
 794                 addr = omap_dma_chan_read(c, CPC);
 795         } else {
 796                 addr = omap_dma_chan_read_3_3(c, CDAC);
 797
 798                 /*
 799                  * CDAC == 0 indicates that the DMA transfer on the channel
 800                  * has not been started (no data has been transferred so
 801                  * far).  Return the programmed destination start address in
 802                  * this case.
 803                  */
 804                 if (addr == 0)
 805                         addr = omap_dma_chan_read(c, CDSA);
 806         }
 807
 808         if (dma_omap1())
 809                 addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000;
 810
 811         return addr;
 812 }
 813
 814 static enum dma_status omap_dma_tx_status(struct dma_chan *chan,
 815         dma_cookie_t cookie, struct dma_tx_state *txstate)
 816 {
 817         struct omap_chan *c = to_omap_dma_chan(chan);
 818         struct virt_dma_desc *vd;
 819         enum dma_status ret;
 820         unsigned long flags;
 821
 822         ret = dma_cookie_status(chan, cookie, txstate);
 823
 824         if (!c->paused && c->running) {
 825                 uint32_t ccr = omap_dma_chan_read(c, CCR);
 826                 /*
 827                  * The channel is no longer active, set the return value
 828                  * accordingly
 829                  */
 830                 if (!(ccr & CCR_ENABLE))
 831                         ret = DMA_COMPLETE;
 832         }
 833
 834         if (ret == DMA_COMPLETE || !txstate)
 835                 return ret;
 836
 837         spin_lock_irqsave(&c->vc.lock, flags);
 838         vd = vchan_find_desc(&c->vc, cookie);
 839         if (vd) {
 840                 txstate->residue = omap_dma_desc_size(to_omap_dma_desc(&vd->tx));
 841         } else if (c->desc && c->desc->vd.tx.cookie == cookie) {
 842                 struct omap_desc *d = c->desc;
 843                 dma_addr_t pos;
 844
 845                 if (d->dir == DMA_MEM_TO_DEV)
 846                         pos = omap_dma_get_src_pos(c);
 847                 else if (d->dir == DMA_DEV_TO_MEM  || d->dir == DMA_MEM_TO_MEM)
 848                         pos = omap_dma_get_dst_pos(c);
 849                 else
 850                         pos = 0;
 851
 852                 txstate->residue = omap_dma_desc_size_pos(d, pos);
 853         } else {
 854                 txstate->residue = 0;
 855         }
 856         if (ret == DMA_IN_PROGRESS && c->paused)
 857                 ret = DMA_PAUSED;
 858         spin_unlock_irqrestore(&c->vc.lock, flags);
 859
 860         return ret;
 861 }
 862
 863 static void omap_dma_issue_pending(struct dma_chan *chan)
 864 {
 865         struct omap_chan *c = to_omap_dma_chan(chan);
 866         unsigned long flags;
 867
 868         spin_lock_irqsave(&c->vc.lock, flags);
 869         if (vchan_issue_pending(&c->vc) && !c->desc)
 870                 omap_dma_start_desc(c);
 871         spin_unlock_irqrestore(&c->vc.lock, flags);
 872 }
 873
 874 static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
 875         struct dma_chan *chan, struct scatterlist *sgl, unsigned sglen,
 876         enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
 877 {
 878         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 879         struct omap_chan *c = to_omap_dma_chan(chan);
 880         enum dma_slave_buswidth dev_width;
 881         struct scatterlist *sgent;
 882         struct omap_desc *d;
 883         dma_addr_t dev_addr;
 884         unsigned i, es, en, frame_bytes;
 885         bool ll_failed = false;
 886         u32 burst;
 887         u32 port_window, port_window_bytes;
 888
 889         if (dir == DMA_DEV_TO_MEM) {
 890                 dev_addr = c->cfg.src_addr;
 891                 dev_width = c->cfg.src_addr_width;
 892                 burst = c->cfg.src_maxburst;
 893                 port_window = c->cfg.src_port_window_size;
 894         } else if (dir == DMA_MEM_TO_DEV) {
 895                 dev_addr = c->cfg.dst_addr;
 896                 dev_width = c->cfg.dst_addr_width;
 897                 burst = c->cfg.dst_maxburst;
 898                 port_window = c->cfg.dst_port_window_size;
 899         } else {
 900                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
 901                 return NULL;
 902         }
 903
 904         /* Bus width translates to the element size (ES) */
 905         switch (dev_width) {
 906         case DMA_SLAVE_BUSWIDTH_1_BYTE:
 907                 es = CSDP_DATA_TYPE_8;
 908                 break;
 909         case DMA_SLAVE_BUSWIDTH_2_BYTES:
 910                 es = CSDP_DATA_TYPE_16;
 911                 break;
 912         case DMA_SLAVE_BUSWIDTH_4_BYTES:
 913                 es = CSDP_DATA_TYPE_32;
 914                 break;
 915         default: /* not reached */
 916                 return NULL;
 917         }
 918
 919         /* When the port_window is used, one frame must cover the window */
 920         if (port_window) {
 921                 burst = port_window;
 922                 port_window_bytes = port_window * es_bytes[es];
 923         }
 924
 925         /* Now allocate and setup the descriptor. */
 926         d = kzalloc(sizeof(*d) + sglen * sizeof(d->sg[0]), GFP_ATOMIC);
 927         if (!d)
 928                 return NULL;
 929
 930         d->dir = dir;
 931         d->dev_addr = dev_addr;
 932         d->es = es;
 933
 934         d->ccr = c->ccr | CCR_SYNC_FRAME;
 935         if (dir == DMA_DEV_TO_MEM) {
 936                 d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED;
 937
 938                 d->ccr |= CCR_DST_AMODE_POSTINC;
 939                 if (port_window) {
 940                         d->ccr |= CCR_SRC_AMODE_DBLIDX;
 941                         d->ei = 1;
 942                         /*
 943                          * One frame covers the port_window and by  configure
 944                          * the source frame index to be -1 * (port_window - 1)
 945                          * we instruct the sDMA that after a frame is processed
 946                          * it should move back to the start of the window.
 947                          */
 948                         d->fi = -(port_window_bytes - 1);
 949
 950                         if (port_window_bytes >= 64)
 951                                 d->csdp = CSDP_SRC_BURST_64 | CSDP_SRC_PACKED;
 952                         else if (port_window_bytes >= 32)
 953                                 d->csdp = CSDP_SRC_BURST_32 | CSDP_SRC_PACKED;
 954                         else if (port_window_bytes >= 16)
 955                                 d->csdp = CSDP_SRC_BURST_16 | CSDP_SRC_PACKED;
 956                 } else {
 957                         d->ccr |= CCR_SRC_AMODE_CONSTANT;
 958                 }
 959         } else {
 960                 d->csdp = CSDP_SRC_BURST_64 | CSDP_SRC_PACKED;
 961
 962                 d->ccr |= CCR_SRC_AMODE_POSTINC;
 963                 if (port_window) {
 964                         d->ccr |= CCR_DST_AMODE_DBLIDX;
 965
 966                         if (port_window_bytes >= 64)
 967                                 d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED;
 968                         else if (port_window_bytes >= 32)
 969                                 d->csdp = CSDP_DST_BURST_32 | CSDP_DST_PACKED;
 970                         else if (port_window_bytes >= 16)
 971                                 d->csdp = CSDP_DST_BURST_16 | CSDP_DST_PACKED;
 972                 } else {
 973                         d->ccr |= CCR_DST_AMODE_CONSTANT;
 974                 }
 975         }
 976
 977         d->cicr = CICR_DROP_IE | CICR_BLOCK_IE;
 978         d->csdp |= es;
 979
 980         if (dma_omap1()) {
 981                 d->cicr |= CICR_TOUT_IE;
 982
 983                 if (dir == DMA_DEV_TO_MEM)
 984                         d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_TIPB;
 985                 else
 986                         d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF;
 987         } else {
 988                 if (dir == DMA_DEV_TO_MEM)
 989                         d->ccr |= CCR_TRIGGER_SRC;
 990
 991                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
 992
 993                 if (port_window)
 994                         d->csdp |= CSDP_WRITE_LAST_NON_POSTED;
 995         }
 996         if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS)
 997                 d->clnk_ctrl = c->dma_ch;
 998
 999         /*
1000          * Build our scatterlist entries: each contains the address,
1001          * the number of elements (EN) in each frame, and the number of
1002          * frames (FN).  Number of bytes for this entry = ES * EN * FN.
1003          *
1004          * Burst size translates to number of elements with frame sync.
1005          * Note: DMA engine defines burst to be the number of dev-width
1006          * transfers.
1007          */
1008         en = burst;
1009         frame_bytes = es_bytes[es] * en;
1010
1011         if (sglen >= 2)
1012                 d->using_ll = od->ll123_supported;
1013
1014         for_each_sg(sgl, sgent, sglen, i) {
1015                 struct omap_sg *osg = &d->sg[i];
1016
1017                 osg->addr = sg_dma_address(sgent);
1018                 osg->en = en;
1019                 osg->fn = sg_dma_len(sgent) / frame_bytes;
1020                 if (port_window && dir == DMA_MEM_TO_DEV) {
1021                         osg->ei = 1;
1022                         /*
1023                          * One frame covers the port_window and by  configure
1024                          * the source frame index to be -1 * (port_window - 1)
1025                          * we instruct the sDMA that after a frame is processed
1026                          * it should move back to the start of the window.
1027                          */
1028                         osg->fi = -(port_window_bytes - 1);
1029                 }
1030
1031                 if (d->using_ll) {
1032                         osg->t2_desc = dma_pool_alloc(od->desc_pool, GFP_ATOMIC,
1033                                                       &osg->t2_desc_paddr);
1034                         if (!osg->t2_desc) {
1035                                 dev_err(chan->device->dev,
1036                                         "t2_desc[%d] allocation failed\n", i);
1037                                 ll_failed = true;
1038                                 d->using_ll = false;
1039                                 continue;
1040                         }
1041
1042                         omap_dma_fill_type2_desc(d, i, dir, (i == sglen - 1));
1043                 }
1044         }
1045
1046         d->sglen = sglen;
1047
1048         /* Release the dma_pool entries if one allocation failed */
1049         if (ll_failed) {
1050                 for (i = 0; i < d->sglen; i++) {
1051                         struct omap_sg *osg = &d->sg[i];
1052
1053                         if (osg->t2_desc) {
1054                                 dma_pool_free(od->desc_pool, osg->t2_desc,
1055                                               osg->t2_desc_paddr);
1056                                 osg->t2_desc = NULL;
1057                         }
1058                 }
1059         }
1060
1061         return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
1062 }
1063
1064 static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic(
1065         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
1066         size_t period_len, enum dma_transfer_direction dir, unsigned long flags)
1067 {
1068         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1069         struct omap_chan *c = to_omap_dma_chan(chan);
1070         enum dma_slave_buswidth dev_width;
1071         struct omap_desc *d;
1072         dma_addr_t dev_addr;
1073         unsigned es;
1074         u32 burst;
1075
1076         if (dir == DMA_DEV_TO_MEM) {
1077                 dev_addr = c->cfg.src_addr;
1078                 dev_width = c->cfg.src_addr_width;
1079                 burst = c->cfg.src_maxburst;
1080         } else if (dir == DMA_MEM_TO_DEV) {
1081                 dev_addr = c->cfg.dst_addr;
1082                 dev_width = c->cfg.dst_addr_width;
1083                 burst = c->cfg.dst_maxburst;
1084         } else {
1085                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
1086                 return NULL;
1087         }
1088
1089         /* Bus width translates to the element size (ES) */
1090         switch (dev_width) {
1091         case DMA_SLAVE_BUSWIDTH_1_BYTE:
1092                 es = CSDP_DATA_TYPE_8;
1093                 break;
1094         case DMA_SLAVE_BUSWIDTH_2_BYTES:
1095                 es = CSDP_DATA_TYPE_16;
1096                 break;
1097         case DMA_SLAVE_BUSWIDTH_4_BYTES:
1098                 es = CSDP_DATA_TYPE_32;
1099                 break;
1100         default: /* not reached */
1101                 return NULL;
1102         }
1103
1104         /* Now allocate and setup the descriptor. */
1105         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1106         if (!d)
1107                 return NULL;
1108
1109         d->dir = dir;
1110         d->dev_addr = dev_addr;
1111         d->fi = burst;
1112         d->es = es;
1113         d->sg[0].addr = buf_addr;
1114         d->sg[0].en = period_len / es_bytes[es];
1115         d->sg[0].fn = buf_len / period_len;
1116         d->sglen = 1;
1117
1118         d->ccr = c->ccr;
1119         if (dir == DMA_DEV_TO_MEM)
1120                 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT;
1121         else
1122                 d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC;
1123
1124         d->cicr = CICR_DROP_IE;
1125         if (flags & DMA_PREP_INTERRUPT)
1126                 d->cicr |= CICR_FRAME_IE;
1127
1128         d->csdp = es;
1129
1130         if (dma_omap1()) {
1131                 d->cicr |= CICR_TOUT_IE;
1132
1133                 if (dir == DMA_DEV_TO_MEM)
1134                         d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_MPUI;
1135                 else
1136                         d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF;
1137         } else {
1138                 if (burst)
1139                         d->ccr |= CCR_SYNC_PACKET;
1140                 else
1141                         d->ccr |= CCR_SYNC_ELEMENT;
1142
1143                 if (dir == DMA_DEV_TO_MEM) {
1144                         d->ccr |= CCR_TRIGGER_SRC;
1145                         d->csdp |= CSDP_DST_PACKED;
1146                 } else {
1147                         d->csdp |= CSDP_SRC_PACKED;
1148                 }
1149
1150                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1151
1152                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1153         }
1154
1155         if (__dma_omap15xx(od->plat->dma_attr))
1156                 d->ccr |= CCR_AUTO_INIT | CCR_REPEAT;
1157         else
1158                 d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK;
1159
1160         c->cyclic = true;
1161
1162         return vchan_tx_prep(&c->vc, &d->vd, flags);
1163 }
1164
1165 static struct dma_async_tx_descriptor *omap_dma_prep_dma_memcpy(
1166         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1167         size_t len, unsigned long tx_flags)
1168 {
1169         struct omap_chan *c = to_omap_dma_chan(chan);
1170         struct omap_desc *d;
1171         uint8_t data_type;
1172
1173         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1174         if (!d)
1175                 return NULL;
1176
1177         data_type = __ffs((src | dest | len));
1178         if (data_type > CSDP_DATA_TYPE_32)
1179                 data_type = CSDP_DATA_TYPE_32;
1180
1181         d->dir = DMA_MEM_TO_MEM;
1182         d->dev_addr = src;
1183         d->fi = 0;
1184         d->es = data_type;
1185         d->sg[0].en = len / BIT(data_type);
1186         d->sg[0].fn = 1;
1187         d->sg[0].addr = dest;
1188         d->sglen = 1;
1189         d->ccr = c->ccr;
1190         d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC;
1191
1192         d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
1193
1194         d->csdp = data_type;
1195
1196         if (dma_omap1()) {
1197                 d->cicr |= CICR_TOUT_IE;
1198                 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1199         } else {
1200                 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1201                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1202                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1203         }
1204
1205         return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
1206 }
1207
1208 static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved(
1209         struct dma_chan *chan, struct dma_interleaved_template *xt,
1210         unsigned long flags)
1211 {
1212         struct omap_chan *c = to_omap_dma_chan(chan);
1213         struct omap_desc *d;
1214         struct omap_sg *sg;
1215         uint8_t data_type;
1216         size_t src_icg, dst_icg;
1217
1218         /* Slave mode is not supported */
1219         if (is_slave_direction(xt->dir))
1220                 return NULL;
1221
1222         if (xt->frame_size != 1 || xt->numf == 0)
1223                 return NULL;
1224
1225         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1226         if (!d)
1227                 return NULL;
1228
1229         data_type = __ffs((xt->src_start | xt->dst_start | xt->sgl[0].size));
1230         if (data_type > CSDP_DATA_TYPE_32)
1231                 data_type = CSDP_DATA_TYPE_32;
1232
1233         sg = &d->sg[0];
1234         d->dir = DMA_MEM_TO_MEM;
1235         d->dev_addr = xt->src_start;
1236         d->es = data_type;
1237         sg->en = xt->sgl[0].size / BIT(data_type);
1238         sg->fn = xt->numf;
1239         sg->addr = xt->dst_start;
1240         d->sglen = 1;
1241         d->ccr = c->ccr;
1242
1243         src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
1244         dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
1245         if (src_icg) {
1246                 d->ccr |= CCR_SRC_AMODE_DBLIDX;
1247                 d->ei = 1;
1248                 d->fi = src_icg;
1249         } else if (xt->src_inc) {
1250                 d->ccr |= CCR_SRC_AMODE_POSTINC;
1251                 d->fi = 0;
1252         } else {
1253                 dev_err(chan->device->dev,
1254                         "%s: SRC constant addressing is not supported\n",
1255                         __func__);
1256                 kfree(d);
1257                 return NULL;
1258         }
1259
1260         if (dst_icg) {
1261                 d->ccr |= CCR_DST_AMODE_DBLIDX;
1262                 sg->ei = 1;
1263                 sg->fi = dst_icg;
1264         } else if (xt->dst_inc) {
1265                 d->ccr |= CCR_DST_AMODE_POSTINC;
1266                 sg->fi = 0;
1267         } else {
1268                 dev_err(chan->device->dev,
1269                         "%s: DST constant addressing is not supported\n",
1270                         __func__);
1271                 kfree(d);
1272                 return NULL;
1273         }
1274
1275         d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
1276
1277         d->csdp = data_type;
1278
1279         if (dma_omap1()) {
1280                 d->cicr |= CICR_TOUT_IE;
1281                 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1282         } else {
1283                 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1284                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1285                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1286         }
1287
1288         return vchan_tx_prep(&c->vc, &d->vd, flags);
1289 }
1290
1291 static int omap_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg)
1292 {
1293         struct omap_chan *c = to_omap_dma_chan(chan);
1294
1295         if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
1296             cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
1297                 return -EINVAL;
1298
1299         memcpy(&c->cfg, cfg, sizeof(c->cfg));
1300
1301         return 0;
1302 }
1303
1304 static int omap_dma_terminate_all(struct dma_chan *chan)
1305 {
1306         struct omap_chan *c = to_omap_dma_chan(chan);
1307         unsigned long flags;
1308         LIST_HEAD(head);
1309
1310         spin_lock_irqsave(&c->vc.lock, flags);
1311
1312         /*
1313          * Stop DMA activity: we assume the callback will not be called
1314          * after omap_dma_stop() returns (even if it does, it will see
1315          * c->desc is NULL and exit.)
1316          */
1317         if (c->desc) {
1318                 omap_dma_desc_free(&c->desc->vd);
1319                 c->desc = NULL;
1320                 /* Avoid stopping the dma twice */
1321                 if (!c->paused)
1322                         omap_dma_stop(c);
1323         }
1324
1325         c->cyclic = false;
1326         c->paused = false;
1327
1328         vchan_get_all_descriptors(&c->vc, &head);
1329         spin_unlock_irqrestore(&c->vc.lock, flags);
1330         vchan_dma_desc_free_list(&c->vc, &head);
1331
1332         return 0;
1333 }
1334
1335 static void omap_dma_synchronize(struct dma_chan *chan)
1336 {
1337         struct omap_chan *c = to_omap_dma_chan(chan);
1338
1339         vchan_synchronize(&c->vc);
1340 }
1341
1342 static int omap_dma_pause(struct dma_chan *chan)
1343 {
1344         struct omap_chan *c = to_omap_dma_chan(chan);
1345         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1346         unsigned long flags;
1347         int ret = -EINVAL;
1348         bool can_pause = false;
1349
1350         spin_lock_irqsave(&od->irq_lock, flags);
1351
1352         if (!c->desc)
1353                 goto out;
1354
1355         if (c->cyclic)
1356                 can_pause = true;
1357
1358         /*
1359          * We do not allow DMA_MEM_TO_DEV transfers to be paused.
1360          * From the AM572x TRM, 16.1.4.18 Disabling a Channel During Transfer:
1361          * "When a channel is disabled during a transfer, the channel undergoes
1362          * an abort, unless it is hardware-source-synchronized …".
1363          * A source-synchronised channel is one where the fetching of data is
1364          * under control of the device. In other words, a device-to-memory
1365          * transfer. So, a destination-synchronised channel (which would be a
1366          * memory-to-device transfer) undergoes an abort if the the CCR_ENABLE
1367          * bit is cleared.
1368          * From 16.1.4.20.4.6.2 Abort: "If an abort trigger occurs, the channel
1369          * aborts immediately after completion of current read/write
1370          * transactions and then the FIFO is cleaned up." The term "cleaned up"
1371          * is not defined. TI recommends to check that RD_ACTIVE and WR_ACTIVE
1372          * are both clear _before_ disabling the channel, otherwise data loss
1373          * will occur.
1374          * The problem is that if the channel is active, then device activity
1375          * can result in DMA activity starting between reading those as both
1376          * clear and the write to DMA_CCR to clear the enable bit hitting the
1377          * hardware. If the DMA hardware can't drain the data in its FIFO to the
1378          * destination, then data loss "might" occur (say if we write to an UART
1379          * and the UART is not accepting any further data).
1380          */
1381         else if (c->desc->dir == DMA_DEV_TO_MEM)
1382                 can_pause = true;
1383
1384         if (can_pause && !c->paused) {
1385                 ret = omap_dma_stop(c);
1386                 if (!ret)
1387                         c->paused = true;
1388         }
1389 out:
1390         spin_unlock_irqrestore(&od->irq_lock, flags);
1391
1392         return ret;
1393 }
1394
1395 static int omap_dma_resume(struct dma_chan *chan)
1396 {
1397         struct omap_chan *c = to_omap_dma_chan(chan);
1398         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1399         unsigned long flags;
1400         int ret = -EINVAL;
1401
1402         spin_lock_irqsave(&od->irq_lock, flags);
1403
1404         if (c->paused && c->desc) {
1405                 mb();
1406
1407                 /* Restore channel link register */
1408                 omap_dma_chan_write(c, CLNK_CTRL, c->desc->clnk_ctrl);
1409
1410                 omap_dma_start(c, c->desc);
1411                 c->paused = false;
1412                 ret = 0;
1413         }
1414         spin_unlock_irqrestore(&od->irq_lock, flags);
1415
1416         return ret;
1417 }
1418
1419 static int omap_dma_chan_init(struct omap_dmadev *od)
1420 {
1421         struct omap_chan *c;
1422
1423         c = kzalloc(sizeof(*c), GFP_KERNEL);
1424         if (!c)
1425                 return -ENOMEM;
1426
1427         c->reg_map = od->reg_map;
1428         c->vc.desc_free = omap_dma_desc_free;
1429         vchan_init(&c->vc, &od->ddev);
1430
1431         return 0;
1432 }
1433
1434 static void omap_dma_free(struct omap_dmadev *od)
1435 {
1436         while (!list_empty(&od->ddev.channels)) {
1437                 struct omap_chan *c = list_first_entry(&od->ddev.channels,
1438                         struct omap_chan, vc.chan.device_node);
1439
1440                 list_del(&c->vc.chan.device_node);
1441                 tasklet_kill(&c->vc.task);
1442                 kfree(c);
1443         }
1444 }
1445
1446 #define OMAP_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
1447                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
1448                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
1449
1450 static int omap_dma_probe(struct platform_device *pdev)
1451 {
1452         struct omap_dmadev *od;
1453         struct resource *res;
1454         int rc, i, irq;
1455
1456         od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
1457         if (!od)
1458                 return -ENOMEM;
1459
1460         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1461         od->base = devm_ioremap_resource(&pdev->dev, res);
1462         if (IS_ERR(od->base))
1463                 return PTR_ERR(od->base);
1464
1465         od->plat = omap_get_plat_info();
1466         if (!od->plat)
1467                 return -EPROBE_DEFER;
1468
1469         od->reg_map = od->plat->reg_map;
1470
1471         dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
1472         dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);
1473         dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);
1474         dma_cap_set(DMA_INTERLEAVE, od->ddev.cap_mask);
1475         od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources;
1476         od->ddev.device_free_chan_resources = omap_dma_free_chan_resources;
1477         od->ddev.device_tx_status = omap_dma_tx_status;
1478         od->ddev.device_issue_pending = omap_dma_issue_pending;
1479         od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg;
1480         od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic;
1481         od->ddev.device_prep_dma_memcpy = omap_dma_prep_dma_memcpy;
1482         od->ddev.device_prep_interleaved_dma = omap_dma_prep_dma_interleaved;
1483         od->ddev.device_config = omap_dma_slave_config;
1484         od->ddev.device_pause = omap_dma_pause;
1485         od->ddev.device_resume = omap_dma_resume;
1486         od->ddev.device_terminate_all = omap_dma_terminate_all;
1487         od->ddev.device_synchronize = omap_dma_synchronize;
1488         od->ddev.src_addr_widths = OMAP_DMA_BUSWIDTHS;
1489         od->ddev.dst_addr_widths = OMAP_DMA_BUSWIDTHS;
1490         od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1491         od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1492         od->ddev.dev = &pdev->dev;
1493         INIT_LIST_HEAD(&od->ddev.channels);
1494         spin_lock_init(&od->lock);
1495         spin_lock_init(&od->irq_lock);
1496
1497         if (!pdev->dev.of_node) {
1498                 od->dma_requests = od->plat->dma_attr->lch_count;
1499                 if (unlikely(!od->dma_requests))
1500                         od->dma_requests = OMAP_SDMA_REQUESTS;
1501         } else if (of_property_read_u32(pdev->dev.of_node, "dma-requests",
1502                                         &od->dma_requests)) {
1503                 dev_info(&pdev->dev,
1504                          "Missing dma-requests property, using %u.\n",
1505                          OMAP_SDMA_REQUESTS);
1506                 od->dma_requests = OMAP_SDMA_REQUESTS;
1507         }
1508
1509         od->lch_map = devm_kcalloc(&pdev->dev, od->dma_requests,
1510                                    sizeof(*od->lch_map), GFP_KERNEL);
1511         if (!od->lch_map)
1512                 return -ENOMEM;
1513
1514         for (i = 0; i < od->dma_requests; i++) {
1515                 rc = omap_dma_chan_init(od);
1516                 if (rc) {
1517                         omap_dma_free(od);
1518                         return rc;
1519                 }
1520         }
1521
1522         irq = platform_get_irq(pdev, 1);
1523         if (irq <= 0) {
1524                 dev_info(&pdev->dev, "failed to get L1 IRQ: %d\n", irq);
1525                 od->legacy = true;
1526         } else {
1527                 /* Disable all interrupts */
1528                 od->irq_enable_mask = 0;
1529                 omap_dma_glbl_write(od, IRQENABLE_L1, 0);
1530
1531                 rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq,
1532                                       IRQF_SHARED, "omap-dma-engine", od);
1533                 if (rc)
1534                         return rc;
1535         }
1536
1537         if (omap_dma_glbl_read(od, CAPS_0) & CAPS_0_SUPPORT_LL123)
1538                 od->ll123_supported = true;
1539
1540         od->ddev.filter.map = od->plat->slave_map;
1541         od->ddev.filter.mapcnt = od->plat->slavecnt;
1542         od->ddev.filter.fn = omap_dma_filter_fn;
1543
1544         if (od->ll123_supported) {
1545                 od->desc_pool = dma_pool_create(dev_name(&pdev->dev),
1546                                                 &pdev->dev,
1547                                                 sizeof(struct omap_type2_desc),
1548                                                 4, 0);
1549                 if (!od->desc_pool) {
1550                         dev_err(&pdev->dev,
1551                                 "unable to allocate descriptor pool\n");
1552                         od->ll123_supported = false;
1553                 }
1554         }
1555
1556         rc = dma_async_device_register(&od->ddev);
1557         if (rc) {
1558                 pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n",
1559                         rc);
1560                 omap_dma_free(od);
1561                 return rc;
1562         }
1563
1564         platform_set_drvdata(pdev, od);
1565
1566         if (pdev->dev.of_node) {
1567                 omap_dma_info.dma_cap = od->ddev.cap_mask;
1568
1569                 /* Device-tree DMA controller registration */
1570                 rc = of_dma_controller_register(pdev->dev.of_node,
1571                                 of_dma_simple_xlate, &omap_dma_info);
1572                 if (rc) {
1573                         pr_warn("OMAP-DMA: failed to register DMA controller\n");
1574                         dma_async_device_unregister(&od->ddev);
1575                         omap_dma_free(od);
1576                 }
1577         }
1578
1579         dev_info(&pdev->dev, "OMAP DMA engine driver%s\n",
1580                  od->ll123_supported ? " (LinkedList1/2/3 supported)" : "");
1581
1582         return rc;
1583 }
1584
1585 static int omap_dma_remove(struct platform_device *pdev)
1586 {
1587         struct omap_dmadev *od = platform_get_drvdata(pdev);
1588         int irq;
1589
1590         if (pdev->dev.of_node)
1591                 of_dma_controller_free(pdev->dev.of_node);
1592
1593         irq = platform_get_irq(pdev, 1);
1594         devm_free_irq(&pdev->dev, irq, od);
1595
1596         dma_async_device_unregister(&od->ddev);
1597
1598         if (!od->legacy) {
1599                 /* Disable all interrupts */
1600                 omap_dma_glbl_write(od, IRQENABLE_L0, 0);
1601         }
1602
1603         if (od->ll123_supported)
1604                 dma_pool_destroy(od->desc_pool);
1605
1606         omap_dma_free(od);
1607
1608         return 0;
1609 }
1610
1611 static const struct of_device_id omap_dma_match[] = {
1612         { .compatible = "ti,omap2420-sdma", },
1613         { .compatible = "ti,omap2430-sdma", },
1614         { .compatible = "ti,omap3430-sdma", },
1615         { .compatible = "ti,omap3630-sdma", },
1616         { .compatible = "ti,omap4430-sdma", },
1617         {},
1618 };
1619 MODULE_DEVICE_TABLE(of, omap_dma_match);
1620
1621 static struct platform_driver omap_dma_driver = {
1622         .probe  = omap_dma_probe,
1623         .remove = omap_dma_remove,
1624         .driver = {
1625                 .name = "omap-dma-engine",
1626                 .of_match_table = of_match_ptr(omap_dma_match),
1627         },
1628 };
1629
1630 bool omap_dma_filter_fn(struct dma_chan *chan, void *param)
1631 {
1632         if (chan->device->dev->driver == &omap_dma_driver.driver) {
1633                 struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1634                 struct omap_chan *c = to_omap_dma_chan(chan);
1635                 unsigned req = *(unsigned *)param;
1636
1637                 if (req <= od->dma_requests) {
1638                         c->dma_sig = req;
1639                         return true;
1640                 }
1641         }
1642         return false;
1643 }
1644 EXPORT_SYMBOL_GPL(omap_dma_filter_fn);
1645
1646 static int omap_dma_init(void)
1647 {
1648         return platform_driver_register(&omap_dma_driver);
1649 }
1650 subsys_initcall(omap_dma_init);
1651
1652 static void __exit omap_dma_exit(void)
1653 {
1654         platform_driver_unregister(&omap_dma_driver);
1655 }
1656 module_exit(omap_dma_exit);
1657
1658 MODULE_AUTHOR("Russell King");
1659 MODULE_LICENSE("GPL");