arch/arm/plat-omap/dma.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * linux/arch/arm/plat-omap/dma.c
   4  *
   5  * Copyright (C) 2003 - 2008 Nokia Corporation
   6  * Author: Juha Yrjölä <juha.yrjola@nokia.com>
   7  * DMA channel linking for 1610 by Samuel Ortiz <samuel.ortiz@nokia.com>
   8  * Graphics DMA and LCD DMA graphics tranformations
   9  * by Imre Deak <imre.deak@nokia.com>
  10  * OMAP2/3 support Copyright (C) 2004-2007 Texas Instruments, Inc.
  11  * Merged to support both OMAP1 and OMAP2 by Tony Lindgren <tony@atomide.com>
  12  * Some functions based on earlier dma-omap.c Copyright (C) 2001 RidgeRun, Inc.
  13  *
  14  * Copyright (C) 2009 Texas Instruments
  15  * Added OMAP4 support - Santosh Shilimkar <santosh.shilimkar@ti.com>
  16  *
  17  * Support functions for the OMAP internal DMA channels.
  18  *
  19  * Copyright (C) 2010 Texas Instruments Incorporated - https://www.ti.com/
  20  * Converted DMA library into DMA platform driver.
  21  *      - G, Manjunath Kondaiah <manjugk@ti.com>
  22  */
  23
  24 #include <linux/module.h>
  25 #include <linux/init.h>
  26 #include <linux/sched.h>
  27 #include <linux/spinlock.h>
  28 #include <linux/errno.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/irq.h>
  31 #include <linux/io.h>
  32 #include <linux/slab.h>
  33 #include <linux/delay.h>
  34
  35 #include <linux/omap-dma.h>
  36
  37 #include <mach/hardware.h>
  38 #include <linux/soc/ti/omap1-io.h>
  39 #include <linux/soc/ti/omap1-soc.h>
  40
  41 /*
  42  * MAX_LOGICAL_DMA_CH_COUNT: the maximum number of logical DMA
  43  * channels that an instance of the SDMA IP block can support.  Used
  44  * to size arrays.  (The actual maximum on a particular SoC may be less
  45  * than this -- for example, OMAP1 SDMA instances only support 17 logical
  46  * DMA channels.)
  47  */
  48 #define MAX_LOGICAL_DMA_CH_COUNT                32
  49
  50 #undef DEBUG
  51
  52 #define OMAP_DMA_ACTIVE                 0x01
  53
  54 #define OMAP_FUNC_MUX_ARM_BASE          (0xfffe1000 + 0xec)
  55
  56 static struct omap_system_dma_plat_info *p;
  57 static struct omap_dma_dev_attr *d;
  58 static int enable_1510_mode;
  59 static u32 errata;
  60
  61 struct dma_link_info {
  62         int *linked_dmach_q;
  63         int no_of_lchs_linked;
  64
  65         int q_count;
  66         int q_tail;
  67         int q_head;
  68
  69         int chain_state;
  70         int chain_mode;
  71
  72 };
  73
  74 static int dma_lch_count;
  75 static int dma_chan_count;
  76 static int omap_dma_reserve_channels;
  77
  78 static DEFINE_SPINLOCK(dma_chan_lock);
  79 static struct omap_dma_lch *dma_chan;
  80
  81 static inline void omap_disable_channel_irq(int lch)
  82 {
  83         /* disable channel interrupts */
  84         p->dma_write(0, CICR, lch);
  85         /* Clear CSR */
  86         p->dma_read(CSR, lch);
  87 }
  88
  89 static inline void set_gdma_dev(int req, int dev)
  90 {
  91         u32 reg = OMAP_FUNC_MUX_ARM_BASE + ((req - 1) / 5) * 4;
  92         int shift = ((req - 1) % 5) * 6;
  93         u32 l;
  94
  95         l = omap_readl(reg);
  96         l &= ~(0x3f << shift);
  97         l |= (dev - 1) << shift;
  98         omap_writel(l, reg);
  99 }
 100
 101 #ifdef CONFIG_ARCH_OMAP1
 102 void omap_set_dma_priority(int lch, int dst_port, int priority)
 103 {
 104         unsigned long reg;
 105         u32 l;
 106
 107         if (dma_omap1()) {
 108                 switch (dst_port) {
 109                 case OMAP_DMA_PORT_OCP_T1:      /* FFFECC00 */
 110                         reg = OMAP_TC_OCPT1_PRIOR;
 111                         break;
 112                 case OMAP_DMA_PORT_OCP_T2:      /* FFFECCD0 */
 113                         reg = OMAP_TC_OCPT2_PRIOR;
 114                         break;
 115                 case OMAP_DMA_PORT_EMIFF:       /* FFFECC08 */
 116                         reg = OMAP_TC_EMIFF_PRIOR;
 117                         break;
 118                 case OMAP_DMA_PORT_EMIFS:       /* FFFECC04 */
 119                         reg = OMAP_TC_EMIFS_PRIOR;
 120                         break;
 121                 default:
 122                         BUG();
 123                         return;
 124                 }
 125                 l = omap_readl(reg);
 126                 l &= ~(0xf << 8);
 127                 l |= (priority & 0xf) << 8;
 128                 omap_writel(l, reg);
 129         }
 130 }
 131 #endif
 132
 133 #ifdef CONFIG_ARCH_OMAP2PLUS
 134 void omap_set_dma_priority(int lch, int dst_port, int priority)
 135 {
 136         u32 ccr;
 137
 138         ccr = p->dma_read(CCR, lch);
 139         if (priority)
 140                 ccr |= (1 << 6);
 141         else
 142                 ccr &= ~(1 << 6);
 143         p->dma_write(ccr, CCR, lch);
 144 }
 145 #endif
 146 EXPORT_SYMBOL(omap_set_dma_priority);
 147
 148 #if IS_ENABLED(CONFIG_USB_OMAP)
 149 #ifdef CONFIG_ARCH_OMAP15XX
 150 /* Returns 1 if the DMA module is in OMAP1510-compatible mode, 0 otherwise */
 151 static int omap_dma_in_1510_mode(void)
 152 {
 153         return enable_1510_mode;
 154 }
 155 #else
 156 #define omap_dma_in_1510_mode()         0
 157 #endif
 158
 159 void omap_set_dma_transfer_params(int lch, int data_type, int elem_count,
 160                                   int frame_count, int sync_mode,
 161                                   int dma_trigger, int src_or_dst_synch)
 162 {
 163         u32 l;
 164         u16 ccr;
 165
 166         l = p->dma_read(CSDP, lch);
 167         l &= ~0x03;
 168         l |= data_type;
 169         p->dma_write(l, CSDP, lch);
 170
 171         ccr = p->dma_read(CCR, lch);
 172         ccr &= ~(1 << 5);
 173         if (sync_mode == OMAP_DMA_SYNC_FRAME)
 174                 ccr |= 1 << 5;
 175         p->dma_write(ccr, CCR, lch);
 176
 177         ccr = p->dma_read(CCR2, lch);
 178         ccr &= ~(1 << 2);
 179         if (sync_mode == OMAP_DMA_SYNC_BLOCK)
 180                 ccr |= 1 << 2;
 181         p->dma_write(ccr, CCR2, lch);
 182         p->dma_write(elem_count, CEN, lch);
 183         p->dma_write(frame_count, CFN, lch);
 184 }
 185 EXPORT_SYMBOL(omap_set_dma_transfer_params);
 186
 187 void omap_set_dma_channel_mode(int lch, enum omap_dma_channel_mode mode)
 188 {
 189         if (!dma_omap15xx()) {
 190                 u32 l;
 191
 192                 l = p->dma_read(LCH_CTRL, lch);
 193                 l &= ~0x7;
 194                 l |= mode;
 195                 p->dma_write(l, LCH_CTRL, lch);
 196         }
 197 }
 198 EXPORT_SYMBOL(omap_set_dma_channel_mode);
 199
 200 /* Note that src_port is only for omap1 */
 201 void omap_set_dma_src_params(int lch, int src_port, int src_amode,
 202                              unsigned long src_start,
 203                              int src_ei, int src_fi)
 204 {
 205         u32 l;
 206         u16 w;
 207
 208         w = p->dma_read(CSDP, lch);
 209         w &= ~(0x1f << 2);
 210         w |= src_port << 2;
 211         p->dma_write(w, CSDP, lch);
 212
 213         l = p->dma_read(CCR, lch);
 214         l &= ~(0x03 << 12);
 215         l |= src_amode << 12;
 216         p->dma_write(l, CCR, lch);
 217
 218         p->dma_write(src_start, CSSA, lch);
 219
 220         p->dma_write(src_ei, CSEI, lch);
 221         p->dma_write(src_fi, CSFI, lch);
 222 }
 223 EXPORT_SYMBOL(omap_set_dma_src_params);
 224
 225 void omap_set_dma_src_data_pack(int lch, int enable)
 226 {
 227         u32 l;
 228
 229         l = p->dma_read(CSDP, lch);
 230         l &= ~(1 << 6);
 231         if (enable)
 232                 l |= (1 << 6);
 233         p->dma_write(l, CSDP, lch);
 234 }
 235 EXPORT_SYMBOL(omap_set_dma_src_data_pack);
 236
 237 void omap_set_dma_src_burst_mode(int lch, enum omap_dma_burst_mode burst_mode)
 238 {
 239         unsigned int burst = 0;
 240         u32 l;
 241
 242         l = p->dma_read(CSDP, lch);
 243         l &= ~(0x03 << 7);
 244
 245         switch (burst_mode) {
 246         case OMAP_DMA_DATA_BURST_DIS:
 247                 break;
 248         case OMAP_DMA_DATA_BURST_4:
 249                 burst = 0x2;
 250                 break;
 251         case OMAP_DMA_DATA_BURST_8:
 252                 /*
 253                  * not supported by current hardware on OMAP1
 254                  * w |= (0x03 << 7);
 255                  */
 256                 fallthrough;
 257         case OMAP_DMA_DATA_BURST_16:
 258                 /* OMAP1 don't support burst 16 */
 259                 fallthrough;
 260         default:
 261                 BUG();
 262         }
 263
 264         l |= (burst << 7);
 265         p->dma_write(l, CSDP, lch);
 266 }
 267 EXPORT_SYMBOL(omap_set_dma_src_burst_mode);
 268
 269 /* Note that dest_port is only for OMAP1 */
 270 void omap_set_dma_dest_params(int lch, int dest_port, int dest_amode,
 271                               unsigned long dest_start,
 272                               int dst_ei, int dst_fi)
 273 {
 274         u32 l;
 275
 276         l = p->dma_read(CSDP, lch);
 277         l &= ~(0x1f << 9);
 278         l |= dest_port << 9;
 279         p->dma_write(l, CSDP, lch);
 280
 281         l = p->dma_read(CCR, lch);
 282         l &= ~(0x03 << 14);
 283         l |= dest_amode << 14;
 284         p->dma_write(l, CCR, lch);
 285
 286         p->dma_write(dest_start, CDSA, lch);
 287
 288         p->dma_write(dst_ei, CDEI, lch);
 289         p->dma_write(dst_fi, CDFI, lch);
 290 }
 291 EXPORT_SYMBOL(omap_set_dma_dest_params);
 292
 293 void omap_set_dma_dest_data_pack(int lch, int enable)
 294 {
 295         u32 l;
 296
 297         l = p->dma_read(CSDP, lch);
 298         l &= ~(1 << 13);
 299         if (enable)
 300                 l |= 1 << 13;
 301         p->dma_write(l, CSDP, lch);
 302 }
 303 EXPORT_SYMBOL(omap_set_dma_dest_data_pack);
 304
 305 void omap_set_dma_dest_burst_mode(int lch, enum omap_dma_burst_mode burst_mode)
 306 {
 307         unsigned int burst = 0;
 308         u32 l;
 309
 310         l = p->dma_read(CSDP, lch);
 311         l &= ~(0x03 << 14);
 312
 313         switch (burst_mode) {
 314         case OMAP_DMA_DATA_BURST_DIS:
 315                 break;
 316         case OMAP_DMA_DATA_BURST_4:
 317                 burst = 0x2;
 318                 break;
 319         case OMAP_DMA_DATA_BURST_8:
 320                 burst = 0x3;
 321                 break;
 322         case OMAP_DMA_DATA_BURST_16:
 323                 /* OMAP1 don't support burst 16 */
 324                 fallthrough;
 325         default:
 326                 printk(KERN_ERR "Invalid DMA burst mode\n");
 327                 BUG();
 328                 return;
 329         }
 330         l |= (burst << 14);
 331         p->dma_write(l, CSDP, lch);
 332 }
 333 EXPORT_SYMBOL(omap_set_dma_dest_burst_mode);
 334
 335 static inline void omap_enable_channel_irq(int lch)
 336 {
 337         /* Clear CSR */
 338         p->dma_read(CSR, lch);
 339
 340         /* Enable some nice interrupts. */
 341         p->dma_write(dma_chan[lch].enabled_irqs, CICR, lch);
 342 }
 343
 344 void omap_disable_dma_irq(int lch, u16 bits)
 345 {
 346         dma_chan[lch].enabled_irqs &= ~bits;
 347 }
 348 EXPORT_SYMBOL(omap_disable_dma_irq);
 349
 350 static inline void enable_lnk(int lch)
 351 {
 352         u32 l;
 353
 354         l = p->dma_read(CLNK_CTRL, lch);
 355
 356         l &= ~(1 << 14);
 357
 358         /* Set the ENABLE_LNK bits */
 359         if (dma_chan[lch].next_lch != -1)
 360                 l = dma_chan[lch].next_lch | (1 << 15);
 361
 362         p->dma_write(l, CLNK_CTRL, lch);
 363 }
 364
 365 static inline void disable_lnk(int lch)
 366 {
 367         u32 l;
 368
 369         l = p->dma_read(CLNK_CTRL, lch);
 370
 371         /* Disable interrupts */
 372         omap_disable_channel_irq(lch);
 373
 374         /* Set the STOP_LNK bit */
 375         l |= 1 << 14;
 376
 377         p->dma_write(l, CLNK_CTRL, lch);
 378         dma_chan[lch].flags &= ~OMAP_DMA_ACTIVE;
 379 }
 380 #endif
 381
 382 int omap_request_dma(int dev_id, const char *dev_name,
 383                      void (*callback)(int lch, u16 ch_status, void *data),
 384                      void *data, int *dma_ch_out)
 385 {
 386         int ch, free_ch = -1;
 387         unsigned long flags;
 388         struct omap_dma_lch *chan;
 389
 390         WARN(strcmp(dev_name, "DMA engine"), "Using deprecated platform DMA API - please update to DMA engine");
 391
 392         spin_lock_irqsave(&dma_chan_lock, flags);
 393         for (ch = 0; ch < dma_chan_count; ch++) {
 394                 if (free_ch == -1 && dma_chan[ch].dev_id == -1) {
 395                         free_ch = ch;
 396                         /* Exit after first free channel found */
 397                         break;
 398                 }
 399         }
 400         if (free_ch == -1) {
 401                 spin_unlock_irqrestore(&dma_chan_lock, flags);
 402                 return -EBUSY;
 403         }
 404         chan = dma_chan + free_ch;
 405         chan->dev_id = dev_id;
 406
 407         if (p->clear_lch_regs)
 408                 p->clear_lch_regs(free_ch);
 409
 410         spin_unlock_irqrestore(&dma_chan_lock, flags);
 411
 412         chan->dev_name = dev_name;
 413         chan->callback = callback;
 414         chan->data = data;
 415         chan->flags = 0;
 416
 417         chan->enabled_irqs = OMAP_DMA_DROP_IRQ | OMAP_DMA_BLOCK_IRQ;
 418
 419         chan->enabled_irqs |= OMAP1_DMA_TOUT_IRQ;
 420
 421         if (dma_omap16xx()) {
 422                 /* If the sync device is set, configure it dynamically. */
 423                 if (dev_id != 0) {
 424                         set_gdma_dev(free_ch + 1, dev_id);
 425                         dev_id = free_ch + 1;
 426                 }
 427                 /*
 428                  * Disable the 1510 compatibility mode and set the sync device
 429                  * id.
 430                  */
 431                 p->dma_write(dev_id | (1 << 10), CCR, free_ch);
 432         } else {
 433                 p->dma_write(dev_id, CCR, free_ch);
 434         }
 435
 436         *dma_ch_out = free_ch;
 437
 438         return 0;
 439 }
 440 EXPORT_SYMBOL(omap_request_dma);
 441
 442 void omap_free_dma(int lch)
 443 {
 444         unsigned long flags;
 445
 446         if (dma_chan[lch].dev_id == -1) {
 447                 pr_err("omap_dma: trying to free unallocated DMA channel %d\n",
 448                        lch);
 449                 return;
 450         }
 451
 452         /* Disable all DMA interrupts for the channel. */
 453         omap_disable_channel_irq(lch);
 454
 455         /* Make sure the DMA transfer is stopped. */
 456         p->dma_write(0, CCR, lch);
 457
 458         spin_lock_irqsave(&dma_chan_lock, flags);
 459         dma_chan[lch].dev_id = -1;
 460         dma_chan[lch].next_lch = -1;
 461         dma_chan[lch].callback = NULL;
 462         spin_unlock_irqrestore(&dma_chan_lock, flags);
 463 }
 464 EXPORT_SYMBOL(omap_free_dma);
 465
 466 /*
 467  * Clears any DMA state so the DMA engine is ready to restart with new buffers
 468  * through omap_start_dma(). Any buffers in flight are discarded.
 469  */
 470 static void omap_clear_dma(int lch)
 471 {
 472         unsigned long flags;
 473
 474         local_irq_save(flags);
 475         p->clear_dma(lch);
 476         local_irq_restore(flags);
 477 }
 478
 479 #if IS_ENABLED(CONFIG_USB_OMAP)
 480 void omap_start_dma(int lch)
 481 {
 482         u32 l;
 483
 484         /*
 485          * The CPC/CDAC register needs to be initialized to zero
 486          * before starting dma transfer.
 487          */
 488         if (dma_omap15xx())
 489                 p->dma_write(0, CPC, lch);
 490         else
 491                 p->dma_write(0, CDAC, lch);
 492
 493         if (!omap_dma_in_1510_mode() && dma_chan[lch].next_lch != -1) {
 494                 int next_lch, cur_lch;
 495                 char dma_chan_link_map[MAX_LOGICAL_DMA_CH_COUNT];
 496
 497                 /* Set the link register of the first channel */
 498                 enable_lnk(lch);
 499
 500                 memset(dma_chan_link_map, 0, sizeof(dma_chan_link_map));
 501                 dma_chan_link_map[lch] = 1;
 502
 503                 cur_lch = dma_chan[lch].next_lch;
 504                 do {
 505                         next_lch = dma_chan[cur_lch].next_lch;
 506
 507                         /* The loop case: we've been here already */
 508                         if (dma_chan_link_map[cur_lch])
 509                                 break;
 510                         /* Mark the current channel */
 511                         dma_chan_link_map[cur_lch] = 1;
 512
 513                         enable_lnk(cur_lch);
 514                         omap_enable_channel_irq(cur_lch);
 515
 516                         cur_lch = next_lch;
 517                 } while (next_lch != -1);
 518         } else if (IS_DMA_ERRATA(DMA_ERRATA_PARALLEL_CHANNELS))
 519                 p->dma_write(lch, CLNK_CTRL, lch);
 520
 521         omap_enable_channel_irq(lch);
 522
 523         l = p->dma_read(CCR, lch);
 524
 525         if (IS_DMA_ERRATA(DMA_ERRATA_IFRAME_BUFFERING))
 526                         l |= OMAP_DMA_CCR_BUFFERING_DISABLE;
 527         l |= OMAP_DMA_CCR_EN;
 528
 529         /*
 530          * As dma_write() uses IO accessors which are weakly ordered, there
 531          * is no guarantee that data in coherent DMA memory will be visible
 532          * to the DMA device.  Add a memory barrier here to ensure that any
 533          * such data is visible prior to enabling DMA.
 534          */
 535         mb();
 536         p->dma_write(l, CCR, lch);
 537
 538         dma_chan[lch].flags |= OMAP_DMA_ACTIVE;
 539 }
 540 EXPORT_SYMBOL(omap_start_dma);
 541
 542 void omap_stop_dma(int lch)
 543 {
 544         u32 l;
 545
 546         /* Disable all interrupts on the channel */
 547         omap_disable_channel_irq(lch);
 548
 549         l = p->dma_read(CCR, lch);
 550         if (IS_DMA_ERRATA(DMA_ERRATA_i541) &&
 551                         (l & OMAP_DMA_CCR_SEL_SRC_DST_SYNC)) {
 552                 int i = 0;
 553                 u32 sys_cf;
 554
 555                 /* Configure No-Standby */
 556                 l = p->dma_read(OCP_SYSCONFIG, lch);
 557                 sys_cf = l;
 558                 l &= ~DMA_SYSCONFIG_MIDLEMODE_MASK;
 559                 l |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE);
 560                 p->dma_write(l , OCP_SYSCONFIG, 0);
 561
 562                 l = p->dma_read(CCR, lch);
 563                 l &= ~OMAP_DMA_CCR_EN;
 564                 p->dma_write(l, CCR, lch);
 565
 566                 /* Wait for sDMA FIFO drain */
 567                 l = p->dma_read(CCR, lch);
 568                 while (i < 100 && (l & (OMAP_DMA_CCR_RD_ACTIVE |
 569                                         OMAP_DMA_CCR_WR_ACTIVE))) {
 570                         udelay(5);
 571                         i++;
 572                         l = p->dma_read(CCR, lch);
 573                 }
 574                 if (i >= 100)
 575                         pr_err("DMA drain did not complete on lch %d\n", lch);
 576                 /* Restore OCP_SYSCONFIG */
 577                 p->dma_write(sys_cf, OCP_SYSCONFIG, lch);
 578         } else {
 579                 l &= ~OMAP_DMA_CCR_EN;
 580                 p->dma_write(l, CCR, lch);
 581         }
 582
 583         /*
 584          * Ensure that data transferred by DMA is visible to any access
 585          * after DMA has been disabled.  This is important for coherent
 586          * DMA regions.
 587          */
 588         mb();
 589
 590         if (!omap_dma_in_1510_mode() && dma_chan[lch].next_lch != -1) {
 591                 int next_lch, cur_lch = lch;
 592                 char dma_chan_link_map[MAX_LOGICAL_DMA_CH_COUNT];
 593
 594                 memset(dma_chan_link_map, 0, sizeof(dma_chan_link_map));
 595                 do {
 596                         /* The loop case: we've been here already */
 597                         if (dma_chan_link_map[cur_lch])
 598                                 break;
 599                         /* Mark the current channel */
 600                         dma_chan_link_map[cur_lch] = 1;
 601
 602                         disable_lnk(cur_lch);
 603
 604                         next_lch = dma_chan[cur_lch].next_lch;
 605                         cur_lch = next_lch;
 606                 } while (next_lch != -1);
 607         }
 608
 609         dma_chan[lch].flags &= ~OMAP_DMA_ACTIVE;
 610 }
 611 EXPORT_SYMBOL(omap_stop_dma);
 612
 613 /*
 614  * Allows changing the DMA callback function or data. This may be needed if
 615  * the driver shares a single DMA channel for multiple dma triggers.
 616  */
 617 /*
 618  * Returns current physical source address for the given DMA channel.
 619  * If the channel is running the caller must disable interrupts prior calling
 620  * this function and process the returned value before re-enabling interrupt to
 621  * prevent races with the interrupt handler. Note that in continuous mode there
 622  * is a chance for CSSA_L register overflow between the two reads resulting
 623  * in incorrect return value.
 624  */
 625 dma_addr_t omap_get_dma_src_pos(int lch)
 626 {
 627         dma_addr_t offset = 0;
 628
 629         if (dma_omap15xx())
 630                 offset = p->dma_read(CPC, lch);
 631         else
 632                 offset = p->dma_read(CSAC, lch);
 633
 634         if (IS_DMA_ERRATA(DMA_ERRATA_3_3) && offset == 0)
 635                 offset = p->dma_read(CSAC, lch);
 636
 637         if (!dma_omap15xx()) {
 638                 /*
 639                  * CDAC == 0 indicates that the DMA transfer on the channel has
 640                  * not been started (no data has been transferred so far).
 641                  * Return the programmed source start address in this case.
 642                  */
 643                 if (likely(p->dma_read(CDAC, lch)))
 644                         offset = p->dma_read(CSAC, lch);
 645                 else
 646                         offset = p->dma_read(CSSA, lch);
 647         }
 648
 649         offset |= (p->dma_read(CSSA, lch) & 0xFFFF0000);
 650
 651         return offset;
 652 }
 653 EXPORT_SYMBOL(omap_get_dma_src_pos);
 654
 655 /*
 656  * Returns current physical destination address for the given DMA channel.
 657  * If the channel is running the caller must disable interrupts prior calling
 658  * this function and process the returned value before re-enabling interrupt to
 659  * prevent races with the interrupt handler. Note that in continuous mode there
 660  * is a chance for CDSA_L register overflow between the two reads resulting
 661  * in incorrect return value.
 662  */
 663 dma_addr_t omap_get_dma_dst_pos(int lch)
 664 {
 665         dma_addr_t offset = 0;
 666
 667         if (dma_omap15xx())
 668                 offset = p->dma_read(CPC, lch);
 669         else
 670                 offset = p->dma_read(CDAC, lch);
 671
 672         /*
 673          * omap 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
 674          * read before the DMA controller finished disabling the channel.
 675          */
 676         if (!dma_omap15xx() && offset == 0) {
 677                 offset = p->dma_read(CDAC, lch);
 678                 /*
 679                  * CDAC == 0 indicates that the DMA transfer on the channel has
 680                  * not been started (no data has been transferred so far).
 681                  * Return the programmed destination start address in this case.
 682                  */
 683                 if (unlikely(!offset))
 684                         offset = p->dma_read(CDSA, lch);
 685         }
 686
 687         offset |= (p->dma_read(CDSA, lch) & 0xFFFF0000);
 688
 689         return offset;
 690 }
 691 EXPORT_SYMBOL(omap_get_dma_dst_pos);
 692
 693 int omap_get_dma_active_status(int lch)
 694 {
 695         return (p->dma_read(CCR, lch) & OMAP_DMA_CCR_EN) != 0;
 696 }
 697 EXPORT_SYMBOL(omap_get_dma_active_status);
 698 #endif
 699
 700 int omap_dma_running(void)
 701 {
 702         int lch;
 703
 704         if (omap_lcd_dma_running())
 705                 return 1;
 706
 707         for (lch = 0; lch < dma_chan_count; lch++)
 708                 if (p->dma_read(CCR, lch) & OMAP_DMA_CCR_EN)
 709                         return 1;
 710
 711         return 0;
 712 }
 713
 714 /*----------------------------------------------------------------------------*/
 715
 716 static int omap1_dma_handle_ch(int ch)
 717 {
 718         u32 csr;
 719
 720         if (enable_1510_mode && ch >= 6) {
 721                 csr = dma_chan[ch].saved_csr;
 722                 dma_chan[ch].saved_csr = 0;
 723         } else
 724                 csr = p->dma_read(CSR, ch);
 725         if (enable_1510_mode && ch <= 2 && (csr >> 7) != 0) {
 726                 dma_chan[ch + 6].saved_csr = csr >> 7;
 727                 csr &= 0x7f;
 728         }
 729         if ((csr & 0x3f) == 0)
 730                 return 0;
 731         if (unlikely(dma_chan[ch].dev_id == -1)) {
 732                 pr_warn("Spurious interrupt from DMA channel %d (CSR %04x)\n",
 733                         ch, csr);
 734                 return 0;
 735         }
 736         if (unlikely(csr & OMAP1_DMA_TOUT_IRQ))
 737                 pr_warn("DMA timeout with device %d\n", dma_chan[ch].dev_id);
 738         if (unlikely(csr & OMAP_DMA_DROP_IRQ))
 739                 pr_warn("DMA synchronization event drop occurred with device %d\n",
 740                         dma_chan[ch].dev_id);
 741         if (likely(csr & OMAP_DMA_BLOCK_IRQ))
 742                 dma_chan[ch].flags &= ~OMAP_DMA_ACTIVE;
 743         if (likely(dma_chan[ch].callback != NULL))
 744                 dma_chan[ch].callback(ch, csr, dma_chan[ch].data);
 745
 746         return 1;
 747 }
 748
 749 static irqreturn_t omap1_dma_irq_handler(int irq, void *dev_id)
 750 {
 751         int ch = ((int) dev_id) - 1;
 752         int handled = 0;
 753
 754         for (;;) {
 755                 int handled_now = 0;
 756
 757                 handled_now += omap1_dma_handle_ch(ch);
 758                 if (enable_1510_mode && dma_chan[ch + 6].saved_csr)
 759                         handled_now += omap1_dma_handle_ch(ch + 6);
 760                 if (!handled_now)
 761                         break;
 762                 handled += handled_now;
 763         }
 764
 765         return handled ? IRQ_HANDLED : IRQ_NONE;
 766 }
 767
 768 struct omap_system_dma_plat_info *omap_get_plat_info(void)
 769 {
 770         return p;
 771 }
 772 EXPORT_SYMBOL_GPL(omap_get_plat_info);
 773
 774 static int omap_system_dma_probe(struct platform_device *pdev)
 775 {
 776         int ch, ret = 0;
 777         int dma_irq;
 778         char irq_name[4];
 779
 780         p = pdev->dev.platform_data;
 781         if (!p) {
 782                 dev_err(&pdev->dev,
 783                         "%s: System DMA initialized without platform data\n",
 784                         __func__);
 785                 return -EINVAL;
 786         }
 787
 788         d                       = p->dma_attr;
 789         errata                  = p->errata;
 790
 791         if ((d->dev_caps & RESERVE_CHANNEL) && omap_dma_reserve_channels
 792                         && (omap_dma_reserve_channels < d->lch_count))
 793                 d->lch_count    = omap_dma_reserve_channels;
 794
 795         dma_lch_count           = d->lch_count;
 796         dma_chan_count          = dma_lch_count;
 797         enable_1510_mode        = d->dev_caps & ENABLE_1510_MODE;
 798
 799         dma_chan = devm_kcalloc(&pdev->dev, dma_lch_count,
 800                                 sizeof(*dma_chan), GFP_KERNEL);
 801         if (!dma_chan)
 802                 return -ENOMEM;
 803
 804         for (ch = 0; ch < dma_chan_count; ch++) {
 805                 omap_clear_dma(ch);
 806
 807                 dma_chan[ch].dev_id = -1;
 808                 dma_chan[ch].next_lch = -1;
 809
 810                 if (ch >= 6 && enable_1510_mode)
 811                         continue;
 812
 813                 /*
 814                  * request_irq() doesn't like dev_id (ie. ch) being
 815                  * zero, so we have to kludge around this.
 816                  */
 817                 sprintf(&irq_name[0], "%d", ch);
 818                 dma_irq = platform_get_irq_byname(pdev, irq_name);
 819
 820                 if (dma_irq < 0) {
 821                         ret = dma_irq;
 822                         goto exit_dma_irq_fail;
 823                 }
 824
 825                 /* INT_DMA_LCD is handled in lcd_dma.c */
 826                 if (dma_irq == INT_DMA_LCD)
 827                         continue;
 828
 829                 ret = request_irq(dma_irq,
 830                                 omap1_dma_irq_handler, 0, "DMA",
 831                                 (void *) (ch + 1));
 832                 if (ret != 0)
 833                         goto exit_dma_irq_fail;
 834         }
 835
 836         /* reserve dma channels 0 and 1 in high security devices on 34xx */
 837         if (d->dev_caps & HS_CHANNELS_RESERVED) {
 838                 pr_info("Reserving DMA channels 0 and 1 for HS ROM code\n");
 839                 dma_chan[0].dev_id = 0;
 840                 dma_chan[1].dev_id = 1;
 841         }
 842         p->show_dma_caps();
 843         return 0;
 844
 845 exit_dma_irq_fail:
 846         return ret;
 847 }
 848
 849 static int omap_system_dma_remove(struct platform_device *pdev)
 850 {
 851         int dma_irq, irq_rel = 0;
 852
 853         for ( ; irq_rel < dma_chan_count; irq_rel++) {
 854                 dma_irq = platform_get_irq(pdev, irq_rel);
 855                 free_irq(dma_irq, (void *)(irq_rel + 1));
 856         }
 857
 858         return 0;
 859 }
 860
 861 static struct platform_driver omap_system_dma_driver = {
 862         .probe          = omap_system_dma_probe,
 863         .remove         = omap_system_dma_remove,
 864         .driver         = {
 865                 .name   = "omap_dma_system"
 866         },
 867 };
 868
 869 static int __init omap_system_dma_init(void)
 870 {
 871         return platform_driver_register(&omap_system_dma_driver);
 872 }
 873 arch_initcall(omap_system_dma_init);
 874
 875 static void __exit omap_system_dma_exit(void)
 876 {
 877         platform_driver_unregister(&omap_system_dma_driver);
 878 }
 879
 880 MODULE_DESCRIPTION("OMAP SYSTEM DMA DRIVER");
 881 MODULE_LICENSE("GPL");
 882 MODULE_AUTHOR("Texas Instruments Inc");
 883
 884 /*
 885  * Reserve the omap SDMA channels using cmdline bootarg
 886  * "omap_dma_reserve_ch=". The valid range is 1 to 32
 887  */
 888 static int __init omap_dma_cmdline_reserve_ch(char *str)
 889 {
 890         if (get_option(&str, &omap_dma_reserve_channels) != 1)
 891                 omap_dma_reserve_channels = 0;
 892         return 1;
 893 }
 894
 895 __setup("omap_dma_reserve_ch=", omap_dma_cmdline_reserve_ch);
 896
 897