drivers/dma/ti/edma.c

   1 /*
   2  * TI EDMA DMA engine driver
   3  *
   4  * Copyright 2012 Texas Instruments
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as
   8  * published by the Free Software Foundation version 2.
   9  *
  10  * This program is distributed "as is" WITHOUT ANY WARRANTY of any
  11  * kind, whether express or implied; without even the implied warranty
  12  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  */
  15
  16 #include <linux/dmaengine.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/bitmap.h>
  19 #include <linux/err.h>
  20 #include <linux/init.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/list.h>
  23 #include <linux/module.h>
  24 #include <linux/platform_device.h>
  25 #include <linux/slab.h>
  26 #include <linux/spinlock.h>
  27 #include <linux/of.h>
  28 #include <linux/of_dma.h>
  29 #include <linux/of_irq.h>
  30 #include <linux/of_address.h>
  31 #include <linux/of_device.h>
  32 #include <linux/pm_runtime.h>
  33
  34 #include <linux/platform_data/edma.h>
  35
  36 #include "../dmaengine.h"
  37 #include "../virt-dma.h"
  38
  39 /* Offsets matching "struct edmacc_param" */
  40 #define PARM_OPT                0x00
  41 #define PARM_SRC                0x04
  42 #define PARM_A_B_CNT            0x08
  43 #define PARM_DST                0x0c
  44 #define PARM_SRC_DST_BIDX       0x10
  45 #define PARM_LINK_BCNTRLD       0x14
  46 #define PARM_SRC_DST_CIDX       0x18
  47 #define PARM_CCNT               0x1c
  48
  49 #define PARM_SIZE               0x20
  50
  51 /* Offsets for EDMA CC global channel registers and their shadows */
  52 #define SH_ER                   0x00    /* 64 bits */
  53 #define SH_ECR                  0x08    /* 64 bits */
  54 #define SH_ESR                  0x10    /* 64 bits */
  55 #define SH_CER                  0x18    /* 64 bits */
  56 #define SH_EER                  0x20    /* 64 bits */
  57 #define SH_EECR                 0x28    /* 64 bits */
  58 #define SH_EESR                 0x30    /* 64 bits */
  59 #define SH_SER                  0x38    /* 64 bits */
  60 #define SH_SECR                 0x40    /* 64 bits */
  61 #define SH_IER                  0x50    /* 64 bits */
  62 #define SH_IECR                 0x58    /* 64 bits */
  63 #define SH_IESR                 0x60    /* 64 bits */
  64 #define SH_IPR                  0x68    /* 64 bits */
  65 #define SH_ICR                  0x70    /* 64 bits */
  66 #define SH_IEVAL                0x78
  67 #define SH_QER                  0x80
  68 #define SH_QEER                 0x84
  69 #define SH_QEECR                0x88
  70 #define SH_QEESR                0x8c
  71 #define SH_QSER                 0x90
  72 #define SH_QSECR                0x94
  73 #define SH_SIZE                 0x200
  74
  75 /* Offsets for EDMA CC global registers */
  76 #define EDMA_REV                0x0000
  77 #define EDMA_CCCFG              0x0004
  78 #define EDMA_QCHMAP             0x0200  /* 8 registers */
  79 #define EDMA_DMAQNUM            0x0240  /* 8 registers (4 on OMAP-L1xx) */
  80 #define EDMA_QDMAQNUM           0x0260
  81 #define EDMA_QUETCMAP           0x0280
  82 #define EDMA_QUEPRI             0x0284
  83 #define EDMA_EMR                0x0300  /* 64 bits */
  84 #define EDMA_EMCR               0x0308  /* 64 bits */
  85 #define EDMA_QEMR               0x0310
  86 #define EDMA_QEMCR              0x0314
  87 #define EDMA_CCERR              0x0318
  88 #define EDMA_CCERRCLR           0x031c
  89 #define EDMA_EEVAL              0x0320
  90 #define EDMA_DRAE               0x0340  /* 4 x 64 bits*/
  91 #define EDMA_QRAE               0x0380  /* 4 registers */
  92 #define EDMA_QUEEVTENTRY        0x0400  /* 2 x 16 registers */
  93 #define EDMA_QSTAT              0x0600  /* 2 registers */
  94 #define EDMA_QWMTHRA            0x0620
  95 #define EDMA_QWMTHRB            0x0624
  96 #define EDMA_CCSTAT             0x0640
  97
  98 #define EDMA_M                  0x1000  /* global channel registers */
  99 #define EDMA_ECR                0x1008
 100 #define EDMA_ECRH               0x100C
 101 #define EDMA_SHADOW0            0x2000  /* 4 shadow regions */
 102 #define EDMA_PARM               0x4000  /* PaRAM entries */
 103
 104 #define PARM_OFFSET(param_no)   (EDMA_PARM + ((param_no) << 5))
 105
 106 #define EDMA_DCHMAP             0x0100  /* 64 registers */
 107
 108 /* CCCFG register */
 109 #define GET_NUM_DMACH(x)        (x & 0x7) /* bits 0-2 */
 110 #define GET_NUM_QDMACH(x)       ((x & 0x70) >> 4) /* bits 4-6 */
 111 #define GET_NUM_PAENTRY(x)      ((x & 0x7000) >> 12) /* bits 12-14 */
 112 #define GET_NUM_EVQUE(x)        ((x & 0x70000) >> 16) /* bits 16-18 */
 113 #define GET_NUM_REGN(x)         ((x & 0x300000) >> 20) /* bits 20-21 */
 114 #define CHMAP_EXIST             BIT(24)
 115
 116 /* CCSTAT register */
 117 #define EDMA_CCSTAT_ACTV        BIT(4)
 118
 119 /*
 120  * Max of 20 segments per channel to conserve PaRAM slots
 121  * Also note that MAX_NR_SG should be atleast the no.of periods
 122  * that are required for ASoC, otherwise DMA prep calls will
 123  * fail. Today davinci-pcm is the only user of this driver and
 124  * requires atleast 17 slots, so we setup the default to 20.
 125  */
 126 #define MAX_NR_SG               20
 127 #define EDMA_MAX_SLOTS          MAX_NR_SG
 128 #define EDMA_DESCRIPTORS        16
 129
 130 #define EDMA_CHANNEL_ANY                -1      /* for edma_alloc_channel() */
 131 #define EDMA_SLOT_ANY                   -1      /* for edma_alloc_slot() */
 132 #define EDMA_CONT_PARAMS_ANY             1001
 133 #define EDMA_CONT_PARAMS_FIXED_EXACT     1002
 134 #define EDMA_CONT_PARAMS_FIXED_NOT_EXACT 1003
 135
 136 /*
 137  * 64bit array registers are split into two 32bit registers:
 138  * reg0: channel/event 0-31
 139  * reg1: channel/event 32-63
 140  *
 141  * bit 5 in the channel number tells the array index (0/1)
 142  * bit 0-4 (0x1f) is the bit offset within the register
 143  */
 144 #define EDMA_REG_ARRAY_INDEX(channel)   ((channel) >> 5)
 145 #define EDMA_CHANNEL_BIT(channel)       (BIT((channel) & 0x1f))
 146
 147 /* PaRAM slots are laid out like this */
 148 struct edmacc_param {
 149         u32 opt;
 150         u32 src;
 151         u32 a_b_cnt;
 152         u32 dst;
 153         u32 src_dst_bidx;
 154         u32 link_bcntrld;
 155         u32 src_dst_cidx;
 156         u32 ccnt;
 157 } __packed;
 158
 159 /* fields in edmacc_param.opt */
 160 #define SAM             BIT(0)
 161 #define DAM             BIT(1)
 162 #define SYNCDIM         BIT(2)
 163 #define STATIC          BIT(3)
 164 #define EDMA_FWID       (0x07 << 8)
 165 #define TCCMODE         BIT(11)
 166 #define EDMA_TCC(t)     ((t) << 12)
 167 #define TCINTEN         BIT(20)
 168 #define ITCINTEN        BIT(21)
 169 #define TCCHEN          BIT(22)
 170 #define ITCCHEN         BIT(23)
 171
 172 struct edma_pset {
 173         u32                             len;
 174         dma_addr_t                      addr;
 175         struct edmacc_param             param;
 176 };
 177
 178 struct edma_desc {
 179         struct virt_dma_desc            vdesc;
 180         struct list_head                node;
 181         enum dma_transfer_direction     direction;
 182         int                             cyclic;
 183         bool                            polled;
 184         int                             absync;
 185         int                             pset_nr;
 186         struct edma_chan                *echan;
 187         int                             processed;
 188
 189         /*
 190          * The following 4 elements are used for residue accounting.
 191          *
 192          * - processed_stat: the number of SG elements we have traversed
 193          * so far to cover accounting. This is updated directly to processed
 194          * during edma_callback and is always <= processed, because processed
 195          * refers to the number of pending transfer (programmed to EDMA
 196          * controller), where as processed_stat tracks number of transfers
 197          * accounted for so far.
 198          *
 199          * - residue: The amount of bytes we have left to transfer for this desc
 200          *
 201          * - residue_stat: The residue in bytes of data we have covered
 202          * so far for accounting. This is updated directly to residue
 203          * during callbacks to keep it current.
 204          *
 205          * - sg_len: Tracks the length of the current intermediate transfer,
 206          * this is required to update the residue during intermediate transfer
 207          * completion callback.
 208          */
 209         int                             processed_stat;
 210         u32                             sg_len;
 211         u32                             residue;
 212         u32                             residue_stat;
 213
 214         struct edma_pset                pset[0];
 215 };
 216
 217 struct edma_cc;
 218
 219 struct edma_tc {
 220         struct device_node              *node;
 221         u16                             id;
 222 };
 223
 224 struct edma_chan {
 225         struct virt_dma_chan            vchan;
 226         struct list_head                node;
 227         struct edma_desc                *edesc;
 228         struct edma_cc                  *ecc;
 229         struct edma_tc                  *tc;
 230         int                             ch_num;
 231         bool                            alloced;
 232         bool                            hw_triggered;
 233         int                             slot[EDMA_MAX_SLOTS];
 234         int                             missed;
 235         struct dma_slave_config         cfg;
 236 };
 237
 238 struct edma_cc {
 239         struct device                   *dev;
 240         struct edma_soc_info            *info;
 241         void __iomem                    *base;
 242         int                             id;
 243         bool                            legacy_mode;
 244
 245         /* eDMA3 resource information */
 246         unsigned                        num_channels;
 247         unsigned                        num_qchannels;
 248         unsigned                        num_region;
 249         unsigned                        num_slots;
 250         unsigned                        num_tc;
 251         bool                            chmap_exist;
 252         enum dma_event_q                default_queue;
 253
 254         unsigned int                    ccint;
 255         unsigned int                    ccerrint;
 256
 257         /*
 258          * The slot_inuse bit for each PaRAM slot is clear unless the slot is
 259          * in use by Linux or if it is allocated to be used by DSP.
 260          */
 261         unsigned long *slot_inuse;
 262
 263         struct dma_device               dma_slave;
 264         struct dma_device               *dma_memcpy;
 265         struct edma_chan                *slave_chans;
 266         struct edma_tc                  *tc_list;
 267         int                             dummy_slot;
 268 };
 269
 270 /* dummy param set used to (re)initialize parameter RAM slots */
 271 static const struct edmacc_param dummy_paramset = {
 272         .link_bcntrld = 0xffff,
 273         .ccnt = 1,
 274 };
 275
 276 #define EDMA_BINDING_LEGACY     0
 277 #define EDMA_BINDING_TPCC       1
 278 static const u32 edma_binding_type[] = {
 279         [EDMA_BINDING_LEGACY] = EDMA_BINDING_LEGACY,
 280         [EDMA_BINDING_TPCC] = EDMA_BINDING_TPCC,
 281 };
 282
 283 static const struct of_device_id edma_of_ids[] = {
 284         {
 285                 .compatible = "ti,edma3",
 286                 .data = &edma_binding_type[EDMA_BINDING_LEGACY],
 287         },
 288         {
 289                 .compatible = "ti,edma3-tpcc",
 290                 .data = &edma_binding_type[EDMA_BINDING_TPCC],
 291         },
 292         {}
 293 };
 294 MODULE_DEVICE_TABLE(of, edma_of_ids);
 295
 296 static const struct of_device_id edma_tptc_of_ids[] = {
 297         { .compatible = "ti,edma3-tptc", },
 298         {}
 299 };
 300 MODULE_DEVICE_TABLE(of, edma_tptc_of_ids);
 301
 302 static inline unsigned int edma_read(struct edma_cc *ecc, int offset)
 303 {
 304         return (unsigned int)__raw_readl(ecc->base + offset);
 305 }
 306
 307 static inline void edma_write(struct edma_cc *ecc, int offset, int val)
 308 {
 309         __raw_writel(val, ecc->base + offset);
 310 }
 311
 312 static inline void edma_modify(struct edma_cc *ecc, int offset, unsigned and,
 313                                unsigned or)
 314 {
 315         unsigned val = edma_read(ecc, offset);
 316
 317         val &= and;
 318         val |= or;
 319         edma_write(ecc, offset, val);
 320 }
 321
 322 static inline void edma_and(struct edma_cc *ecc, int offset, unsigned and)
 323 {
 324         unsigned val = edma_read(ecc, offset);
 325
 326         val &= and;
 327         edma_write(ecc, offset, val);
 328 }
 329
 330 static inline void edma_or(struct edma_cc *ecc, int offset, unsigned or)
 331 {
 332         unsigned val = edma_read(ecc, offset);
 333
 334         val |= or;
 335         edma_write(ecc, offset, val);
 336 }
 337
 338 static inline unsigned int edma_read_array(struct edma_cc *ecc, int offset,
 339                                            int i)
 340 {
 341         return edma_read(ecc, offset + (i << 2));
 342 }
 343
 344 static inline void edma_write_array(struct edma_cc *ecc, int offset, int i,
 345                                     unsigned val)
 346 {
 347         edma_write(ecc, offset + (i << 2), val);
 348 }
 349
 350 static inline void edma_modify_array(struct edma_cc *ecc, int offset, int i,
 351                                      unsigned and, unsigned or)
 352 {
 353         edma_modify(ecc, offset + (i << 2), and, or);
 354 }
 355
 356 static inline void edma_or_array(struct edma_cc *ecc, int offset, int i,
 357                                  unsigned or)
 358 {
 359         edma_or(ecc, offset + (i << 2), or);
 360 }
 361
 362 static inline void edma_or_array2(struct edma_cc *ecc, int offset, int i, int j,
 363                                   unsigned or)
 364 {
 365         edma_or(ecc, offset + ((i * 2 + j) << 2), or);
 366 }
 367
 368 static inline void edma_write_array2(struct edma_cc *ecc, int offset, int i,
 369                                      int j, unsigned val)
 370 {
 371         edma_write(ecc, offset + ((i * 2 + j) << 2), val);
 372 }
 373
 374 static inline unsigned int edma_shadow0_read(struct edma_cc *ecc, int offset)
 375 {
 376         return edma_read(ecc, EDMA_SHADOW0 + offset);
 377 }
 378
 379 static inline unsigned int edma_shadow0_read_array(struct edma_cc *ecc,
 380                                                    int offset, int i)
 381 {
 382         return edma_read(ecc, EDMA_SHADOW0 + offset + (i << 2));
 383 }
 384
 385 static inline void edma_shadow0_write(struct edma_cc *ecc, int offset,
 386                                       unsigned val)
 387 {
 388         edma_write(ecc, EDMA_SHADOW0 + offset, val);
 389 }
 390
 391 static inline void edma_shadow0_write_array(struct edma_cc *ecc, int offset,
 392                                             int i, unsigned val)
 393 {
 394         edma_write(ecc, EDMA_SHADOW0 + offset + (i << 2), val);
 395 }
 396
 397 static inline unsigned int edma_param_read(struct edma_cc *ecc, int offset,
 398                                            int param_no)
 399 {
 400         return edma_read(ecc, EDMA_PARM + offset + (param_no << 5));
 401 }
 402
 403 static inline void edma_param_write(struct edma_cc *ecc, int offset,
 404                                     int param_no, unsigned val)
 405 {
 406         edma_write(ecc, EDMA_PARM + offset + (param_no << 5), val);
 407 }
 408
 409 static inline void edma_param_modify(struct edma_cc *ecc, int offset,
 410                                      int param_no, unsigned and, unsigned or)
 411 {
 412         edma_modify(ecc, EDMA_PARM + offset + (param_no << 5), and, or);
 413 }
 414
 415 static inline void edma_param_and(struct edma_cc *ecc, int offset, int param_no,
 416                                   unsigned and)
 417 {
 418         edma_and(ecc, EDMA_PARM + offset + (param_no << 5), and);
 419 }
 420
 421 static inline void edma_param_or(struct edma_cc *ecc, int offset, int param_no,
 422                                  unsigned or)
 423 {
 424         edma_or(ecc, EDMA_PARM + offset + (param_no << 5), or);
 425 }
 426
 427 static void edma_assign_priority_to_queue(struct edma_cc *ecc, int queue_no,
 428                                           int priority)
 429 {
 430         int bit = queue_no * 4;
 431
 432         edma_modify(ecc, EDMA_QUEPRI, ~(0x7 << bit), ((priority & 0x7) << bit));
 433 }
 434
 435 static void edma_set_chmap(struct edma_chan *echan, int slot)
 436 {
 437         struct edma_cc *ecc = echan->ecc;
 438         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 439
 440         if (ecc->chmap_exist) {
 441                 slot = EDMA_CHAN_SLOT(slot);
 442                 edma_write_array(ecc, EDMA_DCHMAP, channel, (slot << 5));
 443         }
 444 }
 445
 446 static void edma_setup_interrupt(struct edma_chan *echan, bool enable)
 447 {
 448         struct edma_cc *ecc = echan->ecc;
 449         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 450         int idx = EDMA_REG_ARRAY_INDEX(channel);
 451         int ch_bit = EDMA_CHANNEL_BIT(channel);
 452
 453         if (enable) {
 454                 edma_shadow0_write_array(ecc, SH_ICR, idx, ch_bit);
 455                 edma_shadow0_write_array(ecc, SH_IESR, idx, ch_bit);
 456         } else {
 457                 edma_shadow0_write_array(ecc, SH_IECR, idx, ch_bit);
 458         }
 459 }
 460
 461 /*
 462  * paRAM slot management functions
 463  */
 464 static void edma_write_slot(struct edma_cc *ecc, unsigned slot,
 465                             const struct edmacc_param *param)
 466 {
 467         slot = EDMA_CHAN_SLOT(slot);
 468         if (slot >= ecc->num_slots)
 469                 return;
 470         memcpy_toio(ecc->base + PARM_OFFSET(slot), param, PARM_SIZE);
 471 }
 472
 473 static int edma_read_slot(struct edma_cc *ecc, unsigned slot,
 474                            struct edmacc_param *param)
 475 {
 476         slot = EDMA_CHAN_SLOT(slot);
 477         if (slot >= ecc->num_slots)
 478                 return -EINVAL;
 479         memcpy_fromio(param, ecc->base + PARM_OFFSET(slot), PARM_SIZE);
 480
 481         return 0;
 482 }
 483
 484 /**
 485  * edma_alloc_slot - allocate DMA parameter RAM
 486  * @ecc: pointer to edma_cc struct
 487  * @slot: specific slot to allocate; negative for "any unused slot"
 488  *
 489  * This allocates a parameter RAM slot, initializing it to hold a
 490  * dummy transfer.  Slots allocated using this routine have not been
 491  * mapped to a hardware DMA channel, and will normally be used by
 492  * linking to them from a slot associated with a DMA channel.
 493  *
 494  * Normal use is to pass EDMA_SLOT_ANY as the @slot, but specific
 495  * slots may be allocated on behalf of DSP firmware.
 496  *
 497  * Returns the number of the slot, else negative errno.
 498  */
 499 static int edma_alloc_slot(struct edma_cc *ecc, int slot)
 500 {
 501         if (slot >= 0) {
 502                 slot = EDMA_CHAN_SLOT(slot);
 503                 /* Requesting entry paRAM slot for a HW triggered channel. */
 504                 if (ecc->chmap_exist && slot < ecc->num_channels)
 505                         slot = EDMA_SLOT_ANY;
 506         }
 507
 508         if (slot < 0) {
 509                 if (ecc->chmap_exist)
 510                         slot = 0;
 511                 else
 512                         slot = ecc->num_channels;
 513                 for (;;) {
 514                         slot = find_next_zero_bit(ecc->slot_inuse,
 515                                                   ecc->num_slots,
 516                                                   slot);
 517                         if (slot == ecc->num_slots)
 518                                 return -ENOMEM;
 519                         if (!test_and_set_bit(slot, ecc->slot_inuse))
 520                                 break;
 521                 }
 522         } else if (slot >= ecc->num_slots) {
 523                 return -EINVAL;
 524         } else if (test_and_set_bit(slot, ecc->slot_inuse)) {
 525                 return -EBUSY;
 526         }
 527
 528         edma_write_slot(ecc, slot, &dummy_paramset);
 529
 530         return EDMA_CTLR_CHAN(ecc->id, slot);
 531 }
 532
 533 static void edma_free_slot(struct edma_cc *ecc, unsigned slot)
 534 {
 535         slot = EDMA_CHAN_SLOT(slot);
 536         if (slot >= ecc->num_slots)
 537                 return;
 538
 539         edma_write_slot(ecc, slot, &dummy_paramset);
 540         clear_bit(slot, ecc->slot_inuse);
 541 }
 542
 543 /**
 544  * edma_link - link one parameter RAM slot to another
 545  * @ecc: pointer to edma_cc struct
 546  * @from: parameter RAM slot originating the link
 547  * @to: parameter RAM slot which is the link target
 548  *
 549  * The originating slot should not be part of any active DMA transfer.
 550  */
 551 static void edma_link(struct edma_cc *ecc, unsigned from, unsigned to)
 552 {
 553         if (unlikely(EDMA_CTLR(from) != EDMA_CTLR(to)))
 554                 dev_warn(ecc->dev, "Ignoring eDMA instance for linking\n");
 555
 556         from = EDMA_CHAN_SLOT(from);
 557         to = EDMA_CHAN_SLOT(to);
 558         if (from >= ecc->num_slots || to >= ecc->num_slots)
 559                 return;
 560
 561         edma_param_modify(ecc, PARM_LINK_BCNTRLD, from, 0xffff0000,
 562                           PARM_OFFSET(to));
 563 }
 564
 565 /**
 566  * edma_get_position - returns the current transfer point
 567  * @ecc: pointer to edma_cc struct
 568  * @slot: parameter RAM slot being examined
 569  * @dst:  true selects the dest position, false the source
 570  *
 571  * Returns the position of the current active slot
 572  */
 573 static dma_addr_t edma_get_position(struct edma_cc *ecc, unsigned slot,
 574                                     bool dst)
 575 {
 576         u32 offs;
 577
 578         slot = EDMA_CHAN_SLOT(slot);
 579         offs = PARM_OFFSET(slot);
 580         offs += dst ? PARM_DST : PARM_SRC;
 581
 582         return edma_read(ecc, offs);
 583 }
 584
 585 /*
 586  * Channels with event associations will be triggered by their hardware
 587  * events, and channels without such associations will be triggered by
 588  * software.  (At this writing there is no interface for using software
 589  * triggers except with channels that don't support hardware triggers.)
 590  */
 591 static void edma_start(struct edma_chan *echan)
 592 {
 593         struct edma_cc *ecc = echan->ecc;
 594         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 595         int idx = EDMA_REG_ARRAY_INDEX(channel);
 596         int ch_bit = EDMA_CHANNEL_BIT(channel);
 597
 598         if (!echan->hw_triggered) {
 599                 /* EDMA channels without event association */
 600                 dev_dbg(ecc->dev, "ESR%d %08x\n", idx,
 601                         edma_shadow0_read_array(ecc, SH_ESR, idx));
 602                 edma_shadow0_write_array(ecc, SH_ESR, idx, ch_bit);
 603         } else {
 604                 /* EDMA channel with event association */
 605                 dev_dbg(ecc->dev, "ER%d %08x\n", idx,
 606                         edma_shadow0_read_array(ecc, SH_ER, idx));
 607                 /* Clear any pending event or error */
 608                 edma_write_array(ecc, EDMA_ECR, idx, ch_bit);
 609                 edma_write_array(ecc, EDMA_EMCR, idx, ch_bit);
 610                 /* Clear any SER */
 611                 edma_shadow0_write_array(ecc, SH_SECR, idx, ch_bit);
 612                 edma_shadow0_write_array(ecc, SH_EESR, idx, ch_bit);
 613                 dev_dbg(ecc->dev, "EER%d %08x\n", idx,
 614                         edma_shadow0_read_array(ecc, SH_EER, idx));
 615         }
 616 }
 617
 618 static void edma_stop(struct edma_chan *echan)
 619 {
 620         struct edma_cc *ecc = echan->ecc;
 621         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 622         int idx = EDMA_REG_ARRAY_INDEX(channel);
 623         int ch_bit = EDMA_CHANNEL_BIT(channel);
 624
 625         edma_shadow0_write_array(ecc, SH_EECR, idx, ch_bit);
 626         edma_shadow0_write_array(ecc, SH_ECR, idx, ch_bit);
 627         edma_shadow0_write_array(ecc, SH_SECR, idx, ch_bit);
 628         edma_write_array(ecc, EDMA_EMCR, idx, ch_bit);
 629
 630         /* clear possibly pending completion interrupt */
 631         edma_shadow0_write_array(ecc, SH_ICR, idx, ch_bit);
 632
 633         dev_dbg(ecc->dev, "EER%d %08x\n", idx,
 634                 edma_shadow0_read_array(ecc, SH_EER, idx));
 635
 636         /* REVISIT:  consider guarding against inappropriate event
 637          * chaining by overwriting with dummy_paramset.
 638          */
 639 }
 640
 641 /*
 642  * Temporarily disable EDMA hardware events on the specified channel,
 643  * preventing them from triggering new transfers
 644  */
 645 static void edma_pause(struct edma_chan *echan)
 646 {
 647         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 648
 649         edma_shadow0_write_array(echan->ecc, SH_EECR,
 650                                  EDMA_REG_ARRAY_INDEX(channel),
 651                                  EDMA_CHANNEL_BIT(channel));
 652 }
 653
 654 /* Re-enable EDMA hardware events on the specified channel.  */
 655 static void edma_resume(struct edma_chan *echan)
 656 {
 657         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 658
 659         edma_shadow0_write_array(echan->ecc, SH_EESR,
 660                                  EDMA_REG_ARRAY_INDEX(channel),
 661                                  EDMA_CHANNEL_BIT(channel));
 662 }
 663
 664 static void edma_trigger_channel(struct edma_chan *echan)
 665 {
 666         struct edma_cc *ecc = echan->ecc;
 667         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 668         int idx = EDMA_REG_ARRAY_INDEX(channel);
 669         int ch_bit = EDMA_CHANNEL_BIT(channel);
 670
 671         edma_shadow0_write_array(ecc, SH_ESR, idx, ch_bit);
 672
 673         dev_dbg(ecc->dev, "ESR%d %08x\n", idx,
 674                 edma_shadow0_read_array(ecc, SH_ESR, idx));
 675 }
 676
 677 static void edma_clean_channel(struct edma_chan *echan)
 678 {
 679         struct edma_cc *ecc = echan->ecc;
 680         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 681         int idx = EDMA_REG_ARRAY_INDEX(channel);
 682         int ch_bit = EDMA_CHANNEL_BIT(channel);
 683
 684         dev_dbg(ecc->dev, "EMR%d %08x\n", idx,
 685                 edma_read_array(ecc, EDMA_EMR, idx));
 686         edma_shadow0_write_array(ecc, SH_ECR, idx, ch_bit);
 687         /* Clear the corresponding EMR bits */
 688         edma_write_array(ecc, EDMA_EMCR, idx, ch_bit);
 689         /* Clear any SER */
 690         edma_shadow0_write_array(ecc, SH_SECR, idx, ch_bit);
 691         edma_write(ecc, EDMA_CCERRCLR, BIT(16) | BIT(1) | BIT(0));
 692 }
 693
 694 /* Move channel to a specific event queue */
 695 static void edma_assign_channel_eventq(struct edma_chan *echan,
 696                                        enum dma_event_q eventq_no)
 697 {
 698         struct edma_cc *ecc = echan->ecc;
 699         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 700         int bit = (channel & 0x7) * 4;
 701
 702         /* default to low priority queue */
 703         if (eventq_no == EVENTQ_DEFAULT)
 704                 eventq_no = ecc->default_queue;
 705         if (eventq_no >= ecc->num_tc)
 706                 return;
 707
 708         eventq_no &= 7;
 709         edma_modify_array(ecc, EDMA_DMAQNUM, (channel >> 3), ~(0x7 << bit),
 710                           eventq_no << bit);
 711 }
 712
 713 static int edma_alloc_channel(struct edma_chan *echan,
 714                               enum dma_event_q eventq_no)
 715 {
 716         struct edma_cc *ecc = echan->ecc;
 717         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 718
 719         /* ensure access through shadow region 0 */
 720         edma_or_array2(ecc, EDMA_DRAE, 0, EDMA_REG_ARRAY_INDEX(channel),
 721                        EDMA_CHANNEL_BIT(channel));
 722
 723         /* ensure no events are pending */
 724         edma_stop(echan);
 725
 726         edma_setup_interrupt(echan, true);
 727
 728         edma_assign_channel_eventq(echan, eventq_no);
 729
 730         return 0;
 731 }
 732
 733 static void edma_free_channel(struct edma_chan *echan)
 734 {
 735         /* ensure no events are pending */
 736         edma_stop(echan);
 737         /* REVISIT should probably take out of shadow region 0 */
 738         edma_setup_interrupt(echan, false);
 739 }
 740
 741 static inline struct edma_cc *to_edma_cc(struct dma_device *d)
 742 {
 743         return container_of(d, struct edma_cc, dma_slave);
 744 }
 745
 746 static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
 747 {
 748         return container_of(c, struct edma_chan, vchan.chan);
 749 }
 750
 751 static inline struct edma_desc *to_edma_desc(struct dma_async_tx_descriptor *tx)
 752 {
 753         return container_of(tx, struct edma_desc, vdesc.tx);
 754 }
 755
 756 static void edma_desc_free(struct virt_dma_desc *vdesc)
 757 {
 758         kfree(container_of(vdesc, struct edma_desc, vdesc));
 759 }
 760
 761 /* Dispatch a queued descriptor to the controller (caller holds lock) */
 762 static void edma_execute(struct edma_chan *echan)
 763 {
 764         struct edma_cc *ecc = echan->ecc;
 765         struct virt_dma_desc *vdesc;
 766         struct edma_desc *edesc;
 767         struct device *dev = echan->vchan.chan.device->dev;
 768         int i, j, left, nslots;
 769
 770         if (!echan->edesc) {
 771                 /* Setup is needed for the first transfer */
 772                 vdesc = vchan_next_desc(&echan->vchan);
 773                 if (!vdesc)
 774                         return;
 775                 list_del(&vdesc->node);
 776                 echan->edesc = to_edma_desc(&vdesc->tx);
 777         }
 778
 779         edesc = echan->edesc;
 780
 781         /* Find out how many left */
 782         left = edesc->pset_nr - edesc->processed;
 783         nslots = min(MAX_NR_SG, left);
 784         edesc->sg_len = 0;
 785
 786         /* Write descriptor PaRAM set(s) */
 787         for (i = 0; i < nslots; i++) {
 788                 j = i + edesc->processed;
 789                 edma_write_slot(ecc, echan->slot[i], &edesc->pset[j].param);
 790                 edesc->sg_len += edesc->pset[j].len;
 791                 dev_vdbg(dev,
 792                          "\n pset[%d]:\n"
 793                          "  chnum\t%d\n"
 794                          "  slot\t%d\n"
 795                          "  opt\t%08x\n"
 796                          "  src\t%08x\n"
 797                          "  dst\t%08x\n"
 798                          "  abcnt\t%08x\n"
 799                          "  ccnt\t%08x\n"
 800                          "  bidx\t%08x\n"
 801                          "  cidx\t%08x\n"
 802                          "  lkrld\t%08x\n",
 803                          j, echan->ch_num, echan->slot[i],
 804                          edesc->pset[j].param.opt,
 805                          edesc->pset[j].param.src,
 806                          edesc->pset[j].param.dst,
 807                          edesc->pset[j].param.a_b_cnt,
 808                          edesc->pset[j].param.ccnt,
 809                          edesc->pset[j].param.src_dst_bidx,
 810                          edesc->pset[j].param.src_dst_cidx,
 811                          edesc->pset[j].param.link_bcntrld);
 812                 /* Link to the previous slot if not the last set */
 813                 if (i != (nslots - 1))
 814                         edma_link(ecc, echan->slot[i], echan->slot[i + 1]);
 815         }
 816
 817         edesc->processed += nslots;
 818
 819         /*
 820          * If this is either the last set in a set of SG-list transactions
 821          * then setup a link to the dummy slot, this results in all future
 822          * events being absorbed and that's OK because we're done
 823          */
 824         if (edesc->processed == edesc->pset_nr) {
 825                 if (edesc->cyclic)
 826                         edma_link(ecc, echan->slot[nslots - 1], echan->slot[1]);
 827                 else
 828                         edma_link(ecc, echan->slot[nslots - 1],
 829                                   echan->ecc->dummy_slot);
 830         }
 831
 832         if (echan->missed) {
 833                 /*
 834                  * This happens due to setup times between intermediate
 835                  * transfers in long SG lists which have to be broken up into
 836                  * transfers of MAX_NR_SG
 837                  */
 838                 dev_dbg(dev, "missed event on channel %d\n", echan->ch_num);
 839                 edma_clean_channel(echan);
 840                 edma_stop(echan);
 841                 edma_start(echan);
 842                 edma_trigger_channel(echan);
 843                 echan->missed = 0;
 844         } else if (edesc->processed <= MAX_NR_SG) {
 845                 dev_dbg(dev, "first transfer starting on channel %d\n",
 846                         echan->ch_num);
 847                 edma_start(echan);
 848         } else {
 849                 dev_dbg(dev, "chan: %d: completed %d elements, resuming\n",
 850                         echan->ch_num, edesc->processed);
 851                 edma_resume(echan);
 852         }
 853 }
 854
 855 static int edma_terminate_all(struct dma_chan *chan)
 856 {
 857         struct edma_chan *echan = to_edma_chan(chan);
 858         unsigned long flags;
 859         LIST_HEAD(head);
 860
 861         spin_lock_irqsave(&echan->vchan.lock, flags);
 862
 863         /*
 864          * Stop DMA activity: we assume the callback will not be called
 865          * after edma_dma() returns (even if it does, it will see
 866          * echan->edesc is NULL and exit.)
 867          */
 868         if (echan->edesc) {
 869                 edma_stop(echan);
 870                 /* Move the cyclic channel back to default queue */
 871                 if (!echan->tc && echan->edesc->cyclic)
 872                         edma_assign_channel_eventq(echan, EVENTQ_DEFAULT);
 873
 874                 vchan_terminate_vdesc(&echan->edesc->vdesc);
 875                 echan->edesc = NULL;
 876         }
 877
 878         vchan_get_all_descriptors(&echan->vchan, &head);
 879         spin_unlock_irqrestore(&echan->vchan.lock, flags);
 880         vchan_dma_desc_free_list(&echan->vchan, &head);
 881
 882         return 0;
 883 }
 884
 885 static void edma_synchronize(struct dma_chan *chan)
 886 {
 887         struct edma_chan *echan = to_edma_chan(chan);
 888
 889         vchan_synchronize(&echan->vchan);
 890 }
 891
 892 static int edma_slave_config(struct dma_chan *chan,
 893         struct dma_slave_config *cfg)
 894 {
 895         struct edma_chan *echan = to_edma_chan(chan);
 896
 897         if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
 898             cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
 899                 return -EINVAL;
 900
 901         if (cfg->src_maxburst > chan->device->max_burst ||
 902             cfg->dst_maxburst > chan->device->max_burst)
 903                 return -EINVAL;
 904
 905         memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
 906
 907         return 0;
 908 }
 909
 910 static int edma_dma_pause(struct dma_chan *chan)
 911 {
 912         struct edma_chan *echan = to_edma_chan(chan);
 913
 914         if (!echan->edesc)
 915                 return -EINVAL;
 916
 917         edma_pause(echan);
 918         return 0;
 919 }
 920
 921 static int edma_dma_resume(struct dma_chan *chan)
 922 {
 923         struct edma_chan *echan = to_edma_chan(chan);
 924
 925         edma_resume(echan);
 926         return 0;
 927 }
 928
 929 /*
 930  * A PaRAM set configuration abstraction used by other modes
 931  * @chan: Channel who's PaRAM set we're configuring
 932  * @pset: PaRAM set to initialize and setup.
 933  * @src_addr: Source address of the DMA
 934  * @dst_addr: Destination address of the DMA
 935  * @burst: In units of dev_width, how much to send
 936  * @dev_width: How much is the dev_width
 937  * @dma_length: Total length of the DMA transfer
 938  * @direction: Direction of the transfer
 939  */
 940 static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
 941                             dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
 942                             unsigned int acnt, unsigned int dma_length,
 943                             enum dma_transfer_direction direction)
 944 {
 945         struct edma_chan *echan = to_edma_chan(chan);
 946         struct device *dev = chan->device->dev;
 947         struct edmacc_param *param = &epset->param;
 948         int bcnt, ccnt, cidx;
 949         int src_bidx, dst_bidx, src_cidx, dst_cidx;
 950         int absync;
 951
 952         /* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */
 953         if (!burst)
 954                 burst = 1;
 955         /*
 956          * If the maxburst is equal to the fifo width, use
 957          * A-synced transfers. This allows for large contiguous
 958          * buffer transfers using only one PaRAM set.
 959          */
 960         if (burst == 1) {
 961                 /*
 962                  * For the A-sync case, bcnt and ccnt are the remainder
 963                  * and quotient respectively of the division of:
 964                  * (dma_length / acnt) by (SZ_64K -1). This is so
 965                  * that in case bcnt over flows, we have ccnt to use.
 966                  * Note: In A-sync tranfer only, bcntrld is used, but it
 967                  * only applies for sg_dma_len(sg) >= SZ_64K.
 968                  * In this case, the best way adopted is- bccnt for the
 969                  * first frame will be the remainder below. Then for
 970                  * every successive frame, bcnt will be SZ_64K-1. This
 971                  * is assured as bcntrld = 0xffff in end of function.
 972                  */
 973                 absync = false;
 974                 ccnt = dma_length / acnt / (SZ_64K - 1);
 975                 bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
 976                 /*
 977                  * If bcnt is non-zero, we have a remainder and hence an
 978                  * extra frame to transfer, so increment ccnt.
 979                  */
 980                 if (bcnt)
 981                         ccnt++;
 982                 else
 983                         bcnt = SZ_64K - 1;
 984                 cidx = acnt;
 985         } else {
 986                 /*
 987                  * If maxburst is greater than the fifo address_width,
 988                  * use AB-synced transfers where A count is the fifo
 989                  * address_width and B count is the maxburst. In this
 990                  * case, we are limited to transfers of C count frames
 991                  * of (address_width * maxburst) where C count is limited
 992                  * to SZ_64K-1. This places an upper bound on the length
 993                  * of an SG segment that can be handled.
 994                  */
 995                 absync = true;
 996                 bcnt = burst;
 997                 ccnt = dma_length / (acnt * bcnt);
 998                 if (ccnt > (SZ_64K - 1)) {
 999                         dev_err(dev, "Exceeded max SG segment size\n");
1000                         return -EINVAL;
1001                 }
1002                 cidx = acnt * bcnt;
1003         }
1004
1005         epset->len = dma_length;
1006
1007         if (direction == DMA_MEM_TO_DEV) {
1008                 src_bidx = acnt;
1009                 src_cidx = cidx;
1010                 dst_bidx = 0;
1011                 dst_cidx = 0;
1012                 epset->addr = src_addr;
1013         } else if (direction == DMA_DEV_TO_MEM)  {
1014                 src_bidx = 0;
1015                 src_cidx = 0;
1016                 dst_bidx = acnt;
1017                 dst_cidx = cidx;
1018                 epset->addr = dst_addr;
1019         } else if (direction == DMA_MEM_TO_MEM)  {
1020                 src_bidx = acnt;
1021                 src_cidx = cidx;
1022                 dst_bidx = acnt;
1023                 dst_cidx = cidx;
1024                 epset->addr = src_addr;
1025         } else {
1026                 dev_err(dev, "%s: direction not implemented yet\n", __func__);
1027                 return -EINVAL;
1028         }
1029
1030         param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
1031         /* Configure A or AB synchronized transfers */
1032         if (absync)
1033                 param->opt |= SYNCDIM;
1034
1035         param->src = src_addr;
1036         param->dst = dst_addr;
1037
1038         param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
1039         param->src_dst_cidx = (dst_cidx << 16) | src_cidx;
1040
1041         param->a_b_cnt = bcnt << 16 | acnt;
1042         param->ccnt = ccnt;
1043         /*
1044          * Only time when (bcntrld) auto reload is required is for
1045          * A-sync case, and in this case, a requirement of reload value
1046          * of SZ_64K-1 only is assured. 'link' is initially set to NULL
1047          * and then later will be populated by edma_execute.
1048          */
1049         param->link_bcntrld = 0xffffffff;
1050         return absync;
1051 }
1052
1053 static struct dma_async_tx_descriptor *edma_prep_slave_sg(
1054         struct dma_chan *chan, struct scatterlist *sgl,
1055         unsigned int sg_len, enum dma_transfer_direction direction,
1056         unsigned long tx_flags, void *context)
1057 {
1058         struct edma_chan *echan = to_edma_chan(chan);
1059         struct device *dev = chan->device->dev;
1060         struct edma_desc *edesc;
1061         dma_addr_t src_addr = 0, dst_addr = 0;
1062         enum dma_slave_buswidth dev_width;
1063         u32 burst;
1064         struct scatterlist *sg;
1065         int i, nslots, ret;
1066
1067         if (unlikely(!echan || !sgl || !sg_len))
1068                 return NULL;
1069
1070         if (direction == DMA_DEV_TO_MEM) {
1071                 src_addr = echan->cfg.src_addr;
1072                 dev_width = echan->cfg.src_addr_width;
1073                 burst = echan->cfg.src_maxburst;
1074         } else if (direction == DMA_MEM_TO_DEV) {
1075                 dst_addr = echan->cfg.dst_addr;
1076                 dev_width = echan->cfg.dst_addr_width;
1077                 burst = echan->cfg.dst_maxburst;
1078         } else {
1079                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
1080                 return NULL;
1081         }
1082
1083         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
1084                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
1085                 return NULL;
1086         }
1087
1088         edesc = kzalloc(struct_size(edesc, pset, sg_len), GFP_ATOMIC);
1089         if (!edesc)
1090                 return NULL;
1091
1092         edesc->pset_nr = sg_len;
1093         edesc->residue = 0;
1094         edesc->direction = direction;
1095         edesc->echan = echan;
1096
1097         /* Allocate a PaRAM slot, if needed */
1098         nslots = min_t(unsigned, MAX_NR_SG, sg_len);
1099
1100         for (i = 0; i < nslots; i++) {
1101                 if (echan->slot[i] < 0) {
1102                         echan->slot[i] =
1103                                 edma_alloc_slot(echan->ecc, EDMA_SLOT_ANY);
1104                         if (echan->slot[i] < 0) {
1105                                 kfree(edesc);
1106                                 dev_err(dev, "%s: Failed to allocate slot\n",
1107                                         __func__);
1108                                 return NULL;
1109                         }
1110                 }
1111         }
1112
1113         /* Configure PaRAM sets for each SG */
1114         for_each_sg(sgl, sg, sg_len, i) {
1115                 /* Get address for each SG */
1116                 if (direction == DMA_DEV_TO_MEM)
1117                         dst_addr = sg_dma_address(sg);
1118                 else
1119                         src_addr = sg_dma_address(sg);
1120
1121                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
1122                                        dst_addr, burst, dev_width,
1123                                        sg_dma_len(sg), direction);
1124                 if (ret < 0) {
1125                         kfree(edesc);
1126                         return NULL;
1127                 }
1128
1129                 edesc->absync = ret;
1130                 edesc->residue += sg_dma_len(sg);
1131
1132                 if (i == sg_len - 1)
1133                         /* Enable completion interrupt */
1134                         edesc->pset[i].param.opt |= TCINTEN;
1135                 else if (!((i+1) % MAX_NR_SG))
1136                         /*
1137                          * Enable early completion interrupt for the
1138                          * intermediateset. In this case the driver will be
1139                          * notified when the paRAM set is submitted to TC. This
1140                          * will allow more time to set up the next set of slots.
1141                          */
1142                         edesc->pset[i].param.opt |= (TCINTEN | TCCMODE);
1143         }
1144         edesc->residue_stat = edesc->residue;
1145
1146         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
1147 }
1148
1149 static struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
1150         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1151         size_t len, unsigned long tx_flags)
1152 {
1153         int ret, nslots;
1154         struct edma_desc *edesc;
1155         struct device *dev = chan->device->dev;
1156         struct edma_chan *echan = to_edma_chan(chan);
1157         unsigned int width, pset_len, array_size;
1158
1159         if (unlikely(!echan || !len))
1160                 return NULL;
1161
1162         /* Align the array size (acnt block) with the transfer properties */
1163         switch (__ffs((src | dest | len))) {
1164         case 0:
1165                 array_size = SZ_32K - 1;
1166                 break;
1167         case 1:
1168                 array_size = SZ_32K - 2;
1169                 break;
1170         default:
1171                 array_size = SZ_32K - 4;
1172                 break;
1173         }
1174
1175         if (len < SZ_64K) {
1176                 /*
1177                  * Transfer size less than 64K can be handled with one paRAM
1178                  * slot and with one burst.
1179                  * ACNT = length
1180                  */
1181                 width = len;
1182                 pset_len = len;
1183                 nslots = 1;
1184         } else {
1185                 /*
1186                  * Transfer size bigger than 64K will be handled with maximum of
1187                  * two paRAM slots.
1188                  * slot1: (full_length / 32767) times 32767 bytes bursts.
1189                  *        ACNT = 32767, length1: (full_length / 32767) * 32767
1190                  * slot2: the remaining amount of data after slot1.
1191                  *        ACNT = full_length - length1, length2 = ACNT
1192                  *
1193                  * When the full_length is multibple of 32767 one slot can be
1194                  * used to complete the transfer.
1195                  */
1196                 width = array_size;
1197                 pset_len = rounddown(len, width);
1198                 /* One slot is enough for lengths multiple of (SZ_32K -1) */
1199                 if (unlikely(pset_len == len))
1200                         nslots = 1;
1201                 else
1202                         nslots = 2;
1203         }
1204
1205         edesc = kzalloc(struct_size(edesc, pset, nslots), GFP_ATOMIC);
1206         if (!edesc)
1207                 return NULL;
1208
1209         edesc->pset_nr = nslots;
1210         edesc->residue = edesc->residue_stat = len;
1211         edesc->direction = DMA_MEM_TO_MEM;
1212         edesc->echan = echan;
1213
1214         ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,
1215                                width, pset_len, DMA_MEM_TO_MEM);
1216         if (ret < 0) {
1217                 kfree(edesc);
1218                 return NULL;
1219         }
1220
1221         edesc->absync = ret;
1222
1223         edesc->pset[0].param.opt |= ITCCHEN;
1224         if (nslots == 1) {
1225                 /* Enable transfer complete interrupt if requested */
1226                 if (tx_flags & DMA_PREP_INTERRUPT)
1227                         edesc->pset[0].param.opt |= TCINTEN;
1228         } else {
1229                 /* Enable transfer complete chaining for the first slot */
1230                 edesc->pset[0].param.opt |= TCCHEN;
1231
1232                 if (echan->slot[1] < 0) {
1233                         echan->slot[1] = edma_alloc_slot(echan->ecc,
1234                                                          EDMA_SLOT_ANY);
1235                         if (echan->slot[1] < 0) {
1236                                 kfree(edesc);
1237                                 dev_err(dev, "%s: Failed to allocate slot\n",
1238                                         __func__);
1239                                 return NULL;
1240                         }
1241                 }
1242                 dest += pset_len;
1243                 src += pset_len;
1244                 pset_len = width = len % array_size;
1245
1246                 ret = edma_config_pset(chan, &edesc->pset[1], src, dest, 1,
1247                                        width, pset_len, DMA_MEM_TO_MEM);
1248                 if (ret < 0) {
1249                         kfree(edesc);
1250                         return NULL;
1251                 }
1252
1253                 edesc->pset[1].param.opt |= ITCCHEN;
1254                 /* Enable transfer complete interrupt if requested */
1255                 if (tx_flags & DMA_PREP_INTERRUPT)
1256                         edesc->pset[1].param.opt |= TCINTEN;
1257         }
1258
1259         if (!(tx_flags & DMA_PREP_INTERRUPT))
1260                 edesc->polled = true;
1261
1262         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
1263 }
1264
1265 static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
1266         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
1267         size_t period_len, enum dma_transfer_direction direction,
1268         unsigned long tx_flags)
1269 {
1270         struct edma_chan *echan = to_edma_chan(chan);
1271         struct device *dev = chan->device->dev;
1272         struct edma_desc *edesc;
1273         dma_addr_t src_addr, dst_addr;
1274         enum dma_slave_buswidth dev_width;
1275         bool use_intermediate = false;
1276         u32 burst;
1277         int i, ret, nslots;
1278
1279         if (unlikely(!echan || !buf_len || !period_len))
1280                 return NULL;
1281
1282         if (direction == DMA_DEV_TO_MEM) {
1283                 src_addr = echan->cfg.src_addr;
1284                 dst_addr = buf_addr;
1285                 dev_width = echan->cfg.src_addr_width;
1286                 burst = echan->cfg.src_maxburst;
1287         } else if (direction == DMA_MEM_TO_DEV) {
1288                 src_addr = buf_addr;
1289                 dst_addr = echan->cfg.dst_addr;
1290                 dev_width = echan->cfg.dst_addr_width;
1291                 burst = echan->cfg.dst_maxburst;
1292         } else {
1293                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
1294                 return NULL;
1295         }
1296
1297         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
1298                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
1299                 return NULL;
1300         }
1301
1302         if (unlikely(buf_len % period_len)) {
1303                 dev_err(dev, "Period should be multiple of Buffer length\n");
1304                 return NULL;
1305         }
1306
1307         nslots = (buf_len / period_len) + 1;
1308
1309         /*
1310          * Cyclic DMA users such as audio cannot tolerate delays introduced
1311          * by cases where the number of periods is more than the maximum
1312          * number of SGs the EDMA driver can handle at a time. For DMA types
1313          * such as Slave SGs, such delays are tolerable and synchronized,
1314          * but the synchronization is difficult to achieve with Cyclic and
1315          * cannot be guaranteed, so we error out early.
1316          */
1317         if (nslots > MAX_NR_SG) {
1318                 /*
1319                  * If the burst and period sizes are the same, we can put
1320                  * the full buffer into a single period and activate
1321                  * intermediate interrupts. This will produce interrupts
1322                  * after each burst, which is also after each desired period.
1323                  */
1324                 if (burst == period_len) {
1325                         period_len = buf_len;
1326                         nslots = 2;
1327                         use_intermediate = true;
1328                 } else {
1329                         return NULL;
1330                 }
1331         }
1332
1333         edesc = kzalloc(struct_size(edesc, pset, nslots), GFP_ATOMIC);
1334         if (!edesc)
1335                 return NULL;
1336
1337         edesc->cyclic = 1;
1338         edesc->pset_nr = nslots;
1339         edesc->residue = edesc->residue_stat = buf_len;
1340         edesc->direction = direction;
1341         edesc->echan = echan;
1342
1343         dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n",
1344                 __func__, echan->ch_num, nslots, period_len, buf_len);
1345
1346         for (i = 0; i < nslots; i++) {
1347                 /* Allocate a PaRAM slot, if needed */
1348                 if (echan->slot[i] < 0) {
1349                         echan->slot[i] =
1350                                 edma_alloc_slot(echan->ecc, EDMA_SLOT_ANY);
1351                         if (echan->slot[i] < 0) {
1352                                 kfree(edesc);
1353                                 dev_err(dev, "%s: Failed to allocate slot\n",
1354                                         __func__);
1355                                 return NULL;
1356                         }
1357                 }
1358
1359                 if (i == nslots - 1) {
1360                         memcpy(&edesc->pset[i], &edesc->pset[0],
1361                                sizeof(edesc->pset[0]));
1362                         break;
1363                 }
1364
1365                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
1366                                        dst_addr, burst, dev_width, period_len,
1367                                        direction);
1368                 if (ret < 0) {
1369                         kfree(edesc);
1370                         return NULL;
1371                 }
1372
1373                 if (direction == DMA_DEV_TO_MEM)
1374                         dst_addr += period_len;
1375                 else
1376                         src_addr += period_len;
1377
1378                 dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
1379                 dev_vdbg(dev,
1380                         "\n pset[%d]:\n"
1381                         "  chnum\t%d\n"
1382                         "  slot\t%d\n"
1383                         "  opt\t%08x\n"
1384                         "  src\t%08x\n"
1385                         "  dst\t%08x\n"
1386                         "  abcnt\t%08x\n"
1387                         "  ccnt\t%08x\n"
1388                         "  bidx\t%08x\n"
1389                         "  cidx\t%08x\n"
1390                         "  lkrld\t%08x\n",
1391                         i, echan->ch_num, echan->slot[i],
1392                         edesc->pset[i].param.opt,
1393                         edesc->pset[i].param.src,
1394                         edesc->pset[i].param.dst,
1395                         edesc->pset[i].param.a_b_cnt,
1396                         edesc->pset[i].param.ccnt,
1397                         edesc->pset[i].param.src_dst_bidx,
1398                         edesc->pset[i].param.src_dst_cidx,
1399                         edesc->pset[i].param.link_bcntrld);
1400
1401                 edesc->absync = ret;
1402
1403                 /*
1404                  * Enable period interrupt only if it is requested
1405                  */
1406                 if (tx_flags & DMA_PREP_INTERRUPT) {
1407                         edesc->pset[i].param.opt |= TCINTEN;
1408
1409                         /* Also enable intermediate interrupts if necessary */
1410                         if (use_intermediate)
1411                                 edesc->pset[i].param.opt |= ITCINTEN;
1412                 }
1413         }
1414
1415         /* Place the cyclic channel to highest priority queue */
1416         if (!echan->tc)
1417                 edma_assign_channel_eventq(echan, EVENTQ_0);
1418
1419         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
1420 }
1421
1422 static void edma_completion_handler(struct edma_chan *echan)
1423 {
1424         struct device *dev = echan->vchan.chan.device->dev;
1425         struct edma_desc *edesc;
1426
1427         spin_lock(&echan->vchan.lock);
1428         edesc = echan->edesc;
1429         if (edesc) {
1430                 if (edesc->cyclic) {
1431                         vchan_cyclic_callback(&edesc->vdesc);
1432                         spin_unlock(&echan->vchan.lock);
1433                         return;
1434                 } else if (edesc->processed == edesc->pset_nr) {
1435                         edesc->residue = 0;
1436                         edma_stop(echan);
1437                         vchan_cookie_complete(&edesc->vdesc);
1438                         echan->edesc = NULL;
1439
1440                         dev_dbg(dev, "Transfer completed on channel %d\n",
1441                                 echan->ch_num);
1442                 } else {
1443                         dev_dbg(dev, "Sub transfer completed on channel %d\n",
1444                                 echan->ch_num);
1445
1446                         edma_pause(echan);
1447
1448                         /* Update statistics for tx_status */
1449                         edesc->residue -= edesc->sg_len;
1450                         edesc->residue_stat = edesc->residue;
1451                         edesc->processed_stat = edesc->processed;
1452                 }
1453                 edma_execute(echan);
1454         }
1455
1456         spin_unlock(&echan->vchan.lock);
1457 }
1458
1459 /* eDMA interrupt handler */
1460 static irqreturn_t dma_irq_handler(int irq, void *data)
1461 {
1462         struct edma_cc *ecc = data;
1463         int ctlr;
1464         u32 sh_ier;
1465         u32 sh_ipr;
1466         u32 bank;
1467
1468         ctlr = ecc->id;
1469         if (ctlr < 0)
1470                 return IRQ_NONE;
1471
1472         dev_vdbg(ecc->dev, "dma_irq_handler\n");
1473
1474         sh_ipr = edma_shadow0_read_array(ecc, SH_IPR, 0);
1475         if (!sh_ipr) {
1476                 sh_ipr = edma_shadow0_read_array(ecc, SH_IPR, 1);
1477                 if (!sh_ipr)
1478                         return IRQ_NONE;
1479                 sh_ier = edma_shadow0_read_array(ecc, SH_IER, 1);
1480                 bank = 1;
1481         } else {
1482                 sh_ier = edma_shadow0_read_array(ecc, SH_IER, 0);
1483                 bank = 0;
1484         }
1485
1486         do {
1487                 u32 slot;
1488                 u32 channel;
1489
1490                 slot = __ffs(sh_ipr);
1491                 sh_ipr &= ~(BIT(slot));
1492
1493                 if (sh_ier & BIT(slot)) {
1494                         channel = (bank << 5) | slot;
1495                         /* Clear the corresponding IPR bits */
1496                         edma_shadow0_write_array(ecc, SH_ICR, bank, BIT(slot));
1497                         edma_completion_handler(&ecc->slave_chans[channel]);
1498                 }
1499         } while (sh_ipr);
1500
1501         edma_shadow0_write(ecc, SH_IEVAL, 1);
1502         return IRQ_HANDLED;
1503 }
1504
1505 static void edma_error_handler(struct edma_chan *echan)
1506 {
1507         struct edma_cc *ecc = echan->ecc;
1508         struct device *dev = echan->vchan.chan.device->dev;
1509         struct edmacc_param p;
1510         int err;
1511
1512         if (!echan->edesc)
1513                 return;
1514
1515         spin_lock(&echan->vchan.lock);
1516
1517         err = edma_read_slot(ecc, echan->slot[0], &p);
1518
1519         /*
1520          * Issue later based on missed flag which will be sure
1521          * to happen as:
1522          * (1) we finished transmitting an intermediate slot and
1523          *     edma_execute is coming up.
1524          * (2) or we finished current transfer and issue will
1525          *     call edma_execute.
1526          *
1527          * Important note: issuing can be dangerous here and
1528          * lead to some nasty recursion when we are in a NULL
1529          * slot. So we avoid doing so and set the missed flag.
1530          */
1531         if (err || (p.a_b_cnt == 0 && p.ccnt == 0)) {
1532                 dev_dbg(dev, "Error on null slot, setting miss\n");
1533                 echan->missed = 1;
1534         } else {
1535                 /*
1536                  * The slot is already programmed but the event got
1537                  * missed, so its safe to issue it here.
1538                  */
1539                 dev_dbg(dev, "Missed event, TRIGGERING\n");
1540                 edma_clean_channel(echan);
1541                 edma_stop(echan);
1542                 edma_start(echan);
1543                 edma_trigger_channel(echan);
1544         }
1545         spin_unlock(&echan->vchan.lock);
1546 }
1547
1548 static inline bool edma_error_pending(struct edma_cc *ecc)
1549 {
1550         if (edma_read_array(ecc, EDMA_EMR, 0) ||
1551             edma_read_array(ecc, EDMA_EMR, 1) ||
1552             edma_read(ecc, EDMA_QEMR) || edma_read(ecc, EDMA_CCERR))
1553                 return true;
1554
1555         return false;
1556 }
1557
1558 /* eDMA error interrupt handler */
1559 static irqreturn_t dma_ccerr_handler(int irq, void *data)
1560 {
1561         struct edma_cc *ecc = data;
1562         int i, j;
1563         int ctlr;
1564         unsigned int cnt = 0;
1565         unsigned int val;
1566
1567         ctlr = ecc->id;
1568         if (ctlr < 0)
1569                 return IRQ_NONE;
1570
1571         dev_vdbg(ecc->dev, "dma_ccerr_handler\n");
1572
1573         if (!edma_error_pending(ecc)) {
1574                 /*
1575                  * The registers indicate no pending error event but the irq
1576                  * handler has been called.
1577                  * Ask eDMA to re-evaluate the error registers.
1578                  */
1579                 dev_err(ecc->dev, "%s: Error interrupt without error event!\n",
1580                         __func__);
1581                 edma_write(ecc, EDMA_EEVAL, 1);
1582                 return IRQ_NONE;
1583         }
1584
1585         while (1) {
1586                 /* Event missed register(s) */
1587                 for (j = 0; j < 2; j++) {
1588                         unsigned long emr;
1589
1590                         val = edma_read_array(ecc, EDMA_EMR, j);
1591                         if (!val)
1592                                 continue;
1593
1594                         dev_dbg(ecc->dev, "EMR%d 0x%08x\n", j, val);
1595                         emr = val;
1596                         for (i = find_next_bit(&emr, 32, 0); i < 32;
1597                              i = find_next_bit(&emr, 32, i + 1)) {
1598                                 int k = (j << 5) + i;
1599
1600                                 /* Clear the corresponding EMR bits */
1601                                 edma_write_array(ecc, EDMA_EMCR, j, BIT(i));
1602                                 /* Clear any SER */
1603                                 edma_shadow0_write_array(ecc, SH_SECR, j,
1604                                                          BIT(i));
1605                                 edma_error_handler(&ecc->slave_chans[k]);
1606                         }
1607                 }
1608
1609                 val = edma_read(ecc, EDMA_QEMR);
1610                 if (val) {
1611                         dev_dbg(ecc->dev, "QEMR 0x%02x\n", val);
1612                         /* Not reported, just clear the interrupt reason. */
1613                         edma_write(ecc, EDMA_QEMCR, val);
1614                         edma_shadow0_write(ecc, SH_QSECR, val);
1615                 }
1616
1617                 val = edma_read(ecc, EDMA_CCERR);
1618                 if (val) {
1619                         dev_warn(ecc->dev, "CCERR 0x%08x\n", val);
1620                         /* Not reported, just clear the interrupt reason. */
1621                         edma_write(ecc, EDMA_CCERRCLR, val);
1622                 }
1623
1624                 if (!edma_error_pending(ecc))
1625                         break;
1626                 cnt++;
1627                 if (cnt > 10)
1628                         break;
1629         }
1630         edma_write(ecc, EDMA_EEVAL, 1);
1631         return IRQ_HANDLED;
1632 }
1633
1634 /* Alloc channel resources */
1635 static int edma_alloc_chan_resources(struct dma_chan *chan)
1636 {
1637         struct edma_chan *echan = to_edma_chan(chan);
1638         struct edma_cc *ecc = echan->ecc;
1639         struct device *dev = ecc->dev;
1640         enum dma_event_q eventq_no = EVENTQ_DEFAULT;
1641         int ret;
1642
1643         if (echan->tc) {
1644                 eventq_no = echan->tc->id;
1645         } else if (ecc->tc_list) {
1646                 /* memcpy channel */
1647                 echan->tc = &ecc->tc_list[ecc->info->default_queue];
1648                 eventq_no = echan->tc->id;
1649         }
1650
1651         ret = edma_alloc_channel(echan, eventq_no);
1652         if (ret)
1653                 return ret;
1654
1655         echan->slot[0] = edma_alloc_slot(ecc, echan->ch_num);
1656         if (echan->slot[0] < 0) {
1657                 dev_err(dev, "Entry slot allocation failed for channel %u\n",
1658                         EDMA_CHAN_SLOT(echan->ch_num));
1659                 ret = echan->slot[0];
1660                 goto err_slot;
1661         }
1662
1663         /* Set up channel -> slot mapping for the entry slot */
1664         edma_set_chmap(echan, echan->slot[0]);
1665         echan->alloced = true;
1666
1667         dev_dbg(dev, "Got eDMA channel %d for virt channel %d (%s trigger)\n",
1668                 EDMA_CHAN_SLOT(echan->ch_num), chan->chan_id,
1669                 echan->hw_triggered ? "HW" : "SW");
1670
1671         return 0;
1672
1673 err_slot:
1674         edma_free_channel(echan);
1675         return ret;
1676 }
1677
1678 /* Free channel resources */
1679 static void edma_free_chan_resources(struct dma_chan *chan)
1680 {
1681         struct edma_chan *echan = to_edma_chan(chan);
1682         struct device *dev = echan->ecc->dev;
1683         int i;
1684
1685         /* Terminate transfers */
1686         edma_stop(echan);
1687
1688         vchan_free_chan_resources(&echan->vchan);
1689
1690         /* Free EDMA PaRAM slots */
1691         for (i = 0; i < EDMA_MAX_SLOTS; i++) {
1692                 if (echan->slot[i] >= 0) {
1693                         edma_free_slot(echan->ecc, echan->slot[i]);
1694                         echan->slot[i] = -1;
1695                 }
1696         }
1697
1698         /* Set entry slot to the dummy slot */
1699         edma_set_chmap(echan, echan->ecc->dummy_slot);
1700
1701         /* Free EDMA channel */
1702         if (echan->alloced) {
1703                 edma_free_channel(echan);
1704                 echan->alloced = false;
1705         }
1706
1707         echan->tc = NULL;
1708         echan->hw_triggered = false;
1709
1710         dev_dbg(dev, "Free eDMA channel %d for virt channel %d\n",
1711                 EDMA_CHAN_SLOT(echan->ch_num), chan->chan_id);
1712 }
1713
1714 /* Send pending descriptor to hardware */
1715 static void edma_issue_pending(struct dma_chan *chan)
1716 {
1717         struct edma_chan *echan = to_edma_chan(chan);
1718         unsigned long flags;
1719
1720         spin_lock_irqsave(&echan->vchan.lock, flags);
1721         if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
1722                 edma_execute(echan);
1723         spin_unlock_irqrestore(&echan->vchan.lock, flags);
1724 }
1725
1726 /*
1727  * This limit exists to avoid a possible infinite loop when waiting for proof
1728  * that a particular transfer is completed. This limit can be hit if there
1729  * are large bursts to/from slow devices or the CPU is never able to catch
1730  * the DMA hardware idle. On an AM335x transfering 48 bytes from the UART
1731  * RX-FIFO, as many as 55 loops have been seen.
1732  */
1733 #define EDMA_MAX_TR_WAIT_LOOPS 1000
1734
1735 static u32 edma_residue(struct edma_desc *edesc)
1736 {
1737         bool dst = edesc->direction == DMA_DEV_TO_MEM;
1738         int loop_count = EDMA_MAX_TR_WAIT_LOOPS;
1739         struct edma_chan *echan = edesc->echan;
1740         struct edma_pset *pset = edesc->pset;
1741         dma_addr_t done, pos, pos_old;
1742         int channel = EDMA_CHAN_SLOT(echan->ch_num);
1743         int idx = EDMA_REG_ARRAY_INDEX(channel);
1744         int ch_bit = EDMA_CHANNEL_BIT(channel);
1745         int event_reg;
1746         int i;
1747
1748         /*
1749          * We always read the dst/src position from the first RamPar
1750          * pset. That's the one which is active now.
1751          */
1752         pos = edma_get_position(echan->ecc, echan->slot[0], dst);
1753
1754         /*
1755          * "pos" may represent a transfer request that is still being
1756          * processed by the EDMACC or EDMATC. We will busy wait until
1757          * any one of the situations occurs:
1758          *   1. while and event is pending for the channel
1759          *   2. a position updated
1760          *   3. we hit the loop limit
1761          */
1762         if (is_slave_direction(edesc->direction))
1763                 event_reg = SH_ER;
1764         else
1765                 event_reg = SH_ESR;
1766
1767         pos_old = pos;
1768         while (edma_shadow0_read_array(echan->ecc, event_reg, idx) & ch_bit) {
1769                 pos = edma_get_position(echan->ecc, echan->slot[0], dst);
1770                 if (pos != pos_old)
1771                         break;
1772
1773                 if (!--loop_count) {
1774                         dev_dbg_ratelimited(echan->vchan.chan.device->dev,
1775                                 "%s: timeout waiting for PaRAM update\n",
1776                                 __func__);
1777                         break;
1778                 }
1779
1780                 cpu_relax();
1781         }
1782
1783         /*
1784          * Cyclic is simple. Just subtract pset[0].addr from pos.
1785          *
1786          * We never update edesc->residue in the cyclic case, so we
1787          * can tell the remaining room to the end of the circular
1788          * buffer.
1789          */
1790         if (edesc->cyclic) {
1791                 done = pos - pset->addr;
1792                 edesc->residue_stat = edesc->residue - done;
1793                 return edesc->residue_stat;
1794         }
1795
1796         /*
1797          * If the position is 0, then EDMA loaded the closing dummy slot, the
1798          * transfer is completed
1799          */
1800         if (!pos)
1801                 return 0;
1802         /*
1803          * For SG operation we catch up with the last processed
1804          * status.
1805          */
1806         pset += edesc->processed_stat;
1807
1808         for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {
1809                 /*
1810                  * If we are inside this pset address range, we know
1811                  * this is the active one. Get the current delta and
1812                  * stop walking the psets.
1813                  */
1814                 if (pos >= pset->addr && pos < pset->addr + pset->len)
1815                         return edesc->residue_stat - (pos - pset->addr);
1816
1817                 /* Otherwise mark it done and update residue_stat. */
1818                 edesc->processed_stat++;
1819                 edesc->residue_stat -= pset->len;
1820         }
1821         return edesc->residue_stat;
1822 }
1823
1824 /* Check request completion status */
1825 static enum dma_status edma_tx_status(struct dma_chan *chan,
1826                                       dma_cookie_t cookie,
1827                                       struct dma_tx_state *txstate)
1828 {
1829         struct edma_chan *echan = to_edma_chan(chan);
1830         struct dma_tx_state txstate_tmp;
1831         enum dma_status ret;
1832         unsigned long flags;
1833
1834         ret = dma_cookie_status(chan, cookie, txstate);
1835
1836         if (ret == DMA_COMPLETE)
1837                 return ret;
1838
1839         /* Provide a dummy dma_tx_state for completion checking */
1840         if (!txstate)
1841                 txstate = &txstate_tmp;
1842
1843         spin_lock_irqsave(&echan->vchan.lock, flags);
1844         if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie) {
1845                 txstate->residue = edma_residue(echan->edesc);
1846         } else {
1847                 struct virt_dma_desc *vdesc = vchan_find_desc(&echan->vchan,
1848                                                               cookie);
1849
1850                 if (vdesc)
1851                         txstate->residue = to_edma_desc(&vdesc->tx)->residue;
1852                 else
1853                         txstate->residue = 0;
1854         }
1855
1856         /*
1857          * Mark the cookie completed if the residue is 0 for non cyclic
1858          * transfers
1859          */
1860         if (ret != DMA_COMPLETE && !txstate->residue &&
1861             echan->edesc && echan->edesc->polled &&
1862             echan->edesc->vdesc.tx.cookie == cookie) {
1863                 edma_stop(echan);
1864                 vchan_cookie_complete(&echan->edesc->vdesc);
1865                 echan->edesc = NULL;
1866                 edma_execute(echan);
1867                 ret = DMA_COMPLETE;
1868         }
1869
1870         spin_unlock_irqrestore(&echan->vchan.lock, flags);
1871
1872         return ret;
1873 }
1874
1875 static bool edma_is_memcpy_channel(int ch_num, s32 *memcpy_channels)
1876 {
1877         if (!memcpy_channels)
1878                 return false;
1879         while (*memcpy_channels != -1) {
1880                 if (*memcpy_channels == ch_num)
1881                         return true;
1882                 memcpy_channels++;
1883         }
1884         return false;
1885 }
1886
1887 #define EDMA_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
1888                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
1889                                  BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
1890                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
1891
1892 static void edma_dma_init(struct edma_cc *ecc, bool legacy_mode)
1893 {
1894         struct dma_device *s_ddev = &ecc->dma_slave;
1895         struct dma_device *m_ddev = NULL;
1896         s32 *memcpy_channels = ecc->info->memcpy_channels;
1897         int i, j;
1898
1899         dma_cap_zero(s_ddev->cap_mask);
1900         dma_cap_set(DMA_SLAVE, s_ddev->cap_mask);
1901         dma_cap_set(DMA_CYCLIC, s_ddev->cap_mask);
1902         if (ecc->legacy_mode && !memcpy_channels) {
1903                 dev_warn(ecc->dev,
1904                          "Legacy memcpy is enabled, things might not work\n");
1905
1906                 dma_cap_set(DMA_MEMCPY, s_ddev->cap_mask);
1907                 s_ddev->device_prep_dma_memcpy = edma_prep_dma_memcpy;
1908                 s_ddev->directions = BIT(DMA_MEM_TO_MEM);
1909         }
1910
1911         s_ddev->device_prep_slave_sg = edma_prep_slave_sg;
1912         s_ddev->device_prep_dma_cyclic = edma_prep_dma_cyclic;
1913         s_ddev->device_alloc_chan_resources = edma_alloc_chan_resources;
1914         s_ddev->device_free_chan_resources = edma_free_chan_resources;
1915         s_ddev->device_issue_pending = edma_issue_pending;
1916         s_ddev->device_tx_status = edma_tx_status;
1917         s_ddev->device_config = edma_slave_config;
1918         s_ddev->device_pause = edma_dma_pause;
1919         s_ddev->device_resume = edma_dma_resume;
1920         s_ddev->device_terminate_all = edma_terminate_all;
1921         s_ddev->device_synchronize = edma_synchronize;
1922
1923         s_ddev->src_addr_widths = EDMA_DMA_BUSWIDTHS;
1924         s_ddev->dst_addr_widths = EDMA_DMA_BUSWIDTHS;
1925         s_ddev->directions |= (BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV));
1926         s_ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1927         s_ddev->max_burst = SZ_32K - 1; /* CIDX: 16bit signed */
1928
1929         s_ddev->dev = ecc->dev;
1930         INIT_LIST_HEAD(&s_ddev->channels);
1931
1932         if (memcpy_channels) {
1933                 m_ddev = devm_kzalloc(ecc->dev, sizeof(*m_ddev), GFP_KERNEL);
1934                 if (!m_ddev) {
1935                         dev_warn(ecc->dev, "memcpy is disabled due to OoM\n");
1936                         memcpy_channels = NULL;
1937                         goto ch_setup;
1938                 }
1939                 ecc->dma_memcpy = m_ddev;
1940
1941                 dma_cap_zero(m_ddev->cap_mask);
1942                 dma_cap_set(DMA_MEMCPY, m_ddev->cap_mask);
1943
1944                 m_ddev->device_prep_dma_memcpy = edma_prep_dma_memcpy;
1945                 m_ddev->device_alloc_chan_resources = edma_alloc_chan_resources;
1946                 m_ddev->device_free_chan_resources = edma_free_chan_resources;
1947                 m_ddev->device_issue_pending = edma_issue_pending;
1948                 m_ddev->device_tx_status = edma_tx_status;
1949                 m_ddev->device_config = edma_slave_config;
1950                 m_ddev->device_pause = edma_dma_pause;
1951                 m_ddev->device_resume = edma_dma_resume;
1952                 m_ddev->device_terminate_all = edma_terminate_all;
1953                 m_ddev->device_synchronize = edma_synchronize;
1954
1955                 m_ddev->src_addr_widths = EDMA_DMA_BUSWIDTHS;
1956                 m_ddev->dst_addr_widths = EDMA_DMA_BUSWIDTHS;
1957                 m_ddev->directions = BIT(DMA_MEM_TO_MEM);
1958                 m_ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1959
1960                 m_ddev->dev = ecc->dev;
1961                 INIT_LIST_HEAD(&m_ddev->channels);
1962         } else if (!ecc->legacy_mode) {
1963                 dev_info(ecc->dev, "memcpy is disabled\n");
1964         }
1965
1966 ch_setup:
1967         for (i = 0; i < ecc->num_channels; i++) {
1968                 struct edma_chan *echan = &ecc->slave_chans[i];
1969                 echan->ch_num = EDMA_CTLR_CHAN(ecc->id, i);
1970                 echan->ecc = ecc;
1971                 echan->vchan.desc_free = edma_desc_free;
1972
1973                 if (m_ddev && edma_is_memcpy_channel(i, memcpy_channels))
1974                         vchan_init(&echan->vchan, m_ddev);
1975                 else
1976                         vchan_init(&echan->vchan, s_ddev);
1977
1978                 INIT_LIST_HEAD(&echan->node);
1979                 for (j = 0; j < EDMA_MAX_SLOTS; j++)
1980                         echan->slot[j] = -1;
1981         }
1982 }
1983
1984 static int edma_setup_from_hw(struct device *dev, struct edma_soc_info *pdata,
1985                               struct edma_cc *ecc)
1986 {
1987         int i;
1988         u32 value, cccfg;
1989         s8 (*queue_priority_map)[2];
1990
1991         /* Decode the eDMA3 configuration from CCCFG register */
1992         cccfg = edma_read(ecc, EDMA_CCCFG);
1993
1994         value = GET_NUM_REGN(cccfg);
1995         ecc->num_region = BIT(value);
1996
1997         value = GET_NUM_DMACH(cccfg);
1998         ecc->num_channels = BIT(value + 1);
1999
2000         value = GET_NUM_QDMACH(cccfg);
2001         ecc->num_qchannels = value * 2;
2002
2003         value = GET_NUM_PAENTRY(cccfg);
2004         ecc->num_slots = BIT(value + 4);
2005
2006         value = GET_NUM_EVQUE(cccfg);
2007         ecc->num_tc = value + 1;
2008
2009         ecc->chmap_exist = (cccfg & CHMAP_EXIST) ? true : false;
2010
2011         dev_dbg(dev, "eDMA3 CC HW configuration (cccfg: 0x%08x):\n", cccfg);
2012         dev_dbg(dev, "num_region: %u\n", ecc->num_region);
2013         dev_dbg(dev, "num_channels: %u\n", ecc->num_channels);
2014         dev_dbg(dev, "num_qchannels: %u\n", ecc->num_qchannels);
2015         dev_dbg(dev, "num_slots: %u\n", ecc->num_slots);
2016         dev_dbg(dev, "num_tc: %u\n", ecc->num_tc);
2017         dev_dbg(dev, "chmap_exist: %s\n", ecc->chmap_exist ? "yes" : "no");
2018
2019         /* Nothing need to be done if queue priority is provided */
2020         if (pdata->queue_priority_mapping)
2021                 return 0;
2022
2023         /*
2024          * Configure TC/queue priority as follows:
2025          * Q0 - priority 0
2026          * Q1 - priority 1
2027          * Q2 - priority 2
2028          * ...
2029          * The meaning of priority numbers: 0 highest priority, 7 lowest
2030          * priority. So Q0 is the highest priority queue and the last queue has
2031          * the lowest priority.
2032          */
2033         queue_priority_map = devm_kcalloc(dev, ecc->num_tc + 1, sizeof(s8),
2034                                           GFP_KERNEL);
2035         if (!queue_priority_map)
2036                 return -ENOMEM;
2037
2038         for (i = 0; i < ecc->num_tc; i++) {
2039                 queue_priority_map[i][0] = i;
2040                 queue_priority_map[i][1] = i;
2041         }
2042         queue_priority_map[i][0] = -1;
2043         queue_priority_map[i][1] = -1;
2044
2045         pdata->queue_priority_mapping = queue_priority_map;
2046         /* Default queue has the lowest priority */
2047         pdata->default_queue = i - 1;
2048
2049         return 0;
2050 }
2051
2052 #if IS_ENABLED(CONFIG_OF)
2053 static int edma_xbar_event_map(struct device *dev, struct edma_soc_info *pdata,
2054                                size_t sz)
2055 {
2056         const char pname[] = "ti,edma-xbar-event-map";
2057         struct resource res;
2058         void __iomem *xbar;
2059         s16 (*xbar_chans)[2];
2060         size_t nelm = sz / sizeof(s16);
2061         u32 shift, offset, mux;
2062         int ret, i;
2063
2064         xbar_chans = devm_kcalloc(dev, nelm + 2, sizeof(s16), GFP_KERNEL);
2065         if (!xbar_chans)
2066                 return -ENOMEM;
2067
2068         ret = of_address_to_resource(dev->of_node, 1, &res);
2069         if (ret)
2070                 return -ENOMEM;
2071
2072         xbar = devm_ioremap(dev, res.start, resource_size(&res));
2073         if (!xbar)
2074                 return -ENOMEM;
2075
2076         ret = of_property_read_u16_array(dev->of_node, pname, (u16 *)xbar_chans,
2077                                          nelm);
2078         if (ret)
2079                 return -EIO;
2080
2081         /* Invalidate last entry for the other user of this mess */
2082         nelm >>= 1;
2083         xbar_chans[nelm][0] = -1;
2084         xbar_chans[nelm][1] = -1;
2085
2086         for (i = 0; i < nelm; i++) {
2087                 shift = (xbar_chans[i][1] & 0x03) << 3;
2088                 offset = xbar_chans[i][1] & 0xfffffffc;
2089                 mux = readl(xbar + offset);
2090                 mux &= ~(0xff << shift);
2091                 mux |= xbar_chans[i][0] << shift;
2092                 writel(mux, (xbar + offset));
2093         }
2094
2095         pdata->xbar_chans = (const s16 (*)[2]) xbar_chans;
2096         return 0;
2097 }
2098
2099 static struct edma_soc_info *edma_setup_info_from_dt(struct device *dev,
2100                                                      bool legacy_mode)
2101 {
2102         struct edma_soc_info *info;
2103         struct property *prop;
2104         int sz, ret;
2105
2106         info = devm_kzalloc(dev, sizeof(struct edma_soc_info), GFP_KERNEL);
2107         if (!info)
2108                 return ERR_PTR(-ENOMEM);
2109
2110         if (legacy_mode) {
2111                 prop = of_find_property(dev->of_node, "ti,edma-xbar-event-map",
2112                                         &sz);
2113                 if (prop) {
2114                         ret = edma_xbar_event_map(dev, info, sz);
2115                         if (ret)
2116                                 return ERR_PTR(ret);
2117                 }
2118                 return info;
2119         }
2120
2121         /* Get the list of channels allocated to be used for memcpy */
2122         prop = of_find_property(dev->of_node, "ti,edma-memcpy-channels", &sz);
2123         if (prop) {
2124                 const char pname[] = "ti,edma-memcpy-channels";
2125                 size_t nelm = sz / sizeof(s32);
2126                 s32 *memcpy_ch;
2127
2128                 memcpy_ch = devm_kcalloc(dev, nelm + 1, sizeof(s32),
2129                                          GFP_KERNEL);
2130                 if (!memcpy_ch)
2131                         return ERR_PTR(-ENOMEM);
2132
2133                 ret = of_property_read_u32_array(dev->of_node, pname,
2134                                                  (u32 *)memcpy_ch, nelm);
2135                 if (ret)
2136                         return ERR_PTR(ret);
2137
2138                 memcpy_ch[nelm] = -1;
2139                 info->memcpy_channels = memcpy_ch;
2140         }
2141
2142         prop = of_find_property(dev->of_node, "ti,edma-reserved-slot-ranges",
2143                                 &sz);
2144         if (prop) {
2145                 const char pname[] = "ti,edma-reserved-slot-ranges";
2146                 u32 (*tmp)[2];
2147                 s16 (*rsv_slots)[2];
2148                 size_t nelm = sz / sizeof(*tmp);
2149                 struct edma_rsv_info *rsv_info;
2150                 int i;
2151
2152                 if (!nelm)
2153                         return info;
2154
2155                 tmp = kcalloc(nelm, sizeof(*tmp), GFP_KERNEL);
2156                 if (!tmp)
2157                         return ERR_PTR(-ENOMEM);
2158
2159                 rsv_info = devm_kzalloc(dev, sizeof(*rsv_info), GFP_KERNEL);
2160                 if (!rsv_info) {
2161                         kfree(tmp);
2162                         return ERR_PTR(-ENOMEM);
2163                 }
2164
2165                 rsv_slots = devm_kcalloc(dev, nelm + 1, sizeof(*rsv_slots),
2166                                          GFP_KERNEL);
2167                 if (!rsv_slots) {
2168                         kfree(tmp);
2169                         return ERR_PTR(-ENOMEM);
2170                 }
2171
2172                 ret = of_property_read_u32_array(dev->of_node, pname,
2173                                                  (u32 *)tmp, nelm * 2);
2174                 if (ret) {
2175                         kfree(tmp);
2176                         return ERR_PTR(ret);
2177                 }
2178
2179                 for (i = 0; i < nelm; i++) {
2180                         rsv_slots[i][0] = tmp[i][0];
2181                         rsv_slots[i][1] = tmp[i][1];
2182                 }
2183                 rsv_slots[nelm][0] = -1;
2184                 rsv_slots[nelm][1] = -1;
2185
2186                 info->rsv = rsv_info;
2187                 info->rsv->rsv_slots = (const s16 (*)[2])rsv_slots;
2188
2189                 kfree(tmp);
2190         }
2191
2192         return info;
2193 }
2194
2195 static struct dma_chan *of_edma_xlate(struct of_phandle_args *dma_spec,
2196                                       struct of_dma *ofdma)
2197 {
2198         struct edma_cc *ecc = ofdma->of_dma_data;
2199         struct dma_chan *chan = NULL;
2200         struct edma_chan *echan;
2201         int i;
2202
2203         if (!ecc || dma_spec->args_count < 1)
2204                 return NULL;
2205
2206         for (i = 0; i < ecc->num_channels; i++) {
2207                 echan = &ecc->slave_chans[i];
2208                 if (echan->ch_num == dma_spec->args[0]) {
2209                         chan = &echan->vchan.chan;
2210                         break;
2211                 }
2212         }
2213
2214         if (!chan)
2215                 return NULL;
2216
2217         if (echan->ecc->legacy_mode && dma_spec->args_count == 1)
2218                 goto out;
2219
2220         if (!echan->ecc->legacy_mode && dma_spec->args_count == 2 &&
2221             dma_spec->args[1] < echan->ecc->num_tc) {
2222                 echan->tc = &echan->ecc->tc_list[dma_spec->args[1]];
2223                 goto out;
2224         }
2225
2226         return NULL;
2227 out:
2228         /* The channel is going to be used as HW synchronized */
2229         echan->hw_triggered = true;
2230         return dma_get_slave_channel(chan);
2231 }
2232 #else
2233 static struct edma_soc_info *edma_setup_info_from_dt(struct device *dev,
2234                                                      bool legacy_mode)
2235 {
2236         return ERR_PTR(-EINVAL);
2237 }
2238
2239 static struct dma_chan *of_edma_xlate(struct of_phandle_args *dma_spec,
2240                                       struct of_dma *ofdma)
2241 {
2242         return NULL;
2243 }
2244 #endif
2245
2246 static bool edma_filter_fn(struct dma_chan *chan, void *param);
2247
2248 static int edma_probe(struct platform_device *pdev)
2249 {
2250         struct edma_soc_info    *info = pdev->dev.platform_data;
2251         s8                      (*queue_priority_mapping)[2];
2252         int                     i, off;
2253         const s16               (*rsv_slots)[2];
2254         const s16               (*xbar_chans)[2];
2255         int                     irq;
2256         char                    *irq_name;
2257         struct resource         *mem;
2258         struct device_node      *node = pdev->dev.of_node;
2259         struct device           *dev = &pdev->dev;
2260         struct edma_cc          *ecc;
2261         bool                    legacy_mode = true;
2262         int ret;
2263
2264         if (node) {
2265                 const struct of_device_id *match;
2266
2267                 match = of_match_node(edma_of_ids, node);
2268                 if (match && (*(u32 *)match->data) == EDMA_BINDING_TPCC)
2269                         legacy_mode = false;
2270
2271                 info = edma_setup_info_from_dt(dev, legacy_mode);
2272                 if (IS_ERR(info)) {
2273                         dev_err(dev, "failed to get DT data\n");
2274                         return PTR_ERR(info);
2275                 }
2276         }
2277
2278         if (!info)
2279                 return -ENODEV;
2280
2281         pm_runtime_enable(dev);
2282         ret = pm_runtime_get_sync(dev);
2283         if (ret < 0) {
2284                 dev_err(dev, "pm_runtime_get_sync() failed\n");
2285                 return ret;
2286         }
2287
2288         ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
2289         if (ret)
2290                 return ret;
2291
2292         ecc = devm_kzalloc(dev, sizeof(*ecc), GFP_KERNEL);
2293         if (!ecc)
2294                 return -ENOMEM;
2295
2296         ecc->dev = dev;
2297         ecc->id = pdev->id;
2298         ecc->legacy_mode = legacy_mode;
2299         /* When booting with DT the pdev->id is -1 */
2300         if (ecc->id < 0)
2301                 ecc->id = 0;
2302
2303         mem = platform_get_resource_byname(pdev, IORESOURCE_MEM, "edma3_cc");
2304         if (!mem) {
2305                 dev_dbg(dev, "mem resource not found, using index 0\n");
2306                 mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2307                 if (!mem) {
2308                         dev_err(dev, "no mem resource?\n");
2309                         return -ENODEV;
2310                 }
2311         }
2312         ecc->base = devm_ioremap_resource(dev, mem);
2313         if (IS_ERR(ecc->base))
2314                 return PTR_ERR(ecc->base);
2315
2316         platform_set_drvdata(pdev, ecc);
2317
2318         /* Get eDMA3 configuration from IP */
2319         ret = edma_setup_from_hw(dev, info, ecc);
2320         if (ret)
2321                 return ret;
2322
2323         /* Allocate memory based on the information we got from the IP */
2324         ecc->slave_chans = devm_kcalloc(dev, ecc->num_channels,
2325                                         sizeof(*ecc->slave_chans), GFP_KERNEL);
2326         if (!ecc->slave_chans)
2327                 return -ENOMEM;
2328
2329         ecc->slot_inuse = devm_kcalloc(dev, BITS_TO_LONGS(ecc->num_slots),
2330                                        sizeof(unsigned long), GFP_KERNEL);
2331         if (!ecc->slot_inuse)
2332                 return -ENOMEM;
2333
2334         ecc->default_queue = info->default_queue;
2335
2336         if (info->rsv) {
2337                 /* Set the reserved slots in inuse list */
2338                 rsv_slots = info->rsv->rsv_slots;
2339                 if (rsv_slots) {
2340                         for (i = 0; rsv_slots[i][0] != -1; i++)
2341                                 bitmap_set(ecc->slot_inuse, rsv_slots[i][0],
2342                                            rsv_slots[i][1]);
2343                 }
2344         }
2345
2346         for (i = 0; i < ecc->num_slots; i++) {
2347                 /* Reset only unused - not reserved - paRAM slots */
2348                 if (!test_bit(i, ecc->slot_inuse))
2349                         edma_write_slot(ecc, i, &dummy_paramset);
2350         }
2351
2352         /* Clear the xbar mapped channels in unused list */
2353         xbar_chans = info->xbar_chans;
2354         if (xbar_chans) {
2355                 for (i = 0; xbar_chans[i][1] != -1; i++) {
2356                         off = xbar_chans[i][1];
2357                 }
2358         }
2359
2360         irq = platform_get_irq_byname(pdev, "edma3_ccint");
2361         if (irq < 0 && node)
2362                 irq = irq_of_parse_and_map(node, 0);
2363
2364         if (irq >= 0) {
2365                 irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_ccint",
2366                                           dev_name(dev));
2367                 ret = devm_request_irq(dev, irq, dma_irq_handler, 0, irq_name,
2368                                        ecc);
2369                 if (ret) {
2370                         dev_err(dev, "CCINT (%d) failed --> %d\n", irq, ret);
2371                         return ret;
2372                 }
2373                 ecc->ccint = irq;
2374         }
2375
2376         irq = platform_get_irq_byname(pdev, "edma3_ccerrint");
2377         if (irq < 0 && node)
2378                 irq = irq_of_parse_and_map(node, 2);
2379
2380         if (irq >= 0) {
2381                 irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_ccerrint",
2382                                           dev_name(dev));
2383                 ret = devm_request_irq(dev, irq, dma_ccerr_handler, 0, irq_name,
2384                                        ecc);
2385                 if (ret) {
2386                         dev_err(dev, "CCERRINT (%d) failed --> %d\n", irq, ret);
2387                         return ret;
2388                 }
2389                 ecc->ccerrint = irq;
2390         }
2391
2392         ecc->dummy_slot = edma_alloc_slot(ecc, EDMA_SLOT_ANY);
2393         if (ecc->dummy_slot < 0) {
2394                 dev_err(dev, "Can't allocate PaRAM dummy slot\n");
2395                 return ecc->dummy_slot;
2396         }
2397
2398         queue_priority_mapping = info->queue_priority_mapping;
2399
2400         if (!ecc->legacy_mode) {
2401                 int lowest_priority = 0;
2402                 struct of_phandle_args tc_args;
2403
2404                 ecc->tc_list = devm_kcalloc(dev, ecc->num_tc,
2405                                             sizeof(*ecc->tc_list), GFP_KERNEL);
2406                 if (!ecc->tc_list)
2407                         return -ENOMEM;
2408
2409                 for (i = 0;; i++) {
2410                         ret = of_parse_phandle_with_fixed_args(node, "ti,tptcs",
2411                                                                1, i, &tc_args);
2412                         if (ret || i == ecc->num_tc)
2413                                 break;
2414
2415                         ecc->tc_list[i].node = tc_args.np;
2416                         ecc->tc_list[i].id = i;
2417                         queue_priority_mapping[i][1] = tc_args.args[0];
2418                         if (queue_priority_mapping[i][1] > lowest_priority) {
2419                                 lowest_priority = queue_priority_mapping[i][1];
2420                                 info->default_queue = i;
2421                         }
2422                 }
2423         }
2424
2425         /* Event queue priority mapping */
2426         for (i = 0; queue_priority_mapping[i][0] != -1; i++)
2427                 edma_assign_priority_to_queue(ecc, queue_priority_mapping[i][0],
2428                                               queue_priority_mapping[i][1]);
2429
2430         edma_write_array2(ecc, EDMA_DRAE, 0, 0, 0x0);
2431         edma_write_array2(ecc, EDMA_DRAE, 0, 1, 0x0);
2432         edma_write_array(ecc, EDMA_QRAE, 0, 0x0);
2433
2434         ecc->info = info;
2435
2436         /* Init the dma device and channels */
2437         edma_dma_init(ecc, legacy_mode);
2438
2439         for (i = 0; i < ecc->num_channels; i++) {
2440                 /* Assign all channels to the default queue */
2441                 edma_assign_channel_eventq(&ecc->slave_chans[i],
2442                                            info->default_queue);
2443                 /* Set entry slot to the dummy slot */
2444                 edma_set_chmap(&ecc->slave_chans[i], ecc->dummy_slot);
2445         }
2446
2447         ecc->dma_slave.filter.map = info->slave_map;
2448         ecc->dma_slave.filter.mapcnt = info->slavecnt;
2449         ecc->dma_slave.filter.fn = edma_filter_fn;
2450
2451         ret = dma_async_device_register(&ecc->dma_slave);
2452         if (ret) {
2453                 dev_err(dev, "slave ddev registration failed (%d)\n", ret);
2454                 goto err_reg1;
2455         }
2456
2457         if (ecc->dma_memcpy) {
2458                 ret = dma_async_device_register(ecc->dma_memcpy);
2459                 if (ret) {
2460                         dev_err(dev, "memcpy ddev registration failed (%d)\n",
2461                                 ret);
2462                         dma_async_device_unregister(&ecc->dma_slave);
2463                         goto err_reg1;
2464                 }
2465         }
2466
2467         if (node)
2468                 of_dma_controller_register(node, of_edma_xlate, ecc);
2469
2470         dev_info(dev, "TI EDMA DMA engine driver\n");
2471
2472         return 0;
2473
2474 err_reg1:
2475         edma_free_slot(ecc, ecc->dummy_slot);
2476         return ret;
2477 }
2478
2479 static void edma_cleanupp_vchan(struct dma_device *dmadev)
2480 {
2481         struct edma_chan *echan, *_echan;
2482
2483         list_for_each_entry_safe(echan, _echan,
2484                         &dmadev->channels, vchan.chan.device_node) {
2485                 list_del(&echan->vchan.chan.device_node);
2486                 tasklet_kill(&echan->vchan.task);
2487         }
2488 }
2489
2490 static int edma_remove(struct platform_device *pdev)
2491 {
2492         struct device *dev = &pdev->dev;
2493         struct edma_cc *ecc = dev_get_drvdata(dev);
2494
2495         devm_free_irq(dev, ecc->ccint, ecc);
2496         devm_free_irq(dev, ecc->ccerrint, ecc);
2497
2498         edma_cleanupp_vchan(&ecc->dma_slave);
2499
2500         if (dev->of_node)
2501                 of_dma_controller_free(dev->of_node);
2502         dma_async_device_unregister(&ecc->dma_slave);
2503         if (ecc->dma_memcpy)
2504                 dma_async_device_unregister(ecc->dma_memcpy);
2505         edma_free_slot(ecc, ecc->dummy_slot);
2506
2507         return 0;
2508 }
2509
2510 #ifdef CONFIG_PM_SLEEP
2511 static int edma_pm_suspend(struct device *dev)
2512 {
2513         struct edma_cc *ecc = dev_get_drvdata(dev);
2514         struct edma_chan *echan = ecc->slave_chans;
2515         int i;
2516
2517         for (i = 0; i < ecc->num_channels; i++) {
2518                 if (echan[i].alloced)
2519                         edma_setup_interrupt(&echan[i], false);
2520         }
2521
2522         return 0;
2523 }
2524
2525 static int edma_pm_resume(struct device *dev)
2526 {
2527         struct edma_cc *ecc = dev_get_drvdata(dev);
2528         struct edma_chan *echan = ecc->slave_chans;
2529         int i;
2530         s8 (*queue_priority_mapping)[2];
2531
2532         /* re initialize dummy slot to dummy param set */
2533         edma_write_slot(ecc, ecc->dummy_slot, &dummy_paramset);
2534
2535         queue_priority_mapping = ecc->info->queue_priority_mapping;
2536
2537         /* Event queue priority mapping */
2538         for (i = 0; queue_priority_mapping[i][0] != -1; i++)
2539                 edma_assign_priority_to_queue(ecc, queue_priority_mapping[i][0],
2540                                               queue_priority_mapping[i][1]);
2541
2542         for (i = 0; i < ecc->num_channels; i++) {
2543                 if (echan[i].alloced) {
2544                         /* ensure access through shadow region 0 */
2545                         edma_or_array2(ecc, EDMA_DRAE, 0,
2546                                        EDMA_REG_ARRAY_INDEX(i),
2547                                        EDMA_CHANNEL_BIT(i));
2548
2549                         edma_setup_interrupt(&echan[i], true);
2550
2551                         /* Set up channel -> slot mapping for the entry slot */
2552                         edma_set_chmap(&echan[i], echan[i].slot[0]);
2553                 }
2554         }
2555
2556         return 0;
2557 }
2558 #endif
2559
2560 static const struct dev_pm_ops edma_pm_ops = {
2561         SET_LATE_SYSTEM_SLEEP_PM_OPS(edma_pm_suspend, edma_pm_resume)
2562 };
2563
2564 static struct platform_driver edma_driver = {
2565         .probe          = edma_probe,
2566         .remove         = edma_remove,
2567         .driver = {
2568                 .name   = "edma",
2569                 .pm     = &edma_pm_ops,
2570                 .of_match_table = edma_of_ids,
2571         },
2572 };
2573
2574 static int edma_tptc_probe(struct platform_device *pdev)
2575 {
2576         pm_runtime_enable(&pdev->dev);
2577         return pm_runtime_get_sync(&pdev->dev);
2578 }
2579
2580 static struct platform_driver edma_tptc_driver = {
2581         .probe          = edma_tptc_probe,
2582         .driver = {
2583                 .name   = "edma3-tptc",
2584                 .of_match_table = edma_tptc_of_ids,
2585         },
2586 };
2587
2588 static bool edma_filter_fn(struct dma_chan *chan, void *param)
2589 {
2590         bool match = false;
2591
2592         if (chan->device->dev->driver == &edma_driver.driver) {
2593                 struct edma_chan *echan = to_edma_chan(chan);
2594                 unsigned ch_req = *(unsigned *)param;
2595                 if (ch_req == echan->ch_num) {
2596                         /* The channel is going to be used as HW synchronized */
2597                         echan->hw_triggered = true;
2598                         match = true;
2599                 }
2600         }
2601         return match;
2602 }
2603
2604 static int edma_init(void)
2605 {
2606         int ret;
2607
2608         ret = platform_driver_register(&edma_tptc_driver);
2609         if (ret)
2610                 return ret;
2611
2612         return platform_driver_register(&edma_driver);
2613 }
2614 subsys_initcall(edma_init);
2615
2616 static void __exit edma_exit(void)
2617 {
2618         platform_driver_unregister(&edma_driver);
2619         platform_driver_unregister(&edma_tptc_driver);
2620 }
2621 module_exit(edma_exit);
2622
2623 MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
2624 MODULE_DESCRIPTION("TI EDMA DMA engine driver");
2625 MODULE_LICENSE("GPL v2");