drivers/dma/sun4i-dma.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2014 Emilio López
   4  * Emilio López <emilio@elopez.com.ar>
   5  */
   6
   7 #include <linux/bitmap.h>
   8 #include <linux/bitops.h>
   9 #include <linux/clk.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/dmaengine.h>
  12 #include <linux/dmapool.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/module.h>
  15 #include <linux/of_dma.h>
  16 #include <linux/platform_device.h>
  17 #include <linux/slab.h>
  18 #include <linux/spinlock.h>
  19
  20 #include "virt-dma.h"
  21
  22 /** Common macros to normal and dedicated DMA registers **/
  23
  24 #define SUN4I_DMA_CFG_LOADING                   BIT(31)
  25 #define SUN4I_DMA_CFG_DST_DATA_WIDTH(width)     ((width) << 25)
  26 #define SUN4I_DMA_CFG_DST_BURST_LENGTH(len)     ((len) << 23)
  27 #define SUN4I_DMA_CFG_DST_ADDR_MODE(mode)       ((mode) << 21)
  28 #define SUN4I_DMA_CFG_DST_DRQ_TYPE(type)        ((type) << 16)
  29 #define SUN4I_DMA_CFG_SRC_DATA_WIDTH(width)     ((width) << 9)
  30 #define SUN4I_DMA_CFG_SRC_BURST_LENGTH(len)     ((len) << 7)
  31 #define SUN4I_DMA_CFG_SRC_ADDR_MODE(mode)       ((mode) << 5)
  32 #define SUN4I_DMA_CFG_SRC_DRQ_TYPE(type)        (type)
  33
  34 /** Normal DMA register values **/
  35
  36 /* Normal DMA source/destination data request type values */
  37 #define SUN4I_NDMA_DRQ_TYPE_SDRAM               0x16
  38 #define SUN4I_NDMA_DRQ_TYPE_LIMIT               (0x1F + 1)
  39
  40 /** Normal DMA register layout **/
  41
  42 /* Dedicated DMA source/destination address mode values */
  43 #define SUN4I_NDMA_ADDR_MODE_LINEAR             0
  44 #define SUN4I_NDMA_ADDR_MODE_IO                 1
  45
  46 /* Normal DMA configuration register layout */
  47 #define SUN4I_NDMA_CFG_CONT_MODE                BIT(30)
  48 #define SUN4I_NDMA_CFG_WAIT_STATE(n)            ((n) << 27)
  49 #define SUN4I_NDMA_CFG_DST_NON_SECURE           BIT(22)
  50 #define SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN   BIT(15)
  51 #define SUN4I_NDMA_CFG_SRC_NON_SECURE           BIT(6)
  52
  53 /** Dedicated DMA register values **/
  54
  55 /* Dedicated DMA source/destination address mode values */
  56 #define SUN4I_DDMA_ADDR_MODE_LINEAR             0
  57 #define SUN4I_DDMA_ADDR_MODE_IO                 1
  58 #define SUN4I_DDMA_ADDR_MODE_HORIZONTAL_PAGE    2
  59 #define SUN4I_DDMA_ADDR_MODE_VERTICAL_PAGE      3
  60
  61 /* Dedicated DMA source/destination data request type values */
  62 #define SUN4I_DDMA_DRQ_TYPE_SDRAM               0x1
  63 #define SUN4I_DDMA_DRQ_TYPE_LIMIT               (0x1F + 1)
  64
  65 /** Dedicated DMA register layout **/
  66
  67 /* Dedicated DMA configuration register layout */
  68 #define SUN4I_DDMA_CFG_BUSY                     BIT(30)
  69 #define SUN4I_DDMA_CFG_CONT_MODE                BIT(29)
  70 #define SUN4I_DDMA_CFG_DST_NON_SECURE           BIT(28)
  71 #define SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN   BIT(15)
  72 #define SUN4I_DDMA_CFG_SRC_NON_SECURE           BIT(12)
  73
  74 /* Dedicated DMA parameter register layout */
  75 #define SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(n)    (((n) - 1) << 24)
  76 #define SUN4I_DDMA_PARA_DST_WAIT_CYCLES(n)      (((n) - 1) << 16)
  77 #define SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(n)    (((n) - 1) << 8)
  78 #define SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(n)      (((n) - 1) << 0)
  79
  80 /** DMA register offsets **/
  81
  82 /* General register offsets */
  83 #define SUN4I_DMA_IRQ_ENABLE_REG                0x0
  84 #define SUN4I_DMA_IRQ_PENDING_STATUS_REG        0x4
  85
  86 /* Normal DMA register offsets */
  87 #define SUN4I_NDMA_CHANNEL_REG_BASE(n)          (0x100 + (n) * 0x20)
  88 #define SUN4I_NDMA_CFG_REG                      0x0
  89 #define SUN4I_NDMA_SRC_ADDR_REG                 0x4
  90 #define SUN4I_NDMA_DST_ADDR_REG         0x8
  91 #define SUN4I_NDMA_BYTE_COUNT_REG               0xC
  92
  93 /* Dedicated DMA register offsets */
  94 #define SUN4I_DDMA_CHANNEL_REG_BASE(n)          (0x300 + (n) * 0x20)
  95 #define SUN4I_DDMA_CFG_REG                      0x0
  96 #define SUN4I_DDMA_SRC_ADDR_REG                 0x4
  97 #define SUN4I_DDMA_DST_ADDR_REG         0x8
  98 #define SUN4I_DDMA_BYTE_COUNT_REG               0xC
  99 #define SUN4I_DDMA_PARA_REG                     0x18
 100
 101 /** DMA Driver **/
 102
 103 /*
 104  * Normal DMA has 8 channels, and Dedicated DMA has another 8, so
 105  * that's 16 channels. As for endpoints, there's 29 and 21
 106  * respectively. Given that the Normal DMA endpoints (other than
 107  * SDRAM) can be used as tx/rx, we need 78 vchans in total
 108  */
 109 #define SUN4I_NDMA_NR_MAX_CHANNELS      8
 110 #define SUN4I_DDMA_NR_MAX_CHANNELS      8
 111 #define SUN4I_DMA_NR_MAX_CHANNELS                                       \
 112         (SUN4I_NDMA_NR_MAX_CHANNELS + SUN4I_DDMA_NR_MAX_CHANNELS)
 113 #define SUN4I_NDMA_NR_MAX_VCHANS        (29 * 2 - 1)
 114 #define SUN4I_DDMA_NR_MAX_VCHANS        21
 115 #define SUN4I_DMA_NR_MAX_VCHANS                                         \
 116         (SUN4I_NDMA_NR_MAX_VCHANS + SUN4I_DDMA_NR_MAX_VCHANS)
 117
 118 /* This set of SUN4I_DDMA timing parameters were found experimentally while
 119  * working with the SPI driver and seem to make it behave correctly */
 120 #define SUN4I_DDMA_MAGIC_SPI_PARAMETERS \
 121         (SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(1) |                 \
 122          SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(1) |                         \
 123          SUN4I_DDMA_PARA_DST_WAIT_CYCLES(2) |                           \
 124          SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(2))
 125
 126 /*
 127  * Normal DMA supports individual transfers (segments) up to 128k.
 128  * Dedicated DMA supports transfers up to 16M. We can only report
 129  * one size limit, so we have to use the smaller value.
 130  */
 131 #define SUN4I_NDMA_MAX_SEG_SIZE         SZ_128K
 132 #define SUN4I_DDMA_MAX_SEG_SIZE         SZ_16M
 133 #define SUN4I_DMA_MAX_SEG_SIZE          SUN4I_NDMA_MAX_SEG_SIZE
 134
 135 struct sun4i_dma_pchan {
 136         /* Register base of channel */
 137         void __iomem                    *base;
 138         /* vchan currently being serviced */
 139         struct sun4i_dma_vchan          *vchan;
 140         /* Is this a dedicated pchan? */
 141         int                             is_dedicated;
 142 };
 143
 144 struct sun4i_dma_vchan {
 145         struct virt_dma_chan            vc;
 146         struct dma_slave_config         cfg;
 147         struct sun4i_dma_pchan          *pchan;
 148         struct sun4i_dma_promise        *processing;
 149         struct sun4i_dma_contract       *contract;
 150         u8                              endpoint;
 151         int                             is_dedicated;
 152 };
 153
 154 struct sun4i_dma_promise {
 155         u32                             cfg;
 156         u32                             para;
 157         dma_addr_t                      src;
 158         dma_addr_t                      dst;
 159         size_t                          len;
 160         struct list_head                list;
 161 };
 162
 163 /* A contract is a set of promises */
 164 struct sun4i_dma_contract {
 165         struct virt_dma_desc            vd;
 166         struct list_head                demands;
 167         struct list_head                completed_demands;
 168         bool                            is_cyclic : 1;
 169         bool                            use_half_int : 1;
 170 };
 171
 172 struct sun4i_dma_dev {
 173         DECLARE_BITMAP(pchans_used, SUN4I_DMA_NR_MAX_CHANNELS);
 174         struct dma_device               slave;
 175         struct sun4i_dma_pchan          *pchans;
 176         struct sun4i_dma_vchan          *vchans;
 177         void __iomem                    *base;
 178         struct clk                      *clk;
 179         int                             irq;
 180         spinlock_t                      lock;
 181 };
 182
 183 static struct sun4i_dma_dev *to_sun4i_dma_dev(struct dma_device *dev)
 184 {
 185         return container_of(dev, struct sun4i_dma_dev, slave);
 186 }
 187
 188 static struct sun4i_dma_vchan *to_sun4i_dma_vchan(struct dma_chan *chan)
 189 {
 190         return container_of(chan, struct sun4i_dma_vchan, vc.chan);
 191 }
 192
 193 static struct sun4i_dma_contract *to_sun4i_dma_contract(struct virt_dma_desc *vd)
 194 {
 195         return container_of(vd, struct sun4i_dma_contract, vd);
 196 }
 197
 198 static struct device *chan2dev(struct dma_chan *chan)
 199 {
 200         return &chan->dev->device;
 201 }
 202
 203 static int convert_burst(u32 maxburst)
 204 {
 205         if (maxburst > 8)
 206                 return -EINVAL;
 207
 208         /* 1 -> 0, 4 -> 1, 8 -> 2 */
 209         return (maxburst >> 2);
 210 }
 211
 212 static int convert_buswidth(enum dma_slave_buswidth addr_width)
 213 {
 214         if (addr_width > DMA_SLAVE_BUSWIDTH_4_BYTES)
 215                 return -EINVAL;
 216
 217         /* 8 (1 byte) -> 0, 16 (2 bytes) -> 1, 32 (4 bytes) -> 2 */
 218         return (addr_width >> 1);
 219 }
 220
 221 static void sun4i_dma_free_chan_resources(struct dma_chan *chan)
 222 {
 223         struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
 224
 225         vchan_free_chan_resources(&vchan->vc);
 226 }
 227
 228 static struct sun4i_dma_pchan *find_and_use_pchan(struct sun4i_dma_dev *priv,
 229                                                   struct sun4i_dma_vchan *vchan)
 230 {
 231         struct sun4i_dma_pchan *pchan = NULL, *pchans = priv->pchans;
 232         unsigned long flags;
 233         int i, max;
 234
 235         /*
 236          * pchans 0-SUN4I_NDMA_NR_MAX_CHANNELS are normal, and
 237          * SUN4I_NDMA_NR_MAX_CHANNELS+ are dedicated ones
 238          */
 239         if (vchan->is_dedicated) {
 240                 i = SUN4I_NDMA_NR_MAX_CHANNELS;
 241                 max = SUN4I_DMA_NR_MAX_CHANNELS;
 242         } else {
 243                 i = 0;
 244                 max = SUN4I_NDMA_NR_MAX_CHANNELS;
 245         }
 246
 247         spin_lock_irqsave(&priv->lock, flags);
 248         for_each_clear_bit_from(i, priv->pchans_used, max) {
 249                 pchan = &pchans[i];
 250                 pchan->vchan = vchan;
 251                 set_bit(i, priv->pchans_used);
 252                 break;
 253         }
 254         spin_unlock_irqrestore(&priv->lock, flags);
 255
 256         return pchan;
 257 }
 258
 259 static void release_pchan(struct sun4i_dma_dev *priv,
 260                           struct sun4i_dma_pchan *pchan)
 261 {
 262         unsigned long flags;
 263         int nr = pchan - priv->pchans;
 264
 265         spin_lock_irqsave(&priv->lock, flags);
 266
 267         pchan->vchan = NULL;
 268         clear_bit(nr, priv->pchans_used);
 269
 270         spin_unlock_irqrestore(&priv->lock, flags);
 271 }
 272
 273 static void configure_pchan(struct sun4i_dma_pchan *pchan,
 274                             struct sun4i_dma_promise *d)
 275 {
 276         /*
 277          * Configure addresses and misc parameters depending on type
 278          * SUN4I_DDMA has an extra field with timing parameters
 279          */
 280         if (pchan->is_dedicated) {
 281                 writel_relaxed(d->src, pchan->base + SUN4I_DDMA_SRC_ADDR_REG);
 282                 writel_relaxed(d->dst, pchan->base + SUN4I_DDMA_DST_ADDR_REG);
 283                 writel_relaxed(d->len, pchan->base + SUN4I_DDMA_BYTE_COUNT_REG);
 284                 writel_relaxed(d->para, pchan->base + SUN4I_DDMA_PARA_REG);
 285                 writel_relaxed(d->cfg, pchan->base + SUN4I_DDMA_CFG_REG);
 286         } else {
 287                 writel_relaxed(d->src, pchan->base + SUN4I_NDMA_SRC_ADDR_REG);
 288                 writel_relaxed(d->dst, pchan->base + SUN4I_NDMA_DST_ADDR_REG);
 289                 writel_relaxed(d->len, pchan->base + SUN4I_NDMA_BYTE_COUNT_REG);
 290                 writel_relaxed(d->cfg, pchan->base + SUN4I_NDMA_CFG_REG);
 291         }
 292 }
 293
 294 static void set_pchan_interrupt(struct sun4i_dma_dev *priv,
 295                                 struct sun4i_dma_pchan *pchan,
 296                                 int half, int end)
 297 {
 298         u32 reg;
 299         int pchan_number = pchan - priv->pchans;
 300         unsigned long flags;
 301
 302         spin_lock_irqsave(&priv->lock, flags);
 303
 304         reg = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
 305
 306         if (half)
 307                 reg |= BIT(pchan_number * 2);
 308         else
 309                 reg &= ~BIT(pchan_number * 2);
 310
 311         if (end)
 312                 reg |= BIT(pchan_number * 2 + 1);
 313         else
 314                 reg &= ~BIT(pchan_number * 2 + 1);
 315
 316         writel_relaxed(reg, priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
 317
 318         spin_unlock_irqrestore(&priv->lock, flags);
 319 }
 320
 321 /*
 322  * Execute pending operations on a vchan
 323  *
 324  * When given a vchan, this function will try to acquire a suitable
 325  * pchan and, if successful, will configure it to fulfill a promise
 326  * from the next pending contract.
 327  *
 328  * This function must be called with &vchan->vc.lock held.
 329  */
 330 static int __execute_vchan_pending(struct sun4i_dma_dev *priv,
 331                                    struct sun4i_dma_vchan *vchan)
 332 {
 333         struct sun4i_dma_promise *promise = NULL;
 334         struct sun4i_dma_contract *contract = NULL;
 335         struct sun4i_dma_pchan *pchan;
 336         struct virt_dma_desc *vd;
 337         int ret;
 338
 339         lockdep_assert_held(&vchan->vc.lock);
 340
 341         /* We need a pchan to do anything, so secure one if available */
 342         pchan = find_and_use_pchan(priv, vchan);
 343         if (!pchan)
 344                 return -EBUSY;
 345
 346         /*
 347          * Channel endpoints must not be repeated, so if this vchan
 348          * has already submitted some work, we can't do anything else
 349          */
 350         if (vchan->processing) {
 351                 dev_dbg(chan2dev(&vchan->vc.chan),
 352                         "processing something to this endpoint already\n");
 353                 ret = -EBUSY;
 354                 goto release_pchan;
 355         }
 356
 357         do {
 358                 /* Figure out which contract we're working with today */
 359                 vd = vchan_next_desc(&vchan->vc);
 360                 if (!vd) {
 361                         dev_dbg(chan2dev(&vchan->vc.chan),
 362                                 "No pending contract found");
 363                         ret = 0;
 364                         goto release_pchan;
 365                 }
 366
 367                 contract = to_sun4i_dma_contract(vd);
 368                 if (list_empty(&contract->demands)) {
 369                         /* The contract has been completed so mark it as such */
 370                         list_del(&contract->vd.node);
 371                         vchan_cookie_complete(&contract->vd);
 372                         dev_dbg(chan2dev(&vchan->vc.chan),
 373                                 "Empty contract found and marked complete");
 374                 }
 375         } while (list_empty(&contract->demands));
 376
 377         /* Now find out what we need to do */
 378         promise = list_first_entry(&contract->demands,
 379                                    struct sun4i_dma_promise, list);
 380         vchan->processing = promise;
 381
 382         /* ... and make it reality */
 383         if (promise) {
 384                 vchan->contract = contract;
 385                 vchan->pchan = pchan;
 386                 set_pchan_interrupt(priv, pchan, contract->use_half_int, 1);
 387                 configure_pchan(pchan, promise);
 388         }
 389
 390         return 0;
 391
 392 release_pchan:
 393         release_pchan(priv, pchan);
 394         return ret;
 395 }
 396
 397 static int sanitize_config(struct dma_slave_config *sconfig,
 398                            enum dma_transfer_direction direction)
 399 {
 400         switch (direction) {
 401         case DMA_MEM_TO_DEV:
 402                 if ((sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) ||
 403                     !sconfig->dst_maxburst)
 404                         return -EINVAL;
 405
 406                 if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
 407                         sconfig->src_addr_width = sconfig->dst_addr_width;
 408
 409                 if (!sconfig->src_maxburst)
 410                         sconfig->src_maxburst = sconfig->dst_maxburst;
 411
 412                 break;
 413
 414         case DMA_DEV_TO_MEM:
 415                 if ((sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) ||
 416                     !sconfig->src_maxburst)
 417                         return -EINVAL;
 418
 419                 if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
 420                         sconfig->dst_addr_width = sconfig->src_addr_width;
 421
 422                 if (!sconfig->dst_maxburst)
 423                         sconfig->dst_maxburst = sconfig->src_maxburst;
 424
 425                 break;
 426         default:
 427                 return 0;
 428         }
 429
 430         return 0;
 431 }
 432
 433 /*
 434  * Generate a promise, to be used in a normal DMA contract.
 435  *
 436  * A NDMA promise contains all the information required to program the
 437  * normal part of the DMA Engine and get data copied. A non-executed
 438  * promise will live in the demands list on a contract. Once it has been
 439  * completed, it will be moved to the completed demands list for later freeing.
 440  * All linked promises will be freed when the corresponding contract is freed
 441  */
 442 static struct sun4i_dma_promise *
 443 generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest,
 444                       size_t len, struct dma_slave_config *sconfig,
 445                       enum dma_transfer_direction direction)
 446 {
 447         struct sun4i_dma_promise *promise;
 448         int ret;
 449
 450         ret = sanitize_config(sconfig, direction);
 451         if (ret)
 452                 return NULL;
 453
 454         promise = kzalloc(sizeof(*promise), GFP_NOWAIT);
 455         if (!promise)
 456                 return NULL;
 457
 458         promise->src = src;
 459         promise->dst = dest;
 460         promise->len = len;
 461         promise->cfg = SUN4I_DMA_CFG_LOADING |
 462                 SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN;
 463
 464         dev_dbg(chan2dev(chan),
 465                 "src burst %d, dst burst %d, src buswidth %d, dst buswidth %d",
 466                 sconfig->src_maxburst, sconfig->dst_maxburst,
 467                 sconfig->src_addr_width, sconfig->dst_addr_width);
 468
 469         /* Source burst */
 470         ret = convert_burst(sconfig->src_maxburst);
 471         if (ret < 0)
 472                 goto fail;
 473         promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret);
 474
 475         /* Destination burst */
 476         ret = convert_burst(sconfig->dst_maxburst);
 477         if (ret < 0)
 478                 goto fail;
 479         promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret);
 480
 481         /* Source bus width */
 482         ret = convert_buswidth(sconfig->src_addr_width);
 483         if (ret < 0)
 484                 goto fail;
 485         promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret);
 486
 487         /* Destination bus width */
 488         ret = convert_buswidth(sconfig->dst_addr_width);
 489         if (ret < 0)
 490                 goto fail;
 491         promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret);
 492
 493         return promise;
 494
 495 fail:
 496         kfree(promise);
 497         return NULL;
 498 }
 499
 500 /*
 501  * Generate a promise, to be used in a dedicated DMA contract.
 502  *
 503  * A DDMA promise contains all the information required to program the
 504  * Dedicated part of the DMA Engine and get data copied. A non-executed
 505  * promise will live in the demands list on a contract. Once it has been
 506  * completed, it will be moved to the completed demands list for later freeing.
 507  * All linked promises will be freed when the corresponding contract is freed
 508  */
 509 static struct sun4i_dma_promise *
 510 generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest,
 511                       size_t len, struct dma_slave_config *sconfig)
 512 {
 513         struct sun4i_dma_promise *promise;
 514         int ret;
 515
 516         promise = kzalloc(sizeof(*promise), GFP_NOWAIT);
 517         if (!promise)
 518                 return NULL;
 519
 520         promise->src = src;
 521         promise->dst = dest;
 522         promise->len = len;
 523         promise->cfg = SUN4I_DMA_CFG_LOADING |
 524                 SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN;
 525
 526         /* Source burst */
 527         ret = convert_burst(sconfig->src_maxburst);
 528         if (ret < 0)
 529                 goto fail;
 530         promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret);
 531
 532         /* Destination burst */
 533         ret = convert_burst(sconfig->dst_maxburst);
 534         if (ret < 0)
 535                 goto fail;
 536         promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret);
 537
 538         /* Source bus width */
 539         ret = convert_buswidth(sconfig->src_addr_width);
 540         if (ret < 0)
 541                 goto fail;
 542         promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret);
 543
 544         /* Destination bus width */
 545         ret = convert_buswidth(sconfig->dst_addr_width);
 546         if (ret < 0)
 547                 goto fail;
 548         promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret);
 549
 550         return promise;
 551
 552 fail:
 553         kfree(promise);
 554         return NULL;
 555 }
 556
 557 /*
 558  * Generate a contract
 559  *
 560  * Contracts function as DMA descriptors. As our hardware does not support
 561  * linked lists, we need to implement SG via software. We use a contract
 562  * to hold all the pieces of the request and process them serially one
 563  * after another. Each piece is represented as a promise.
 564  */
 565 static struct sun4i_dma_contract *generate_dma_contract(void)
 566 {
 567         struct sun4i_dma_contract *contract;
 568
 569         contract = kzalloc(sizeof(*contract), GFP_NOWAIT);
 570         if (!contract)
 571                 return NULL;
 572
 573         INIT_LIST_HEAD(&contract->demands);
 574         INIT_LIST_HEAD(&contract->completed_demands);
 575
 576         return contract;
 577 }
 578
 579 /*
 580  * Get next promise on a cyclic transfer
 581  *
 582  * Cyclic contracts contain a series of promises which are executed on a
 583  * loop. This function returns the next promise from a cyclic contract,
 584  * so it can be programmed into the hardware.
 585  */
 586 static struct sun4i_dma_promise *
 587 get_next_cyclic_promise(struct sun4i_dma_contract *contract)
 588 {
 589         struct sun4i_dma_promise *promise;
 590
 591         promise = list_first_entry_or_null(&contract->demands,
 592                                            struct sun4i_dma_promise, list);
 593         if (!promise) {
 594                 list_splice_init(&contract->completed_demands,
 595                                  &contract->demands);
 596                 promise = list_first_entry(&contract->demands,
 597                                            struct sun4i_dma_promise, list);
 598         }
 599
 600         return promise;
 601 }
 602
 603 /*
 604  * Free a contract and all its associated promises
 605  */
 606 static void sun4i_dma_free_contract(struct virt_dma_desc *vd)
 607 {
 608         struct sun4i_dma_contract *contract = to_sun4i_dma_contract(vd);
 609         struct sun4i_dma_promise *promise, *tmp;
 610
 611         /* Free all the demands and completed demands */
 612         list_for_each_entry_safe(promise, tmp, &contract->demands, list)
 613                 kfree(promise);
 614
 615         list_for_each_entry_safe(promise, tmp, &contract->completed_demands, list)
 616                 kfree(promise);
 617
 618         kfree(contract);
 619 }
 620
 621 static struct dma_async_tx_descriptor *
 622 sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest,
 623                           dma_addr_t src, size_t len, unsigned long flags)
 624 {
 625         struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
 626         struct dma_slave_config *sconfig = &vchan->cfg;
 627         struct sun4i_dma_promise *promise;
 628         struct sun4i_dma_contract *contract;
 629
 630         contract = generate_dma_contract();
 631         if (!contract)
 632                 return NULL;
 633
 634         /*
 635          * We can only do the copy to bus aligned addresses, so
 636          * choose the best one so we get decent performance. We also
 637          * maximize the burst size for this same reason.
 638          */
 639         sconfig->src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 640         sconfig->dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 641         sconfig->src_maxburst = 8;
 642         sconfig->dst_maxburst = 8;
 643
 644         if (vchan->is_dedicated)
 645                 promise = generate_ddma_promise(chan, src, dest, len, sconfig);
 646         else
 647                 promise = generate_ndma_promise(chan, src, dest, len, sconfig,
 648                                                 DMA_MEM_TO_MEM);
 649
 650         if (!promise) {
 651                 kfree(contract);
 652                 return NULL;
 653         }
 654
 655         /* Configure memcpy mode */
 656         if (vchan->is_dedicated) {
 657                 promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM) |
 658                                 SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM);
 659         } else {
 660                 promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM) |
 661                                 SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM);
 662         }
 663
 664         /* Fill the contract with our only promise */
 665         list_add_tail(&promise->list, &contract->demands);
 666
 667         /* And add it to the vchan */
 668         return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
 669 }
 670
 671 static struct dma_async_tx_descriptor *
 672 sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len,
 673                           size_t period_len, enum dma_transfer_direction dir,
 674                           unsigned long flags)
 675 {
 676         struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
 677         struct dma_slave_config *sconfig = &vchan->cfg;
 678         struct sun4i_dma_promise *promise;
 679         struct sun4i_dma_contract *contract;
 680         dma_addr_t src, dest;
 681         u32 endpoints;
 682         int nr_periods, offset, plength, i;
 683         u8 ram_type, io_mode, linear_mode;
 684
 685         if (!is_slave_direction(dir)) {
 686                 dev_err(chan2dev(chan), "Invalid DMA direction\n");
 687                 return NULL;
 688         }
 689
 690         contract = generate_dma_contract();
 691         if (!contract)
 692                 return NULL;
 693
 694         contract->is_cyclic = 1;
 695
 696         if (vchan->is_dedicated) {
 697                 io_mode = SUN4I_DDMA_ADDR_MODE_IO;
 698                 linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR;
 699                 ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM;
 700         } else {
 701                 io_mode = SUN4I_NDMA_ADDR_MODE_IO;
 702                 linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR;
 703                 ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM;
 704         }
 705
 706         if (dir == DMA_MEM_TO_DEV) {
 707                 src = buf;
 708                 dest = sconfig->dst_addr;
 709                 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
 710                             SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) |
 711                             SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) |
 712                             SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode);
 713         } else {
 714                 src = sconfig->src_addr;
 715                 dest = buf;
 716                 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) |
 717                             SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) |
 718                             SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
 719                             SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode);
 720         }
 721
 722         /*
 723          * We will be using half done interrupts to make two periods
 724          * out of a promise, so we need to program the DMA engine less
 725          * often
 726          */
 727
 728         /*
 729          * The engine can interrupt on half-transfer, so we can use
 730          * this feature to program the engine half as often as if we
 731          * didn't use it (keep in mind the hardware doesn't support
 732          * linked lists).
 733          *
 734          * Say you have a set of periods (| marks the start/end, I for
 735          * interrupt, P for programming the engine to do a new
 736          * transfer), the easy but slow way would be to do
 737          *
 738          *  |---|---|---|---| (periods / promises)
 739          *  P  I,P I,P I,P  I
 740          *
 741          * Using half transfer interrupts you can do
 742          *
 743          *  |-------|-------| (promises as configured on hw)
 744          *  |---|---|---|---| (periods)
 745          *  P   I  I,P  I   I
 746          *
 747          * Which requires half the engine programming for the same
 748          * functionality.
 749          *
 750          * This only works if two periods fit in a single promise. That will
 751          * always be the case for dedicated DMA, where the hardware has a much
 752          * larger maximum transfer size than advertised to clients.
 753          */
 754         if (vchan->is_dedicated || period_len <= SUN4I_NDMA_MAX_SEG_SIZE / 2) {
 755                 period_len *= 2;
 756                 contract->use_half_int = 1;
 757         }
 758
 759         nr_periods = DIV_ROUND_UP(len, period_len);
 760         for (i = 0; i < nr_periods; i++) {
 761                 /* Calculate the offset in the buffer and the length needed */
 762                 offset = i * period_len;
 763                 plength = min((len - offset), period_len);
 764                 if (dir == DMA_MEM_TO_DEV)
 765                         src = buf + offset;
 766                 else
 767                         dest = buf + offset;
 768
 769                 /* Make the promise */
 770                 if (vchan->is_dedicated)
 771                         promise = generate_ddma_promise(chan, src, dest,
 772                                                         plength, sconfig);
 773                 else
 774                         promise = generate_ndma_promise(chan, src, dest,
 775                                                         plength, sconfig, dir);
 776
 777                 if (!promise) {
 778                         /* TODO: should we free everything? */
 779                         return NULL;
 780                 }
 781                 promise->cfg |= endpoints;
 782
 783                 /* Then add it to the contract */
 784                 list_add_tail(&promise->list, &contract->demands);
 785         }
 786
 787         /* And add it to the vchan */
 788         return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
 789 }
 790
 791 static struct dma_async_tx_descriptor *
 792 sun4i_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 793                         unsigned int sg_len, enum dma_transfer_direction dir,
 794                         unsigned long flags, void *context)
 795 {
 796         struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
 797         struct dma_slave_config *sconfig = &vchan->cfg;
 798         struct sun4i_dma_promise *promise;
 799         struct sun4i_dma_contract *contract;
 800         u8 ram_type, io_mode, linear_mode;
 801         struct scatterlist *sg;
 802         dma_addr_t srcaddr, dstaddr;
 803         u32 endpoints, para;
 804         int i;
 805
 806         if (!sgl)
 807                 return NULL;
 808
 809         if (!is_slave_direction(dir)) {
 810                 dev_err(chan2dev(chan), "Invalid DMA direction\n");
 811                 return NULL;
 812         }
 813
 814         contract = generate_dma_contract();
 815         if (!contract)
 816                 return NULL;
 817
 818         if (vchan->is_dedicated) {
 819                 io_mode = SUN4I_DDMA_ADDR_MODE_IO;
 820                 linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR;
 821                 ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM;
 822         } else {
 823                 io_mode = SUN4I_NDMA_ADDR_MODE_IO;
 824                 linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR;
 825                 ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM;
 826         }
 827
 828         if (dir == DMA_MEM_TO_DEV)
 829                 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
 830                             SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) |
 831                             SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) |
 832                             SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode);
 833         else
 834                 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) |
 835                             SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) |
 836                             SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
 837                             SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode);
 838
 839         for_each_sg(sgl, sg, sg_len, i) {
 840                 /* Figure out addresses */
 841                 if (dir == DMA_MEM_TO_DEV) {
 842                         srcaddr = sg_dma_address(sg);
 843                         dstaddr = sconfig->dst_addr;
 844                 } else {
 845                         srcaddr = sconfig->src_addr;
 846                         dstaddr = sg_dma_address(sg);
 847                 }
 848
 849                 /*
 850                  * These are the magic DMA engine timings that keep SPI going.
 851                  * I haven't seen any interface on DMAEngine to configure
 852                  * timings, and so far they seem to work for everything we
 853                  * support, so I've kept them here. I don't know if other
 854                  * devices need different timings because, as usual, we only
 855                  * have the "para" bitfield meanings, but no comment on what
 856                  * the values should be when doing a certain operation :|
 857                  */
 858                 para = SUN4I_DDMA_MAGIC_SPI_PARAMETERS;
 859
 860                 /* And make a suitable promise */
 861                 if (vchan->is_dedicated)
 862                         promise = generate_ddma_promise(chan, srcaddr, dstaddr,
 863                                                         sg_dma_len(sg),
 864                                                         sconfig);
 865                 else
 866                         promise = generate_ndma_promise(chan, srcaddr, dstaddr,
 867                                                         sg_dma_len(sg),
 868                                                         sconfig, dir);
 869
 870                 if (!promise)
 871                         return NULL; /* TODO: should we free everything? */
 872
 873                 promise->cfg |= endpoints;
 874                 promise->para = para;
 875
 876                 /* Then add it to the contract */
 877                 list_add_tail(&promise->list, &contract->demands);
 878         }
 879
 880         /*
 881          * Once we've got all the promises ready, add the contract
 882          * to the pending list on the vchan
 883          */
 884         return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
 885 }
 886
 887 static int sun4i_dma_terminate_all(struct dma_chan *chan)
 888 {
 889         struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
 890         struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
 891         struct sun4i_dma_pchan *pchan = vchan->pchan;
 892         LIST_HEAD(head);
 893         unsigned long flags;
 894
 895         spin_lock_irqsave(&vchan->vc.lock, flags);
 896         vchan_get_all_descriptors(&vchan->vc, &head);
 897         spin_unlock_irqrestore(&vchan->vc.lock, flags);
 898
 899         /*
 900          * Clearing the configuration register will halt the pchan. Interrupts
 901          * may still trigger, so don't forget to disable them.
 902          */
 903         if (pchan) {
 904                 if (pchan->is_dedicated)
 905                         writel(0, pchan->base + SUN4I_DDMA_CFG_REG);
 906                 else
 907                         writel(0, pchan->base + SUN4I_NDMA_CFG_REG);
 908                 set_pchan_interrupt(priv, pchan, 0, 0);
 909                 release_pchan(priv, pchan);
 910         }
 911
 912         spin_lock_irqsave(&vchan->vc.lock, flags);
 913         /* Clear these so the vchan is usable again */
 914         vchan->processing = NULL;
 915         vchan->pchan = NULL;
 916         spin_unlock_irqrestore(&vchan->vc.lock, flags);
 917
 918         vchan_dma_desc_free_list(&vchan->vc, &head);
 919
 920         return 0;
 921 }
 922
 923 static int sun4i_dma_config(struct dma_chan *chan,
 924                             struct dma_slave_config *config)
 925 {
 926         struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
 927
 928         memcpy(&vchan->cfg, config, sizeof(*config));
 929
 930         return 0;
 931 }
 932
 933 static struct dma_chan *sun4i_dma_of_xlate(struct of_phandle_args *dma_spec,
 934                                            struct of_dma *ofdma)
 935 {
 936         struct sun4i_dma_dev *priv = ofdma->of_dma_data;
 937         struct sun4i_dma_vchan *vchan;
 938         struct dma_chan *chan;
 939         u8 is_dedicated = dma_spec->args[0];
 940         u8 endpoint = dma_spec->args[1];
 941
 942         /* Check if type is Normal or Dedicated */
 943         if (is_dedicated != 0 && is_dedicated != 1)
 944                 return NULL;
 945
 946         /* Make sure the endpoint looks sane */
 947         if ((is_dedicated && endpoint >= SUN4I_DDMA_DRQ_TYPE_LIMIT) ||
 948             (!is_dedicated && endpoint >= SUN4I_NDMA_DRQ_TYPE_LIMIT))
 949                 return NULL;
 950
 951         chan = dma_get_any_slave_channel(&priv->slave);
 952         if (!chan)
 953                 return NULL;
 954
 955         /* Assign the endpoint to the vchan */
 956         vchan = to_sun4i_dma_vchan(chan);
 957         vchan->is_dedicated = is_dedicated;
 958         vchan->endpoint = endpoint;
 959
 960         return chan;
 961 }
 962
 963 static enum dma_status sun4i_dma_tx_status(struct dma_chan *chan,
 964                                            dma_cookie_t cookie,
 965                                            struct dma_tx_state *state)
 966 {
 967         struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
 968         struct sun4i_dma_pchan *pchan = vchan->pchan;
 969         struct sun4i_dma_contract *contract;
 970         struct sun4i_dma_promise *promise;
 971         struct virt_dma_desc *vd;
 972         unsigned long flags;
 973         enum dma_status ret;
 974         size_t bytes = 0;
 975
 976         ret = dma_cookie_status(chan, cookie, state);
 977         if (!state || (ret == DMA_COMPLETE))
 978                 return ret;
 979
 980         spin_lock_irqsave(&vchan->vc.lock, flags);
 981         vd = vchan_find_desc(&vchan->vc, cookie);
 982         if (!vd)
 983                 goto exit;
 984         contract = to_sun4i_dma_contract(vd);
 985
 986         list_for_each_entry(promise, &contract->demands, list)
 987                 bytes += promise->len;
 988
 989         /*
 990          * The hardware is configured to return the remaining byte
 991          * quantity. If possible, replace the first listed element's
 992          * full size with the actual remaining amount
 993          */
 994         promise = list_first_entry_or_null(&contract->demands,
 995                                            struct sun4i_dma_promise, list);
 996         if (promise && pchan) {
 997                 bytes -= promise->len;
 998                 if (pchan->is_dedicated)
 999                         bytes += readl(pchan->base + SUN4I_DDMA_BYTE_COUNT_REG);
1000                 else
1001                         bytes += readl(pchan->base + SUN4I_NDMA_BYTE_COUNT_REG);
1002         }
1003
1004 exit:
1005
1006         dma_set_residue(state, bytes);
1007         spin_unlock_irqrestore(&vchan->vc.lock, flags);
1008
1009         return ret;
1010 }
1011
1012 static void sun4i_dma_issue_pending(struct dma_chan *chan)
1013 {
1014         struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
1015         struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
1016         unsigned long flags;
1017
1018         spin_lock_irqsave(&vchan->vc.lock, flags);
1019
1020         /*
1021          * If there are pending transactions for this vchan, push one of
1022          * them into the engine to get the ball rolling.
1023          */
1024         if (vchan_issue_pending(&vchan->vc))
1025                 __execute_vchan_pending(priv, vchan);
1026
1027         spin_unlock_irqrestore(&vchan->vc.lock, flags);
1028 }
1029
1030 static irqreturn_t sun4i_dma_interrupt(int irq, void *dev_id)
1031 {
1032         struct sun4i_dma_dev *priv = dev_id;
1033         struct sun4i_dma_pchan *pchans = priv->pchans, *pchan;
1034         struct sun4i_dma_vchan *vchan;
1035         struct sun4i_dma_contract *contract;
1036         struct sun4i_dma_promise *promise;
1037         unsigned long pendirq, irqs, disableirqs;
1038         int bit, i, free_room, allow_mitigation = 1;
1039
1040         pendirq = readl_relaxed(priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1041
1042 handle_pending:
1043
1044         disableirqs = 0;
1045         free_room = 0;
1046
1047         for_each_set_bit(bit, &pendirq, 32) {
1048                 pchan = &pchans[bit >> 1];
1049                 vchan = pchan->vchan;
1050                 if (!vchan) /* a terminated channel may still interrupt */
1051                         continue;
1052                 contract = vchan->contract;
1053
1054                 /*
1055                  * Disable the IRQ and free the pchan if it's an end
1056                  * interrupt (odd bit)
1057                  */
1058                 if (bit & 1) {
1059                         spin_lock(&vchan->vc.lock);
1060
1061                         /*
1062                          * Move the promise into the completed list now that
1063                          * we're done with it
1064                          */
1065                         list_move_tail(&vchan->processing->list,
1066                                        &contract->completed_demands);
1067
1068                         /*
1069                          * Cyclic DMA transfers are special:
1070                          * - There's always something we can dispatch
1071                          * - We need to run the callback
1072                          * - Latency is very important, as this is used by audio
1073                          * We therefore just cycle through the list and dispatch
1074                          * whatever we have here, reusing the pchan. There's
1075                          * no need to run the thread after this.
1076                          *
1077                          * For non-cyclic transfers we need to look around,
1078                          * so we can program some more work, or notify the
1079                          * client that their transfers have been completed.
1080                          */
1081                         if (contract->is_cyclic) {
1082                                 promise = get_next_cyclic_promise(contract);
1083                                 vchan->processing = promise;
1084                                 configure_pchan(pchan, promise);
1085                                 vchan_cyclic_callback(&contract->vd);
1086                         } else {
1087                                 vchan->processing = NULL;
1088                                 vchan->pchan = NULL;
1089
1090                                 free_room = 1;
1091                                 disableirqs |= BIT(bit);
1092                                 release_pchan(priv, pchan);
1093                         }
1094
1095                         spin_unlock(&vchan->vc.lock);
1096                 } else {
1097                         /* Half done interrupt */
1098                         if (contract->is_cyclic)
1099                                 vchan_cyclic_callback(&contract->vd);
1100                         else
1101                                 disableirqs |= BIT(bit);
1102                 }
1103         }
1104
1105         /* Disable the IRQs for events we handled */
1106         spin_lock(&priv->lock);
1107         irqs = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1108         writel_relaxed(irqs & ~disableirqs,
1109                        priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1110         spin_unlock(&priv->lock);
1111
1112         /* Writing 1 to the pending field will clear the pending interrupt */
1113         writel_relaxed(pendirq, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1114
1115         /*
1116          * If a pchan was freed, we may be able to schedule something else,
1117          * so have a look around
1118          */
1119         if (free_room) {
1120                 for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) {
1121                         vchan = &priv->vchans[i];
1122                         spin_lock(&vchan->vc.lock);
1123                         __execute_vchan_pending(priv, vchan);
1124                         spin_unlock(&vchan->vc.lock);
1125                 }
1126         }
1127
1128         /*
1129          * Handle newer interrupts if some showed up, but only do it once
1130          * to avoid a too long a loop
1131          */
1132         if (allow_mitigation) {
1133                 pendirq = readl_relaxed(priv->base +
1134                                         SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1135                 if (pendirq) {
1136                         allow_mitigation = 0;
1137                         goto handle_pending;
1138                 }
1139         }
1140
1141         return IRQ_HANDLED;
1142 }
1143
1144 static int sun4i_dma_probe(struct platform_device *pdev)
1145 {
1146         struct sun4i_dma_dev *priv;
1147         int i, j, ret;
1148
1149         priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
1150         if (!priv)
1151                 return -ENOMEM;
1152
1153         priv->base = devm_platform_ioremap_resource(pdev, 0);
1154         if (IS_ERR(priv->base))
1155                 return PTR_ERR(priv->base);
1156
1157         priv->irq = platform_get_irq(pdev, 0);
1158         if (priv->irq < 0)
1159                 return priv->irq;
1160
1161         priv->clk = devm_clk_get(&pdev->dev, NULL);
1162         if (IS_ERR(priv->clk)) {
1163                 dev_err(&pdev->dev, "No clock specified\n");
1164                 return PTR_ERR(priv->clk);
1165         }
1166
1167         platform_set_drvdata(pdev, priv);
1168         spin_lock_init(&priv->lock);
1169
1170         dma_set_max_seg_size(&pdev->dev, SUN4I_DMA_MAX_SEG_SIZE);
1171
1172         dma_cap_zero(priv->slave.cap_mask);
1173         dma_cap_set(DMA_PRIVATE, priv->slave.cap_mask);
1174         dma_cap_set(DMA_MEMCPY, priv->slave.cap_mask);
1175         dma_cap_set(DMA_CYCLIC, priv->slave.cap_mask);
1176         dma_cap_set(DMA_SLAVE, priv->slave.cap_mask);
1177
1178         INIT_LIST_HEAD(&priv->slave.channels);
1179         priv->slave.device_free_chan_resources  = sun4i_dma_free_chan_resources;
1180         priv->slave.device_tx_status            = sun4i_dma_tx_status;
1181         priv->slave.device_issue_pending        = sun4i_dma_issue_pending;
1182         priv->slave.device_prep_slave_sg        = sun4i_dma_prep_slave_sg;
1183         priv->slave.device_prep_dma_memcpy      = sun4i_dma_prep_dma_memcpy;
1184         priv->slave.device_prep_dma_cyclic      = sun4i_dma_prep_dma_cyclic;
1185         priv->slave.device_config               = sun4i_dma_config;
1186         priv->slave.device_terminate_all        = sun4i_dma_terminate_all;
1187         priv->slave.copy_align                  = 2;
1188         priv->slave.src_addr_widths             = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1189                                                   BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1190                                                   BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1191         priv->slave.dst_addr_widths             = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1192                                                   BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1193                                                   BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1194         priv->slave.directions                  = BIT(DMA_DEV_TO_MEM) |
1195                                                   BIT(DMA_MEM_TO_DEV);
1196         priv->slave.residue_granularity         = DMA_RESIDUE_GRANULARITY_BURST;
1197
1198         priv->slave.dev = &pdev->dev;
1199
1200         priv->pchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_CHANNELS,
1201                                     sizeof(struct sun4i_dma_pchan), GFP_KERNEL);
1202         priv->vchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_VCHANS,
1203                                     sizeof(struct sun4i_dma_vchan), GFP_KERNEL);
1204         if (!priv->vchans || !priv->pchans)
1205                 return -ENOMEM;
1206
1207         /*
1208          * [0..SUN4I_NDMA_NR_MAX_CHANNELS) are normal pchans, and
1209          * [SUN4I_NDMA_NR_MAX_CHANNELS..SUN4I_DMA_NR_MAX_CHANNELS) are
1210          * dedicated ones
1211          */
1212         for (i = 0; i < SUN4I_NDMA_NR_MAX_CHANNELS; i++)
1213                 priv->pchans[i].base = priv->base +
1214                         SUN4I_NDMA_CHANNEL_REG_BASE(i);
1215
1216         for (j = 0; i < SUN4I_DMA_NR_MAX_CHANNELS; i++, j++) {
1217                 priv->pchans[i].base = priv->base +
1218                         SUN4I_DDMA_CHANNEL_REG_BASE(j);
1219                 priv->pchans[i].is_dedicated = 1;
1220         }
1221
1222         for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) {
1223                 struct sun4i_dma_vchan *vchan = &priv->vchans[i];
1224
1225                 spin_lock_init(&vchan->vc.lock);
1226                 vchan->vc.desc_free = sun4i_dma_free_contract;
1227                 vchan_init(&vchan->vc, &priv->slave);
1228         }
1229
1230         ret = clk_prepare_enable(priv->clk);
1231         if (ret) {
1232                 dev_err(&pdev->dev, "Couldn't enable the clock\n");
1233                 return ret;
1234         }
1235
1236         /*
1237          * Make sure the IRQs are all disabled and accounted for. The bootloader
1238          * likes to leave these dirty
1239          */
1240         writel(0, priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1241         writel(0xFFFFFFFF, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1242
1243         ret = devm_request_irq(&pdev->dev, priv->irq, sun4i_dma_interrupt,
1244                                0, dev_name(&pdev->dev), priv);
1245         if (ret) {
1246                 dev_err(&pdev->dev, "Cannot request IRQ\n");
1247                 goto err_clk_disable;
1248         }
1249
1250         ret = dma_async_device_register(&priv->slave);
1251         if (ret) {
1252                 dev_warn(&pdev->dev, "Failed to register DMA engine device\n");
1253                 goto err_clk_disable;
1254         }
1255
1256         ret = of_dma_controller_register(pdev->dev.of_node, sun4i_dma_of_xlate,
1257                                          priv);
1258         if (ret) {
1259                 dev_err(&pdev->dev, "of_dma_controller_register failed\n");
1260                 goto err_dma_unregister;
1261         }
1262
1263         dev_dbg(&pdev->dev, "Successfully probed SUN4I_DMA\n");
1264
1265         return 0;
1266
1267 err_dma_unregister:
1268         dma_async_device_unregister(&priv->slave);
1269 err_clk_disable:
1270         clk_disable_unprepare(priv->clk);
1271         return ret;
1272 }
1273
1274 static void sun4i_dma_remove(struct platform_device *pdev)
1275 {
1276         struct sun4i_dma_dev *priv = platform_get_drvdata(pdev);
1277
1278         /* Disable IRQ so no more work is scheduled */
1279         disable_irq(priv->irq);
1280
1281         of_dma_controller_free(pdev->dev.of_node);
1282         dma_async_device_unregister(&priv->slave);
1283
1284         clk_disable_unprepare(priv->clk);
1285 }
1286
1287 static const struct of_device_id sun4i_dma_match[] = {
1288         { .compatible = "allwinner,sun4i-a10-dma" },
1289         { /* sentinel */ },
1290 };
1291 MODULE_DEVICE_TABLE(of, sun4i_dma_match);
1292
1293 static struct platform_driver sun4i_dma_driver = {
1294         .probe  = sun4i_dma_probe,
1295         .remove_new = sun4i_dma_remove,
1296         .driver = {
1297                 .name           = "sun4i-dma",
1298                 .of_match_table = sun4i_dma_match,
1299         },
1300 };
1301
1302 module_platform_driver(sun4i_dma_driver);
1303
1304 MODULE_DESCRIPTION("Allwinner A10 Dedicated DMA Controller Driver");
1305 MODULE_AUTHOR("Emilio López <emilio@elopez.com.ar>");
1306 MODULE_LICENSE("GPL");