drivers/dma/sa11x0-dma.c

   1 /*
   2  * SA11x0 DMAengine support
   3  *
   4  * Copyright (C) 2012 Russell King
   5  *   Derived in part from arch/arm/mach-sa1100/dma.c,
   6  *   Copyright (C) 2000, 2001 by Nicolas Pitre
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as
  10  * published by the Free Software Foundation.
  11  */
  12 #include <linux/sched.h>
  13 #include <linux/device.h>
  14 #include <linux/dmaengine.h>
  15 #include <linux/init.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/kernel.h>
  18 #include <linux/module.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/slab.h>
  21 #include <linux/spinlock.h>
  22
  23 #include "virt-dma.h"
  24
  25 #define NR_PHY_CHAN     6
  26 #define DMA_ALIGN       3
  27 #define DMA_MAX_SIZE    0x1fff
  28 #define DMA_CHUNK_SIZE  0x1000
  29
  30 #define DMA_DDAR        0x00
  31 #define DMA_DCSR_S      0x04
  32 #define DMA_DCSR_C      0x08
  33 #define DMA_DCSR_R      0x0c
  34 #define DMA_DBSA        0x10
  35 #define DMA_DBTA        0x14
  36 #define DMA_DBSB        0x18
  37 #define DMA_DBTB        0x1c
  38 #define DMA_SIZE        0x20
  39
  40 #define DCSR_RUN        (1 << 0)
  41 #define DCSR_IE         (1 << 1)
  42 #define DCSR_ERROR      (1 << 2)
  43 #define DCSR_DONEA      (1 << 3)
  44 #define DCSR_STRTA      (1 << 4)
  45 #define DCSR_DONEB      (1 << 5)
  46 #define DCSR_STRTB      (1 << 6)
  47 #define DCSR_BIU        (1 << 7)
  48
  49 #define DDAR_RW         (1 << 0)        /* 0 = W, 1 = R */
  50 #define DDAR_E          (1 << 1)        /* 0 = LE, 1 = BE */
  51 #define DDAR_BS         (1 << 2)        /* 0 = BS4, 1 = BS8 */
  52 #define DDAR_DW         (1 << 3)        /* 0 = 8b, 1 = 16b */
  53 #define DDAR_Ser0UDCTr  (0x0 << 4)
  54 #define DDAR_Ser0UDCRc  (0x1 << 4)
  55 #define DDAR_Ser1SDLCTr (0x2 << 4)
  56 #define DDAR_Ser1SDLCRc (0x3 << 4)
  57 #define DDAR_Ser1UARTTr (0x4 << 4)
  58 #define DDAR_Ser1UARTRc (0x5 << 4)
  59 #define DDAR_Ser2ICPTr  (0x6 << 4)
  60 #define DDAR_Ser2ICPRc  (0x7 << 4)
  61 #define DDAR_Ser3UARTTr (0x8 << 4)
  62 #define DDAR_Ser3UARTRc (0x9 << 4)
  63 #define DDAR_Ser4MCP0Tr (0xa << 4)
  64 #define DDAR_Ser4MCP0Rc (0xb << 4)
  65 #define DDAR_Ser4MCP1Tr (0xc << 4)
  66 #define DDAR_Ser4MCP1Rc (0xd << 4)
  67 #define DDAR_Ser4SSPTr  (0xe << 4)
  68 #define DDAR_Ser4SSPRc  (0xf << 4)
  69
  70 struct sa11x0_dma_sg {
  71         u32                     addr;
  72         u32                     len;
  73 };
  74
  75 struct sa11x0_dma_desc {
  76         struct virt_dma_desc    vd;
  77
  78         u32                     ddar;
  79         size_t                  size;
  80         unsigned                period;
  81         bool                    cyclic;
  82
  83         unsigned                sglen;
  84         struct sa11x0_dma_sg    sg[0];
  85 };
  86
  87 struct sa11x0_dma_phy;
  88
  89 struct sa11x0_dma_chan {
  90         struct virt_dma_chan    vc;
  91
  92         /* protected by c->vc.lock */
  93         struct sa11x0_dma_phy   *phy;
  94         enum dma_status         status;
  95
  96         /* protected by d->lock */
  97         struct list_head        node;
  98
  99         u32                     ddar;
 100         const char              *name;
 101 };
 102
 103 struct sa11x0_dma_phy {
 104         void __iomem            *base;
 105         struct sa11x0_dma_dev   *dev;
 106         unsigned                num;
 107
 108         struct sa11x0_dma_chan  *vchan;
 109
 110         /* Protected by c->vc.lock */
 111         unsigned                sg_load;
 112         struct sa11x0_dma_desc  *txd_load;
 113         unsigned                sg_done;
 114         struct sa11x0_dma_desc  *txd_done;
 115         u32                     dbs[2];
 116         u32                     dbt[2];
 117         u32                     dcsr;
 118 };
 119
 120 struct sa11x0_dma_dev {
 121         struct dma_device       slave;
 122         void __iomem            *base;
 123         spinlock_t              lock;
 124         struct tasklet_struct   task;
 125         struct list_head        chan_pending;
 126         struct sa11x0_dma_phy   phy[NR_PHY_CHAN];
 127 };
 128
 129 static struct sa11x0_dma_chan *to_sa11x0_dma_chan(struct dma_chan *chan)
 130 {
 131         return container_of(chan, struct sa11x0_dma_chan, vc.chan);
 132 }
 133
 134 static struct sa11x0_dma_dev *to_sa11x0_dma(struct dma_device *dmadev)
 135 {
 136         return container_of(dmadev, struct sa11x0_dma_dev, slave);
 137 }
 138
 139 static struct sa11x0_dma_desc *sa11x0_dma_next_desc(struct sa11x0_dma_chan *c)
 140 {
 141         struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
 142
 143         return vd ? container_of(vd, struct sa11x0_dma_desc, vd) : NULL;
 144 }
 145
 146 static void sa11x0_dma_free_desc(struct virt_dma_desc *vd)
 147 {
 148         kfree(container_of(vd, struct sa11x0_dma_desc, vd));
 149 }
 150
 151 static void sa11x0_dma_start_desc(struct sa11x0_dma_phy *p, struct sa11x0_dma_desc *txd)
 152 {
 153         list_del(&txd->vd.node);
 154         p->txd_load = txd;
 155         p->sg_load = 0;
 156
 157         dev_vdbg(p->dev->slave.dev, "pchan %u: txd %p[%x]: starting: DDAR:%x\n",
 158                 p->num, &txd->vd, txd->vd.tx.cookie, txd->ddar);
 159 }
 160
 161 static void noinline sa11x0_dma_start_sg(struct sa11x0_dma_phy *p,
 162         struct sa11x0_dma_chan *c)
 163 {
 164         struct sa11x0_dma_desc *txd = p->txd_load;
 165         struct sa11x0_dma_sg *sg;
 166         void __iomem *base = p->base;
 167         unsigned dbsx, dbtx;
 168         u32 dcsr;
 169
 170         if (!txd)
 171                 return;
 172
 173         dcsr = readl_relaxed(base + DMA_DCSR_R);
 174
 175         /* Don't try to load the next transfer if both buffers are started */
 176         if ((dcsr & (DCSR_STRTA | DCSR_STRTB)) == (DCSR_STRTA | DCSR_STRTB))
 177                 return;
 178
 179         if (p->sg_load == txd->sglen) {
 180                 if (!txd->cyclic) {
 181                         struct sa11x0_dma_desc *txn = sa11x0_dma_next_desc(c);
 182
 183                         /*
 184                          * We have reached the end of the current descriptor.
 185                          * Peek at the next descriptor, and if compatible with
 186                          * the current, start processing it.
 187                          */
 188                         if (txn && txn->ddar == txd->ddar) {
 189                                 txd = txn;
 190                                 sa11x0_dma_start_desc(p, txn);
 191                         } else {
 192                                 p->txd_load = NULL;
 193                                 return;
 194                         }
 195                 } else {
 196                         /* Cyclic: reset back to beginning */
 197                         p->sg_load = 0;
 198                 }
 199         }
 200
 201         sg = &txd->sg[p->sg_load++];
 202
 203         /* Select buffer to load according to channel status */
 204         if (((dcsr & (DCSR_BIU | DCSR_STRTB)) == (DCSR_BIU | DCSR_STRTB)) ||
 205             ((dcsr & (DCSR_BIU | DCSR_STRTA)) == 0)) {
 206                 dbsx = DMA_DBSA;
 207                 dbtx = DMA_DBTA;
 208                 dcsr = DCSR_STRTA | DCSR_IE | DCSR_RUN;
 209         } else {
 210                 dbsx = DMA_DBSB;
 211                 dbtx = DMA_DBTB;
 212                 dcsr = DCSR_STRTB | DCSR_IE | DCSR_RUN;
 213         }
 214
 215         writel_relaxed(sg->addr, base + dbsx);
 216         writel_relaxed(sg->len, base + dbtx);
 217         writel(dcsr, base + DMA_DCSR_S);
 218
 219         dev_dbg(p->dev->slave.dev, "pchan %u: load: DCSR:%02x DBS%c:%08x DBT%c:%08x\n",
 220                 p->num, dcsr,
 221                 'A' + (dbsx == DMA_DBSB), sg->addr,
 222                 'A' + (dbtx == DMA_DBTB), sg->len);
 223 }
 224
 225 static void noinline sa11x0_dma_complete(struct sa11x0_dma_phy *p,
 226         struct sa11x0_dma_chan *c)
 227 {
 228         struct sa11x0_dma_desc *txd = p->txd_done;
 229
 230         if (++p->sg_done == txd->sglen) {
 231                 if (!txd->cyclic) {
 232                         vchan_cookie_complete(&txd->vd);
 233
 234                         p->sg_done = 0;
 235                         p->txd_done = p->txd_load;
 236
 237                         if (!p->txd_done)
 238                                 tasklet_schedule(&p->dev->task);
 239                 } else {
 240                         if ((p->sg_done % txd->period) == 0)
 241                                 vchan_cyclic_callback(&txd->vd);
 242
 243                         /* Cyclic: reset back to beginning */
 244                         p->sg_done = 0;
 245                 }
 246         }
 247
 248         sa11x0_dma_start_sg(p, c);
 249 }
 250
 251 static irqreturn_t sa11x0_dma_irq(int irq, void *dev_id)
 252 {
 253         struct sa11x0_dma_phy *p = dev_id;
 254         struct sa11x0_dma_dev *d = p->dev;
 255         struct sa11x0_dma_chan *c;
 256         u32 dcsr;
 257
 258         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
 259         if (!(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB)))
 260                 return IRQ_NONE;
 261
 262         /* Clear reported status bits */
 263         writel_relaxed(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB),
 264                 p->base + DMA_DCSR_C);
 265
 266         dev_dbg(d->slave.dev, "pchan %u: irq: DCSR:%02x\n", p->num, dcsr);
 267
 268         if (dcsr & DCSR_ERROR) {
 269                 dev_err(d->slave.dev, "pchan %u: error. DCSR:%02x DDAR:%08x DBSA:%08x DBTA:%08x DBSB:%08x DBTB:%08x\n",
 270                         p->num, dcsr,
 271                         readl_relaxed(p->base + DMA_DDAR),
 272                         readl_relaxed(p->base + DMA_DBSA),
 273                         readl_relaxed(p->base + DMA_DBTA),
 274                         readl_relaxed(p->base + DMA_DBSB),
 275                         readl_relaxed(p->base + DMA_DBTB));
 276         }
 277
 278         c = p->vchan;
 279         if (c) {
 280                 unsigned long flags;
 281
 282                 spin_lock_irqsave(&c->vc.lock, flags);
 283                 /*
 284                  * Now that we're holding the lock, check that the vchan
 285                  * really is associated with this pchan before touching the
 286                  * hardware.  This should always succeed, because we won't
 287                  * change p->vchan or c->phy while the channel is actively
 288                  * transferring.
 289                  */
 290                 if (c->phy == p) {
 291                         if (dcsr & DCSR_DONEA)
 292                                 sa11x0_dma_complete(p, c);
 293                         if (dcsr & DCSR_DONEB)
 294                                 sa11x0_dma_complete(p, c);
 295                 }
 296                 spin_unlock_irqrestore(&c->vc.lock, flags);
 297         }
 298
 299         return IRQ_HANDLED;
 300 }
 301
 302 static void sa11x0_dma_start_txd(struct sa11x0_dma_chan *c)
 303 {
 304         struct sa11x0_dma_desc *txd = sa11x0_dma_next_desc(c);
 305
 306         /* If the issued list is empty, we have no further txds to process */
 307         if (txd) {
 308                 struct sa11x0_dma_phy *p = c->phy;
 309
 310                 sa11x0_dma_start_desc(p, txd);
 311                 p->txd_done = txd;
 312                 p->sg_done = 0;
 313
 314                 /* The channel should not have any transfers started */
 315                 WARN_ON(readl_relaxed(p->base + DMA_DCSR_R) &
 316                                       (DCSR_STRTA | DCSR_STRTB));
 317
 318                 /* Clear the run and start bits before changing DDAR */
 319                 writel_relaxed(DCSR_RUN | DCSR_STRTA | DCSR_STRTB,
 320                                p->base + DMA_DCSR_C);
 321                 writel_relaxed(txd->ddar, p->base + DMA_DDAR);
 322
 323                 /* Try to start both buffers */
 324                 sa11x0_dma_start_sg(p, c);
 325                 sa11x0_dma_start_sg(p, c);
 326         }
 327 }
 328
 329 static void sa11x0_dma_tasklet(unsigned long arg)
 330 {
 331         struct sa11x0_dma_dev *d = (struct sa11x0_dma_dev *)arg;
 332         struct sa11x0_dma_phy *p;
 333         struct sa11x0_dma_chan *c;
 334         unsigned pch, pch_alloc = 0;
 335
 336         dev_dbg(d->slave.dev, "tasklet enter\n");
 337
 338         list_for_each_entry(c, &d->slave.channels, vc.chan.device_node) {
 339                 spin_lock_irq(&c->vc.lock);
 340                 p = c->phy;
 341                 if (p && !p->txd_done) {
 342                         sa11x0_dma_start_txd(c);
 343                         if (!p->txd_done) {
 344                                 /* No current txd associated with this channel */
 345                                 dev_dbg(d->slave.dev, "pchan %u: free\n", p->num);
 346
 347                                 /* Mark this channel free */
 348                                 c->phy = NULL;
 349                                 p->vchan = NULL;
 350                         }
 351                 }
 352                 spin_unlock_irq(&c->vc.lock);
 353         }
 354
 355         spin_lock_irq(&d->lock);
 356         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 357                 p = &d->phy[pch];
 358
 359                 if (p->vchan == NULL && !list_empty(&d->chan_pending)) {
 360                         c = list_first_entry(&d->chan_pending,
 361                                 struct sa11x0_dma_chan, node);
 362                         list_del_init(&c->node);
 363
 364                         pch_alloc |= 1 << pch;
 365
 366                         /* Mark this channel allocated */
 367                         p->vchan = c;
 368
 369                         dev_dbg(d->slave.dev, "pchan %u: alloc vchan %p\n", pch, &c->vc);
 370                 }
 371         }
 372         spin_unlock_irq(&d->lock);
 373
 374         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 375                 if (pch_alloc & (1 << pch)) {
 376                         p = &d->phy[pch];
 377                         c = p->vchan;
 378
 379                         spin_lock_irq(&c->vc.lock);
 380                         c->phy = p;
 381
 382                         sa11x0_dma_start_txd(c);
 383                         spin_unlock_irq(&c->vc.lock);
 384                 }
 385         }
 386
 387         dev_dbg(d->slave.dev, "tasklet exit\n");
 388 }
 389
 390
 391 static void sa11x0_dma_free_chan_resources(struct dma_chan *chan)
 392 {
 393         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 394         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 395         unsigned long flags;
 396
 397         spin_lock_irqsave(&d->lock, flags);
 398         list_del_init(&c->node);
 399         spin_unlock_irqrestore(&d->lock, flags);
 400
 401         vchan_free_chan_resources(&c->vc);
 402 }
 403
 404 static dma_addr_t sa11x0_dma_pos(struct sa11x0_dma_phy *p)
 405 {
 406         unsigned reg;
 407         u32 dcsr;
 408
 409         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
 410
 411         if ((dcsr & (DCSR_BIU | DCSR_STRTA)) == DCSR_STRTA ||
 412             (dcsr & (DCSR_BIU | DCSR_STRTB)) == DCSR_BIU)
 413                 reg = DMA_DBSA;
 414         else
 415                 reg = DMA_DBSB;
 416
 417         return readl_relaxed(p->base + reg);
 418 }
 419
 420 static enum dma_status sa11x0_dma_tx_status(struct dma_chan *chan,
 421         dma_cookie_t cookie, struct dma_tx_state *state)
 422 {
 423         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 424         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 425         struct sa11x0_dma_phy *p;
 426         struct virt_dma_desc *vd;
 427         unsigned long flags;
 428         enum dma_status ret;
 429
 430         ret = dma_cookie_status(&c->vc.chan, cookie, state);
 431         if (ret == DMA_COMPLETE)
 432                 return ret;
 433
 434         if (!state)
 435                 return c->status;
 436
 437         spin_lock_irqsave(&c->vc.lock, flags);
 438         p = c->phy;
 439
 440         /*
 441          * If the cookie is on our issue queue, then the residue is
 442          * its total size.
 443          */
 444         vd = vchan_find_desc(&c->vc, cookie);
 445         if (vd) {
 446                 state->residue = container_of(vd, struct sa11x0_dma_desc, vd)->size;
 447         } else if (!p) {
 448                 state->residue = 0;
 449         } else {
 450                 struct sa11x0_dma_desc *txd;
 451                 size_t bytes = 0;
 452
 453                 if (p->txd_done && p->txd_done->vd.tx.cookie == cookie)
 454                         txd = p->txd_done;
 455                 else if (p->txd_load && p->txd_load->vd.tx.cookie == cookie)
 456                         txd = p->txd_load;
 457                 else
 458                         txd = NULL;
 459
 460                 ret = c->status;
 461                 if (txd) {
 462                         dma_addr_t addr = sa11x0_dma_pos(p);
 463                         unsigned i;
 464
 465                         dev_vdbg(d->slave.dev, "tx_status: addr:%pad\n", &addr);
 466
 467                         for (i = 0; i < txd->sglen; i++) {
 468                                 dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x\n",
 469                                         i, txd->sg[i].addr, txd->sg[i].len);
 470                                 if (addr >= txd->sg[i].addr &&
 471                                     addr < txd->sg[i].addr + txd->sg[i].len) {
 472                                         unsigned len;
 473
 474                                         len = txd->sg[i].len -
 475                                                 (addr - txd->sg[i].addr);
 476                                         dev_vdbg(d->slave.dev, "tx_status: [%u] +%x\n",
 477                                                 i, len);
 478                                         bytes += len;
 479                                         i++;
 480                                         break;
 481                                 }
 482                         }
 483                         for (; i < txd->sglen; i++) {
 484                                 dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x ++\n",
 485                                         i, txd->sg[i].addr, txd->sg[i].len);
 486                                 bytes += txd->sg[i].len;
 487                         }
 488                 }
 489                 state->residue = bytes;
 490         }
 491         spin_unlock_irqrestore(&c->vc.lock, flags);
 492
 493         dev_vdbg(d->slave.dev, "tx_status: bytes 0x%x\n", state->residue);
 494
 495         return ret;
 496 }
 497
 498 /*
 499  * Move pending txds to the issued list, and re-init pending list.
 500  * If not already pending, add this channel to the list of pending
 501  * channels and trigger the tasklet to run.
 502  */
 503 static void sa11x0_dma_issue_pending(struct dma_chan *chan)
 504 {
 505         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 506         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 507         unsigned long flags;
 508
 509         spin_lock_irqsave(&c->vc.lock, flags);
 510         if (vchan_issue_pending(&c->vc)) {
 511                 if (!c->phy) {
 512                         spin_lock(&d->lock);
 513                         if (list_empty(&c->node)) {
 514                                 list_add_tail(&c->node, &d->chan_pending);
 515                                 tasklet_schedule(&d->task);
 516                                 dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc);
 517                         }
 518                         spin_unlock(&d->lock);
 519                 }
 520         } else
 521                 dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc);
 522         spin_unlock_irqrestore(&c->vc.lock, flags);
 523 }
 524
 525 static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg(
 526         struct dma_chan *chan, struct scatterlist *sg, unsigned int sglen,
 527         enum dma_transfer_direction dir, unsigned long flags, void *context)
 528 {
 529         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 530         struct sa11x0_dma_desc *txd;
 531         struct scatterlist *sgent;
 532         unsigned i, j = sglen;
 533         size_t size = 0;
 534
 535         /* SA11x0 channels can only operate in their native direction */
 536         if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
 537                 dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
 538                         &c->vc, c->ddar, dir);
 539                 return NULL;
 540         }
 541
 542         /* Do not allow zero-sized txds */
 543         if (sglen == 0)
 544                 return NULL;
 545
 546         for_each_sg(sg, sgent, sglen, i) {
 547                 dma_addr_t addr = sg_dma_address(sgent);
 548                 unsigned int len = sg_dma_len(sgent);
 549
 550                 if (len > DMA_MAX_SIZE)
 551                         j += DIV_ROUND_UP(len, DMA_MAX_SIZE & ~DMA_ALIGN) - 1;
 552                 if (addr & DMA_ALIGN) {
 553                         dev_dbg(chan->device->dev, "vchan %p: bad buffer alignment: %pad\n",
 554                                 &c->vc, &addr);
 555                         return NULL;
 556                 }
 557         }
 558
 559         txd = kzalloc(struct_size(txd, sg, j), GFP_ATOMIC);
 560         if (!txd) {
 561                 dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
 562                 return NULL;
 563         }
 564
 565         j = 0;
 566         for_each_sg(sg, sgent, sglen, i) {
 567                 dma_addr_t addr = sg_dma_address(sgent);
 568                 unsigned len = sg_dma_len(sgent);
 569
 570                 size += len;
 571
 572                 do {
 573                         unsigned tlen = len;
 574
 575                         /*
 576                          * Check whether the transfer will fit.  If not, try
 577                          * to split the transfer up such that we end up with
 578                          * equal chunks - but make sure that we preserve the
 579                          * alignment.  This avoids small segments.
 580                          */
 581                         if (tlen > DMA_MAX_SIZE) {
 582                                 unsigned mult = DIV_ROUND_UP(tlen,
 583                                         DMA_MAX_SIZE & ~DMA_ALIGN);
 584
 585                                 tlen = (tlen / mult) & ~DMA_ALIGN;
 586                         }
 587
 588                         txd->sg[j].addr = addr;
 589                         txd->sg[j].len = tlen;
 590
 591                         addr += tlen;
 592                         len -= tlen;
 593                         j++;
 594                 } while (len);
 595         }
 596
 597         txd->ddar = c->ddar;
 598         txd->size = size;
 599         txd->sglen = j;
 600
 601         dev_dbg(chan->device->dev, "vchan %p: txd %p: size %zu nr %u\n",
 602                 &c->vc, &txd->vd, txd->size, txd->sglen);
 603
 604         return vchan_tx_prep(&c->vc, &txd->vd, flags);
 605 }
 606
 607 static struct dma_async_tx_descriptor *sa11x0_dma_prep_dma_cyclic(
 608         struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period,
 609         enum dma_transfer_direction dir, unsigned long flags)
 610 {
 611         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 612         struct sa11x0_dma_desc *txd;
 613         unsigned i, j, k, sglen, sgperiod;
 614
 615         /* SA11x0 channels can only operate in their native direction */
 616         if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
 617                 dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
 618                         &c->vc, c->ddar, dir);
 619                 return NULL;
 620         }
 621
 622         sgperiod = DIV_ROUND_UP(period, DMA_MAX_SIZE & ~DMA_ALIGN);
 623         sglen = size * sgperiod / period;
 624
 625         /* Do not allow zero-sized txds */
 626         if (sglen == 0)
 627                 return NULL;
 628
 629         txd = kzalloc(struct_size(txd, sg, sglen), GFP_ATOMIC);
 630         if (!txd) {
 631                 dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
 632                 return NULL;
 633         }
 634
 635         for (i = k = 0; i < size / period; i++) {
 636                 size_t tlen, len = period;
 637
 638                 for (j = 0; j < sgperiod; j++, k++) {
 639                         tlen = len;
 640
 641                         if (tlen > DMA_MAX_SIZE) {
 642                                 unsigned mult = DIV_ROUND_UP(tlen, DMA_MAX_SIZE & ~DMA_ALIGN);
 643                                 tlen = (tlen / mult) & ~DMA_ALIGN;
 644                         }
 645
 646                         txd->sg[k].addr = addr;
 647                         txd->sg[k].len = tlen;
 648                         addr += tlen;
 649                         len -= tlen;
 650                 }
 651
 652                 WARN_ON(len != 0);
 653         }
 654
 655         WARN_ON(k != sglen);
 656
 657         txd->ddar = c->ddar;
 658         txd->size = size;
 659         txd->sglen = sglen;
 660         txd->cyclic = 1;
 661         txd->period = sgperiod;
 662
 663         return vchan_tx_prep(&c->vc, &txd->vd, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 664 }
 665
 666 static int sa11x0_dma_device_config(struct dma_chan *chan,
 667                                     struct dma_slave_config *cfg)
 668 {
 669         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 670         u32 ddar = c->ddar & ((0xf << 4) | DDAR_RW);
 671         dma_addr_t addr;
 672         enum dma_slave_buswidth width;
 673         u32 maxburst;
 674
 675         if (ddar & DDAR_RW) {
 676                 addr = cfg->src_addr;
 677                 width = cfg->src_addr_width;
 678                 maxburst = cfg->src_maxburst;
 679         } else {
 680                 addr = cfg->dst_addr;
 681                 width = cfg->dst_addr_width;
 682                 maxburst = cfg->dst_maxburst;
 683         }
 684
 685         if ((width != DMA_SLAVE_BUSWIDTH_1_BYTE &&
 686              width != DMA_SLAVE_BUSWIDTH_2_BYTES) ||
 687             (maxburst != 4 && maxburst != 8))
 688                 return -EINVAL;
 689
 690         if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
 691                 ddar |= DDAR_DW;
 692         if (maxburst == 8)
 693                 ddar |= DDAR_BS;
 694
 695         dev_dbg(c->vc.chan.device->dev, "vchan %p: dma_slave_config addr %pad width %u burst %u\n",
 696                 &c->vc, &addr, width, maxburst);
 697
 698         c->ddar = ddar | (addr & 0xf0000000) | (addr & 0x003ffffc) << 6;
 699
 700         return 0;
 701 }
 702
 703 static int sa11x0_dma_device_pause(struct dma_chan *chan)
 704 {
 705         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 706         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 707         struct sa11x0_dma_phy *p;
 708         LIST_HEAD(head);
 709         unsigned long flags;
 710
 711         dev_dbg(d->slave.dev, "vchan %p: pause\n", &c->vc);
 712         spin_lock_irqsave(&c->vc.lock, flags);
 713         if (c->status == DMA_IN_PROGRESS) {
 714                 c->status = DMA_PAUSED;
 715
 716                 p = c->phy;
 717                 if (p) {
 718                         writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
 719                 } else {
 720                         spin_lock(&d->lock);
 721                         list_del_init(&c->node);
 722                         spin_unlock(&d->lock);
 723                 }
 724         }
 725         spin_unlock_irqrestore(&c->vc.lock, flags);
 726
 727         return 0;
 728 }
 729
 730 static int sa11x0_dma_device_resume(struct dma_chan *chan)
 731 {
 732         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 733         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 734         struct sa11x0_dma_phy *p;
 735         LIST_HEAD(head);
 736         unsigned long flags;
 737
 738         dev_dbg(d->slave.dev, "vchan %p: resume\n", &c->vc);
 739         spin_lock_irqsave(&c->vc.lock, flags);
 740         if (c->status == DMA_PAUSED) {
 741                 c->status = DMA_IN_PROGRESS;
 742
 743                 p = c->phy;
 744                 if (p) {
 745                         writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_S);
 746                 } else if (!list_empty(&c->vc.desc_issued)) {
 747                         spin_lock(&d->lock);
 748                         list_add_tail(&c->node, &d->chan_pending);
 749                         spin_unlock(&d->lock);
 750                 }
 751         }
 752         spin_unlock_irqrestore(&c->vc.lock, flags);
 753
 754         return 0;
 755 }
 756
 757 static int sa11x0_dma_device_terminate_all(struct dma_chan *chan)
 758 {
 759         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 760         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 761         struct sa11x0_dma_phy *p;
 762         LIST_HEAD(head);
 763         unsigned long flags;
 764
 765         dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc);
 766         /* Clear the tx descriptor lists */
 767         spin_lock_irqsave(&c->vc.lock, flags);
 768         vchan_get_all_descriptors(&c->vc, &head);
 769
 770         p = c->phy;
 771         if (p) {
 772                 dev_dbg(d->slave.dev, "pchan %u: terminating\n", p->num);
 773                 /* vchan is assigned to a pchan - stop the channel */
 774                 writel(DCSR_RUN | DCSR_IE |
 775                        DCSR_STRTA | DCSR_DONEA |
 776                        DCSR_STRTB | DCSR_DONEB,
 777                        p->base + DMA_DCSR_C);
 778
 779                 if (p->txd_load) {
 780                         if (p->txd_load != p->txd_done)
 781                                 list_add_tail(&p->txd_load->vd.node, &head);
 782                         p->txd_load = NULL;
 783                 }
 784                 if (p->txd_done) {
 785                         list_add_tail(&p->txd_done->vd.node, &head);
 786                         p->txd_done = NULL;
 787                 }
 788                 c->phy = NULL;
 789                 spin_lock(&d->lock);
 790                 p->vchan = NULL;
 791                 spin_unlock(&d->lock);
 792                 tasklet_schedule(&d->task);
 793         }
 794         spin_unlock_irqrestore(&c->vc.lock, flags);
 795         vchan_dma_desc_free_list(&c->vc, &head);
 796
 797         return 0;
 798 }
 799
 800 struct sa11x0_dma_channel_desc {
 801         u32 ddar;
 802         const char *name;
 803 };
 804
 805 #define CD(d1, d2) { .ddar = DDAR_##d1 | d2, .name = #d1 }
 806 static const struct sa11x0_dma_channel_desc chan_desc[] = {
 807         CD(Ser0UDCTr, 0),
 808         CD(Ser0UDCRc, DDAR_RW),
 809         CD(Ser1SDLCTr, 0),
 810         CD(Ser1SDLCRc, DDAR_RW),
 811         CD(Ser1UARTTr, 0),
 812         CD(Ser1UARTRc, DDAR_RW),
 813         CD(Ser2ICPTr, 0),
 814         CD(Ser2ICPRc, DDAR_RW),
 815         CD(Ser3UARTTr, 0),
 816         CD(Ser3UARTRc, DDAR_RW),
 817         CD(Ser4MCP0Tr, 0),
 818         CD(Ser4MCP0Rc, DDAR_RW),
 819         CD(Ser4MCP1Tr, 0),
 820         CD(Ser4MCP1Rc, DDAR_RW),
 821         CD(Ser4SSPTr, 0),
 822         CD(Ser4SSPRc, DDAR_RW),
 823 };
 824
 825 static const struct dma_slave_map sa11x0_dma_map[] = {
 826         { "sa11x0-ir", "tx", "Ser2ICPTr" },
 827         { "sa11x0-ir", "rx", "Ser2ICPRc" },
 828         { "sa11x0-ssp", "tx", "Ser4SSPTr" },
 829         { "sa11x0-ssp", "rx", "Ser4SSPRc" },
 830 };
 831
 832 static bool sa11x0_dma_filter_fn(struct dma_chan *chan, void *param)
 833 {
 834         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 835         const char *p = param;
 836
 837         return !strcmp(c->name, p);
 838 }
 839
 840 static int sa11x0_dma_init_dmadev(struct dma_device *dmadev,
 841         struct device *dev)
 842 {
 843         unsigned i;
 844
 845         INIT_LIST_HEAD(&dmadev->channels);
 846         dmadev->dev = dev;
 847         dmadev->device_free_chan_resources = sa11x0_dma_free_chan_resources;
 848         dmadev->device_config = sa11x0_dma_device_config;
 849         dmadev->device_pause = sa11x0_dma_device_pause;
 850         dmadev->device_resume = sa11x0_dma_device_resume;
 851         dmadev->device_terminate_all = sa11x0_dma_device_terminate_all;
 852         dmadev->device_tx_status = sa11x0_dma_tx_status;
 853         dmadev->device_issue_pending = sa11x0_dma_issue_pending;
 854
 855         for (i = 0; i < ARRAY_SIZE(chan_desc); i++) {
 856                 struct sa11x0_dma_chan *c;
 857
 858                 c = kzalloc(sizeof(*c), GFP_KERNEL);
 859                 if (!c) {
 860                         dev_err(dev, "no memory for channel %u\n", i);
 861                         return -ENOMEM;
 862                 }
 863
 864                 c->status = DMA_IN_PROGRESS;
 865                 c->ddar = chan_desc[i].ddar;
 866                 c->name = chan_desc[i].name;
 867                 INIT_LIST_HEAD(&c->node);
 868
 869                 c->vc.desc_free = sa11x0_dma_free_desc;
 870                 vchan_init(&c->vc, dmadev);
 871         }
 872
 873         return dma_async_device_register(dmadev);
 874 }
 875
 876 static int sa11x0_dma_request_irq(struct platform_device *pdev, int nr,
 877         void *data)
 878 {
 879         int irq = platform_get_irq(pdev, nr);
 880
 881         if (irq <= 0)
 882                 return -ENXIO;
 883
 884         return request_irq(irq, sa11x0_dma_irq, 0, dev_name(&pdev->dev), data);
 885 }
 886
 887 static void sa11x0_dma_free_irq(struct platform_device *pdev, int nr,
 888         void *data)
 889 {
 890         int irq = platform_get_irq(pdev, nr);
 891         if (irq > 0)
 892                 free_irq(irq, data);
 893 }
 894
 895 static void sa11x0_dma_free_channels(struct dma_device *dmadev)
 896 {
 897         struct sa11x0_dma_chan *c, *cn;
 898
 899         list_for_each_entry_safe(c, cn, &dmadev->channels, vc.chan.device_node) {
 900                 list_del(&c->vc.chan.device_node);
 901                 tasklet_kill(&c->vc.task);
 902                 kfree(c);
 903         }
 904 }
 905
 906 static int sa11x0_dma_probe(struct platform_device *pdev)
 907 {
 908         struct sa11x0_dma_dev *d;
 909         struct resource *res;
 910         unsigned i;
 911         int ret;
 912
 913         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 914         if (!res)
 915                 return -ENXIO;
 916
 917         d = kzalloc(sizeof(*d), GFP_KERNEL);
 918         if (!d) {
 919                 ret = -ENOMEM;
 920                 goto err_alloc;
 921         }
 922
 923         spin_lock_init(&d->lock);
 924         INIT_LIST_HEAD(&d->chan_pending);
 925
 926         d->slave.filter.fn = sa11x0_dma_filter_fn;
 927         d->slave.filter.mapcnt = ARRAY_SIZE(sa11x0_dma_map);
 928         d->slave.filter.map = sa11x0_dma_map;
 929
 930         d->base = ioremap(res->start, resource_size(res));
 931         if (!d->base) {
 932                 ret = -ENOMEM;
 933                 goto err_ioremap;
 934         }
 935
 936         tasklet_init(&d->task, sa11x0_dma_tasklet, (unsigned long)d);
 937
 938         for (i = 0; i < NR_PHY_CHAN; i++) {
 939                 struct sa11x0_dma_phy *p = &d->phy[i];
 940
 941                 p->dev = d;
 942                 p->num = i;
 943                 p->base = d->base + i * DMA_SIZE;
 944                 writel_relaxed(DCSR_RUN | DCSR_IE | DCSR_ERROR |
 945                         DCSR_DONEA | DCSR_STRTA | DCSR_DONEB | DCSR_STRTB,
 946                         p->base + DMA_DCSR_C);
 947                 writel_relaxed(0, p->base + DMA_DDAR);
 948
 949                 ret = sa11x0_dma_request_irq(pdev, i, p);
 950                 if (ret) {
 951                         while (i) {
 952                                 i--;
 953                                 sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
 954                         }
 955                         goto err_irq;
 956                 }
 957         }
 958
 959         dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
 960         dma_cap_set(DMA_CYCLIC, d->slave.cap_mask);
 961         d->slave.device_prep_slave_sg = sa11x0_dma_prep_slave_sg;
 962         d->slave.device_prep_dma_cyclic = sa11x0_dma_prep_dma_cyclic;
 963         d->slave.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
 964         d->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
 965         d->slave.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
 966                                    BIT(DMA_SLAVE_BUSWIDTH_2_BYTES);
 967         d->slave.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
 968                                    BIT(DMA_SLAVE_BUSWIDTH_2_BYTES);
 969         ret = sa11x0_dma_init_dmadev(&d->slave, &pdev->dev);
 970         if (ret) {
 971                 dev_warn(d->slave.dev, "failed to register slave async device: %d\n",
 972                         ret);
 973                 goto err_slave_reg;
 974         }
 975
 976         platform_set_drvdata(pdev, d);
 977         return 0;
 978
 979  err_slave_reg:
 980         sa11x0_dma_free_channels(&d->slave);
 981         for (i = 0; i < NR_PHY_CHAN; i++)
 982                 sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
 983  err_irq:
 984         tasklet_kill(&d->task);
 985         iounmap(d->base);
 986  err_ioremap:
 987         kfree(d);
 988  err_alloc:
 989         return ret;
 990 }
 991
 992 static int sa11x0_dma_remove(struct platform_device *pdev)
 993 {
 994         struct sa11x0_dma_dev *d = platform_get_drvdata(pdev);
 995         unsigned pch;
 996
 997         dma_async_device_unregister(&d->slave);
 998
 999         sa11x0_dma_free_channels(&d->slave);
1000         for (pch = 0; pch < NR_PHY_CHAN; pch++)
1001                 sa11x0_dma_free_irq(pdev, pch, &d->phy[pch]);
1002         tasklet_kill(&d->task);
1003         iounmap(d->base);
1004         kfree(d);
1005
1006         return 0;
1007 }
1008
1009 static int sa11x0_dma_suspend(struct device *dev)
1010 {
1011         struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
1012         unsigned pch;
1013
1014         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
1015                 struct sa11x0_dma_phy *p = &d->phy[pch];
1016                 u32 dcsr, saved_dcsr;
1017
1018                 dcsr = saved_dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1019                 if (dcsr & DCSR_RUN) {
1020                         writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
1021                         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1022                 }
1023
1024                 saved_dcsr &= DCSR_RUN | DCSR_IE;
1025                 if (dcsr & DCSR_BIU) {
1026                         p->dbs[0] = readl_relaxed(p->base + DMA_DBSB);
1027                         p->dbt[0] = readl_relaxed(p->base + DMA_DBTB);
1028                         p->dbs[1] = readl_relaxed(p->base + DMA_DBSA);
1029                         p->dbt[1] = readl_relaxed(p->base + DMA_DBTA);
1030                         saved_dcsr |= (dcsr & DCSR_STRTA ? DCSR_STRTB : 0) |
1031                                       (dcsr & DCSR_STRTB ? DCSR_STRTA : 0);
1032                 } else {
1033                         p->dbs[0] = readl_relaxed(p->base + DMA_DBSA);
1034                         p->dbt[0] = readl_relaxed(p->base + DMA_DBTA);
1035                         p->dbs[1] = readl_relaxed(p->base + DMA_DBSB);
1036                         p->dbt[1] = readl_relaxed(p->base + DMA_DBTB);
1037                         saved_dcsr |= dcsr & (DCSR_STRTA | DCSR_STRTB);
1038                 }
1039                 p->dcsr = saved_dcsr;
1040
1041                 writel(DCSR_STRTA | DCSR_STRTB, p->base + DMA_DCSR_C);
1042         }
1043
1044         return 0;
1045 }
1046
1047 static int sa11x0_dma_resume(struct device *dev)
1048 {
1049         struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
1050         unsigned pch;
1051
1052         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
1053                 struct sa11x0_dma_phy *p = &d->phy[pch];
1054                 struct sa11x0_dma_desc *txd = NULL;
1055                 u32 dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1056
1057                 WARN_ON(dcsr & (DCSR_BIU | DCSR_STRTA | DCSR_STRTB | DCSR_RUN));
1058
1059                 if (p->txd_done)
1060                         txd = p->txd_done;
1061                 else if (p->txd_load)
1062                         txd = p->txd_load;
1063
1064                 if (!txd)
1065                         continue;
1066
1067                 writel_relaxed(txd->ddar, p->base + DMA_DDAR);
1068
1069                 writel_relaxed(p->dbs[0], p->base + DMA_DBSA);
1070                 writel_relaxed(p->dbt[0], p->base + DMA_DBTA);
1071                 writel_relaxed(p->dbs[1], p->base + DMA_DBSB);
1072                 writel_relaxed(p->dbt[1], p->base + DMA_DBTB);
1073                 writel_relaxed(p->dcsr, p->base + DMA_DCSR_S);
1074         }
1075
1076         return 0;
1077 }
1078
1079 static const struct dev_pm_ops sa11x0_dma_pm_ops = {
1080         .suspend_noirq = sa11x0_dma_suspend,
1081         .resume_noirq = sa11x0_dma_resume,
1082         .freeze_noirq = sa11x0_dma_suspend,
1083         .thaw_noirq = sa11x0_dma_resume,
1084         .poweroff_noirq = sa11x0_dma_suspend,
1085         .restore_noirq = sa11x0_dma_resume,
1086 };
1087
1088 static struct platform_driver sa11x0_dma_driver = {
1089         .driver = {
1090                 .name   = "sa11x0-dma",
1091                 .pm     = &sa11x0_dma_pm_ops,
1092         },
1093         .probe          = sa11x0_dma_probe,
1094         .remove         = sa11x0_dma_remove,
1095 };
1096
1097 static int __init sa11x0_dma_init(void)
1098 {
1099         return platform_driver_register(&sa11x0_dma_driver);
1100 }
1101 subsys_initcall(sa11x0_dma_init);
1102
1103 static void __exit sa11x0_dma_exit(void)
1104 {
1105         platform_driver_unregister(&sa11x0_dma_driver);
1106 }
1107 module_exit(sa11x0_dma_exit);
1108
1109 MODULE_AUTHOR("Russell King");
1110 MODULE_DESCRIPTION("SA-11x0 DMA driver");
1111 MODULE_LICENSE("GPL v2");
1112 MODULE_ALIAS("platform:sa11x0-dma");